# Disallow wget downloads User-agent: wget Disallow: / # Disallow files on Google User-agent: Googlebot Crawl-delay: 2 Disallow: /1/nieuws/ Disallow: /comics/erotic_comics.htm Disallow: /news/ Disallow: /news/pdf/*.pdf Disallow: /*.phtml$ # Disallow Google-Image User-agent: Googlebot-Image Crawl-delay: 3 Disallow: /comics/erotic/ Disallow: /expositions/ruijters_expo_2008/photos/ Disallow: /bits/*.gif # Disallow files on MSN User-agent: msnbot Crawl-delay: 5 Disallow: /1/nieuws/ Disallow: /news/ Disallow: /news/pdf/*.pdf Disallow: /*.phtml$ # Disallow files on Yahoo! Slurp User-agent: Slurp Crawl-delay: 7 Disallow: /1/nieuws/ Disallow: /news/ Disallow: /*.phtml$ # Disallow files on all other crawlers User-agent: * Crawl-delay: 11 Disallow: /1/nieuws/ Disallow: /aanvang/2000nawoord.htm Disallow: /bits/ Disallow: /cgi-bin/ Disallow: /comics/erotic_comics.htm Disallow: /news/pdf/ Disallow: /search/ Disallow: /store/ Disallow: /payment.htm Disallow: /expositions/ruijters_expo_2008/photos/ Disallow: /*/index.phtml$