# Robots.txt for http://www.opiates.com # # Last edited 09/28/2008 12:30pm Luis Mosquera # Block spiders from all kinds of places # they shouldn't be # User-agent: * Disallow: /classes/ Disallow: /contact/ Disallow: /css/ Disallow: /functions/ Disallow: /includes/ Disallow: /monthly-reports/ Disallow: /NewsAbstractsTextFiles/ Disallow: /plesk-stat/ Disallow: /search/sql/ # Disallow: /ShockwaveLogos/ Disallow: /swap/ # disallow all files ending with these extensions disallow: /*.php$ disallow: /*.js$ disallow: /*.inc$ disallow: /*.css$ disallow: /*.txt$ disallow: /*.sql$ # allow google image bot to search all images User-agent: Googlebot-Image disallow: Allow: /*.gif$ Allow: /*.png$ Allow: /*.jpeg$ Allow: /*.jpg$ Allow: /*.ico$ Allow: /*.jpg$ Allow: /images/ # # Nasty spiders that I don't want # visiting if I can control it # User-agent: Wget Disallow: / # User-agent: Lynx Disallow: / # User-agent: Curl Disallow: / # User-agent: libwww-perl Disallow: / # User-agent: EmailSiphon Disallow: / # User-agent: EmailWolf Disallow: / # User-agent: ExtractorPro Disallow: / # User-agent: Crescent Disallow: / # User-agent: CherryPicker Disallow: / # User-agent: [Ww]eb[Bb]andit Disallow: / # User-agent: WebEMailExtrac.* Disallow: / # User-agent: NICErsPRO Disallow: / # User-agent: Telesoft Disallow: / # User-agent: Zeus.*Webster Disallow: / # User-agent: EmailCollector Disallow: / # User-agent: internetseer.com Disallow: / # User-agent: InternetSeer.com Disallow: / # User-agent: sitecheck.internetseer.com Disallow: /