############################### # # robots.txt file for this website # unique and interesting # addresses all robots by using wild card * # User-agent: ia_archiver Disallow: / User-agent: Titan Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-Agent: Scooter Disallow: / User-agent: * # list folders robots are not allowed to index # Disallow: /images Disallow: /_private Disallow: /books Disallow: /bbd Disallow: /admin # # list specific files robots are not allowed to index # Disallow: /contactcomplete.asp # # # # End of robots.txt file # ###############################