# MarketStreetMagic.com robots.txt # the links page is NOT excluded from any SE spiders # let's both save bandwidth ... # all bots are excluded from the /jsfiles directory (see below) # Some bots will be excluded from certain files User-agent: * Disallow: /administration.htm Disallow: /jsfiles/ User-agent: scooter Disallow: /auto-accessories.htm User-agent: googlebot Disallow: /auto-accessories.htm User-agent: Ask Jeeves Disallow: /auto-accessories.htm User-agent: fast-WebCrawler Disallow: /auto-accessories.htm # and bad bots are to go away User-agent: ia_archiver Disallow: / User-agent: ia_archiver/1.6 Disallow: / User-agent: Alexibot Disallow: / User-agent: Zeus Disallow: / User-agent: Zeus Link Scout Disallow: / User-agent: Zeus 32297 Webster Pro V2.9 Win32 Disallow: / User-agent: sitecheck.internetseer.com Disallow: /