# /robots.txt file to exclude directories not to be crawled #disallow a specific crawler #User-agent: webcrawler #Disallow: #disallow all engines from following directories User-agent: * Disallow: /tmp Disallow: /logs Disallow: /progs Disallow: /gallery2 Disallow: /awstats Disallow: /WCalendar Disallow: /images Disallow: /quizv1 Disallow: /wip Disallow: /quizzard