# Fri, 16 Jan 2009 19:05:56 +0000 # Exclude Files From All Robots: User-agent: * Disallow: /User-agent:almaden Disallow: /User-agent:ASPSeek Disallow: /User-agent:baiduspider Disallow: /User-agent:dumbbot Disallow: /User-agent:Generic Disallow: /User-agent:grub-client Disallow: /User-agent:MSIECrawler Disallow: /User-agent:nexabot Disallow: /User-agent:NPBot Disallow: /User-agent:OWR_Crawler Disallow: /User-agent:psbot Disallow: /User-agent:rabaz Disallow: /User-agent:RPT-HTTPClient Disallow: /User-agent:ScoutAbout Disallow: /User-agent:TurnitinBot Disallow: /User-agent:TurnitinBot/1 Disallow: /User-agent:TurnitinBot/2 Disallow: /User-agent:Wget # End robots.txt file