# # $Id: robots.txt 183 2004-09-25 20:20:44Z bk $ # # See http://info.webcrawler.com/mak/projects/robots/robots.html for # the robot exclusion standard which describes the format of this # document ## 2004-04-29 BKarpovich ## added lines to exclude landing pages User-agent: * Disallow: /cgi-bin/ # cgi-bin Disallow: /land/ # landing pages Disallow: /includes/ # include subdirectory Disallow: /download/ # download subdirectory