# The lines starting with '#', specify a comment # User-agent: webcrawler # Disallow: # (Specifies that the robot called 'webcrawler' has nothing disallowed: it may go anywhere) # User-agent: lycra # Disallow: / # (Robot called 'lycra' has all relative URLs starting with '/' disallowed. Because all relative URL's on a server start with '/', this means the entire site is closed off) # User-agent: * # Disallow: /tmp # Disallow: /logs # (All other robots should not visit URLs starting with /tmp or /log. Note the '*' is a special token, meaning "any other User-agent"; you cannot use wildcard patterns or regular expressions in either User-agent or Disallow lines.) # Wildcards are _not_ supported: instead of 'Disallow: /tmp/*' just say 'Disallow: /tmp'. # You shouldn't put more than one path on a Disallow line (this may change in a future version of the spec) # +++ # www.promotebutton.com # Robot Exclusion File -- robots.txt # Author: Gaylord # Last Updated: 30/12/04 19:35:00 User-agent: * Disallow: /favicon.ico Disallow: /tmp Disallow: /cgi Disallow: /logs Disallow: /graphics