# /robots.txt 

User-agent: *
Disallow: /etc
Disallow: /pub
Disallow: /icons
Disallow: /htdig
Disallow: /usage
Disallow: /cgi-bin