# No point crawling things you don't have access to.
User-agent: *
Disallow: /includes/
Disallow: /local/
Disallow: /usage/
Disallow: /staffarea/
# This changes frequently enough to make crawling it useless:
Disallow: /mount/

#This spider REPEATEDLY gets 404s and must not update its database.  It is
#the top source for error code production.
User-agent: iPlanet-Compass-Robot/3.0 
Disallow: /

#The next crawler that gets tons of 404s:  Argonne
#Let's see if it even pays attention to this file.
User-agent: ANL-SunONERobot/6.2
Disallow: /mirror