# Note: this files uses parameters specific to Google, parameters that are not
# in the robots.txt standard
# http://www.google.com/support/webmasters/,
# http://www.robotstxt.org/wc/faq.html and
# http://en.wikipedia.org/wiki/Robots_Exclusion_Standard were used to research
# said parameters
# some links shouldn't show to an anonymous browser such as GAS but are
# included for completeness

# Updated 2007.06.30.09.44
# cf http://confluence.atlassian.com/display/DISC/Prevent+Search+Engine+Indexing+Using+Robots.txt

User-agent: * # match all bots.
Crawl-delay: 5 # per http://en.wikipedia.org/wiki/Robots.txt#Nonstandard_extensions, sets number of seconds to wait between requests to 5 seconds. may not work
Request-rate: 1/5 # per http://en.wikipedia.org/wiki/Robots.txt#Extended_Standard, maximum rate is one page every 5 seconds. may not work

# sven 890623
Disallow: /~sven/jobs
Disallow: /~sven/tables
Disallow: /~sven/TickerTape
Disallow: /~sven/monastic
Disallow: /~sven/writings
Disallow: /~sven/biography
Disallow: /~sven/comics
Disallow: /~sven/www_list
Disallow: /~sven/books
Disallow: /~sven/friends
Disallow: /~sven/hats/
Dissllow: /~sven/hats/index

# Try to get Jeff's stuff out of the various search engines... -- B 970509
Disallow: /~jeffrey/

# Gah.  Infoseek.  15-8-1999
Disallow: /trip/
Disallow: /schmoo/