User-agent: * # directed to all spiders, not just Scooter
Disallow: /web/whc/archive/out/ # Directory contains early versions of documents already available.
Disallow: /web/whc/conf201/ # Merida meeting materials only (temporary)
Disallow: /web/whc/fr/archive/out # French versions of early materials already available
Disallow: /web/whc/nwhc/ # frames and non-substantive frame content
Disallow: /web/whc/nwhc.fr/ # frames and non-substantive frame content, French pages
Disallow: /*id_search_state=*
Disallow: /*=zip*
Disallow: /*=xls*
Disallow: /en/soc/*id_search_state=*
Disallow: /fr/soc/*id_search_state=*
Disallow: /sites/ # added 2010-09-22frames and non-substantive frame content, French pages
Disallow: /temp/ # temp folders


# robots.txt for http//www.unesco.org/whc/
# User-agent Mozilla/3.0 (compatible; Muscat/1.03)
# Disallow
# 15 June 1997
# updated 3 September 1998, 21 March 1999, 10 November 1999
# (also placed in whc/archive/)
# added 2010-09-22
# User-agent: *
# Request-rate: 1/3
# maximum rate is one page every 5 seconds
# Visit-time: 0600-0845
# only visit between 06:00 and 08:45 UTC (GMT)
# Crawl-delay: 3
User-agent: msnbot
Crawl-delay: 5
# Request-rate: 1/3