# robots.txt, query.nytimes.com 6/25/2013

User-agent: Bytespider
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: *
Disallow: /pages/college/
Disallow: /college/
Disallow: /library/
Disallow: /learning/
Disallow: /aponline/
Disallow: /reuters/
Disallow: /cnet/
Disallow: /partners/
Disallow: /archives/
Disallow: /indexes/
Disallow: /adx/bin/
Disallow: /thestreet/
Disallow: /nytimes-partners/
Disallow: /financialtimes/
Disallow: /svc
Disallow: /mem/
Allow: /pages/
Allow: /2003/
Allow: /2004/
Allow: /2005/
Allow: /top/
Allow: /ref/
Allow: /services/xml/

User-agent: Mediapartners-Google*
Disallow:

Sitemap: http://spiderbites.nytimes.com/sitemaps/query.nytimes.com/sitemap.xml.gz