Sitemap: https://projects.propublica.org/sitemaps/projects-sitemaps/coronavirus-bailouts/siteindex.xml
Sitemap: https://projects.propublica.org/sitemaps/projects-sitemaps/nypd-ccrb/siteindex.xml
Sitemap: https://projects.propublica.org/sitemaps/projects-sitemaps/chicken/sitemap.xml.gz
Sitemap: https://projects.propublica.org/sitemaps/projects-sitemaps/nonprofits/sitemap.xml.gz
Sitemap: https://projects.propublica.org/sitemaps/projects-sitemaps/supreme-connections/sitemap.xml.gz

User-Agent: *
Crawl-Delay: 1
Disallow: /checkup/providers/search*
Disallow: /nursing-homes/findings/search*
Disallow: /docdollars/search*
Disallow: /docdollars/query*
Disallow: /treatment/doctors/search*
Disallow: /nonprofits/search*
Disallow: /nonprofits/name_search*
Disallow: /nonprofits/full_text_search*
Disallow: /nonprofits/redirect_to_990*
Disallow: /nonprofits/display_990*
Disallow: /nonprofits/download-filing*
Disallow: /itemizer/filing*
Disallow: /electionbot/search*
Disallow: /test/*
Disallow: /stg/*
Disallow: /staging/*

# Sends bad Accept header that blows up rails
# http://www.seokicks.de/robot.html
User-Agent: SEOkicks-Robot
Disallow: /
User-Agent: seokicks
Disallow: /