User-agent: *
#Disallow select URLs
Disallow: /account
Disallow: /wishlist
Disallow: /cart
Disallow: /c/
Disallow: /cw/
Disallow: /cdn-cgi/
Disallow: /sitemap_*.xml

#Allow these Bots to crawl Sitemap
User-agent: AdsBot
User-agent: Adsbot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google-Mobile-Apps
User-agent: AdIdxBot
User-agent: Applebot
User-agent: APIs-Google
User-agent: baiduspider
User-agent: Bingbot
User-agent: BingPreview
User-agent: Caliperbot
User-agent: CCBot
User-agent: ContentKing
User-agent: Deepcrawl
User-agent: DuckDuckBot
User-agent: facebot
User-agent: facebookexternalhit/1.0
User-agent: facebookexternalhit/1.1
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: LinkedInBot
User-agent: MicrosoftPreview
User-agent: Mediapartners-Google
User-agent: Neevabot
User-agent: PetalBot
User-agent: SemrushBot
User-agent: Slurp
User-agent: Siteimprove
User-agent: SemanticScholarBot
User-agent: Storebot-Google
User-agent: Twitterbot
User-agent: YandexBot
Allow: /sitemap_*.xml
Disallow: /account
Disallow: /wishlist
Disallow: /cart
Disallow: /c/
Disallow: /cw/
Disallow: /cdn-cgi/

#Disallow everything for AhrefsBot
User-agent: AhrefsBot
Disallow: /

#Sitemaps-https
Sitemap: https://www.routledge.com/sitemap_index.xml