# robots.txt for HELPWITHBOOKS.ORG
# 11 aug 2023 - Made it for helpwithbooks.org, copy of lw file.
                Disallow for ChatGPT added.
# 15 nov 2021 - edit and add to exiletibetans.com
# 19 may 2021 - add bots found in awstats.
#  1 apr 2021 - the bots.
# Info at http://www.robotstxt.org
#
crawl-delay: 20
#
User-agent: *
Disallow: /cgi-bin
#
#-- ChatGPT
User-agent ChatGPTUser
Disallow: /
User-agent: chatgpt 
Disallow: /
User-agent: GPTBot
Disallow: /
#
#-- Common Crawl
User-agent: CCBot
Disallow: /
#
#  Bad bots:
User-agent: AhrefsBot
Disallow: /
#
User-agent: AspiegelBot
Disallow: /
#
User-agent: DotBot
Disallow: /
#
User-agent: MauiBot
Disallow: /
#
User-agent: MJ12Bot
Disallow: /
#
User-agent: oBot
Disallow: /
#
User-agent: PetalBot
Disallow: /
#
User-agent: SEMrushBot
Disallow: /
#
#-- Facebook:
# you may want to allow the facebook crawler if your site is active on fb.
#%% facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)
#%% Disallow: /
#
#%% facebookexternalhit/1.1
#%% Disallow: /
#
#%% Facebook
#%% Disallow: /
#
#-- Show location of site map:
#%% Sitemap: https://www.DOMAIN.TLD/my-sitemap.xml
#
#-- E O F