# Disallow most crawlers
User-agent: *
Disallow: /

User-agent: ltx71
Disallow: /
Disallow: /*



# Allow Google, and other key crawlers
User-agent: Googlebot           # All Google crawlers
User-agent: Googlebot-News      # Crawler for Google News
User-agent: Googlebot-Image     # Crawler for Google Images
User-agent: Bingbot             # Bing’s crawler
User-agent: YandexBot           # Yandex’s crawler
User-agent: Baiduspider         # Baidu’s crawler
User-agent: Facebot             # Facebook’s crawler
User-agent: Twitterbot          # Twitter’s crawler
User-agent: Slurp               # Yahoo’s crawler
User-agent: DuckDuckBot         # DuckDuckGo
User-agent: ia_archiver         # Amazon’s Alexa internet rankings

Allow: /accounts/login/$        # Allow login but without any parameters (must end with /)
Allow: /accounts/signup_pay/$   # Allow signup_pay but without any parameters (must end with /)
Disallow: /accounts/            # Otherwise don't allow accounts to be crawled/indexed

# Allow Lobo's Look and Compare Team but without any parameters (must end with /)
Allow: /stats/lobos_look/ncaa/team/$
Disallow: /stats/lobos_look/ncaa/team/
Allow: /stats/lobos_look/wnba/team/$
Disallow: /stats/lobos_look/wnba/team/
Allow: /stats/compare/ncaa/team/$
Disallow: /stats/compare/ncaa/team/

# Allow Research but without any parameters (must end with /)
Allow: /stats/ncaa/research/player_careers/$
Disallow: /stats/ncaa/research/player_careers/
Allow: /stats/ncaa/research/player_single_seasons/$
Disallow: /stats/ncaa/research/player_single_seasons/
Allow: /stats/ncaa/research/player_total_seasons/$
Disallow: /stats/ncaa/research/player_total_seasons/
Allow: /stats/ncaa/research/player_single_games/$
Disallow: /stats/ncaa/research/player_single_games/
Allow: /stats/ncaa/research/player_total_games/$
Disallow: /stats/ncaa/research/player_total_games/
Allow: /stats/ncaa/research/team_single_seasons/$
Disallow: /stats/ncaa/research/team_single_seasons/
Allow: /stats/ncaa/research/team_total_seasons/$
Disallow: /stats/ncaa/research/team_total_seasons/
Allow: /stats/ncaa/research/team_single_games/$
Disallow: /stats/ncaa/research/team_single_games/
Allow: /stats/ncaa/research/team_total_games/$
Disallow: /stats/ncaa/research/team_total_games/

Allow: /stats/wnba/research/player_careers/$
Disallow: /stats/wnba/research/player_careers/
Allow: /stats/wnba/research/player_single_seasons/$
Disallow: /stats/wnba/research/player_single_seasons/
Allow: /stats/wnba/research/player_total_seasons/$
Disallow: /stats/wnba/research/player_total_seasons/
Allow: /stats/wnba/research/player_single_games/$
Disallow: /stats/wnba/research/player_single_games/
Allow: /stats/wnba/research/player_total_games/$
Disallow: /stats/wnba/research/player_total_games/
Allow: /stats/wnba/research/team_single_seasons/$
Disallow: /stats/wnba/research/team_single_seasons/
Allow: /stats/wnba/research/team_total_seasons/$
Disallow: /stats/wnba/research/team_total_seasons/
Allow: /stats/wnba/research/team_single_games/$
Disallow: /stats/wnba/research/team_single_games/
Allow: /stats/wnba/research/team_total_games/$
Disallow: /stats/wnba/research/team_total_games/

# Disallow Leaderboards
Disallow: /stats/leaderboard/

Allow: /                        # Allow everything else
Crawl-delay: 500

Sitemap: https://herhoopstats.com/sitemap.xml/