# robots.txt for racer.wiki
# Last updated: 2026-04-25
# AI training + AI search welcome on public marketing/info surface.
# Auth, API, admin, and user-data routes are off-limits to all crawlers.

# ============================================================
# AI training + search bots — explicit welcome
# ============================================================
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Bingbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: cohere-ai
User-agent: DuckAssistBot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: YouBot
User-agent: Googlebot
Allow: /
Allow: /garage$
Allow: /sponsorship-guide
Allow: /about
Allow: /pricing
Allow: /press
Allow: /blog
Allow: /tracks
Allow: /classes
Allow: /column
Allow: /llms.txt
Allow: /robots.txt
Allow: /sitemap.xml

# Auth + API + admin off-limits
Disallow: /bb-
Disallow: /api/
Disallow: /admin
Disallow: /sponsorship
Disallow: /sponsorship-scraper
Disallow: /sponsorship-outreach
Disallow: /set-page
Disallow: /set-key
Disallow: /deploy-worker
Disallow: /alert-admin
Disallow: /hunter-api
Disallow: /blackbook/api
Disallow: /bb-auth
Disallow: /bb-setup-engine
Disallow: /bb-magic
Disallow: /bb-google-auth
Disallow: /bb-tonight
Disallow: /bb-classes
Disallow: /bb-engines
Disallow: /bb-garage-api
Disallow: /bb-fan-data
Disallow: /bb-analytics
Disallow: /bb-outreach
Disallow: /bb-ambassador
Disallow: /bb-founders
Disallow: /bb-warroom
Disallow: /bb-admin
Disallow: /g10be-admin
Disallow: /g10be-auth
Disallow: /intel
Disallow: /gonzoguerillamarketer
Disallow: /logout
Disallow: /tonight
Disallow: /live
Disallow: /fan
Disallow: /diagnostic
Disallow: /pit-reporter
Disallow: /batch-poll
Disallow: /tool-review
Disallow: /tool/review
Disallow: /thetool-api
Disallow: /thetool-delete-runs
Disallow: /stripe-webhook
Disallow: /push-subscribe
Disallow: /mrp-proxy
Disallow: /audio-proxy
Disallow: /timegram-template
Disallow: /*-backup-*
Disallow: /*-bk-*
Disallow: /*-patch-*
Disallow: /_deploy_chunks

# ============================================================
# DEFAULT — everyone else, same rules
# ============================================================
User-agent: *
Allow: /
Allow: /garage$
Allow: /sponsorship-guide
Allow: /about
Allow: /pricing
Allow: /press
Allow: /blog
Allow: /tracks
Allow: /classes
Allow: /column
Disallow: /bb-
Disallow: /api/
Disallow: /admin
Disallow: /sponsorship
Disallow: /sponsorship-scraper
Disallow: /sponsorship-outreach
Disallow: /set-page
Disallow: /set-key
Disallow: /deploy-worker
Disallow: /alert-admin
Disallow: /hunter-api
Disallow: /blackbook/api
Disallow: /bb-auth
Disallow: /bb-setup-engine
Disallow: /bb-magic
Disallow: /bb-google-auth
Disallow: /bb-analytics
Disallow: /bb-outreach
Disallow: /bb-ambassador
Disallow: /bb-warroom
Disallow: /bb-admin
Disallow: /g10be-admin
Disallow: /intel
Disallow: /gonzoguerillamarketer
Disallow: /logout
Disallow: /diagnostic
Disallow: /pit-reporter
Disallow: /batch-poll
Disallow: /tool-review
Disallow: /tool/review
Disallow: /thetool-api
Disallow: /stripe-webhook
Disallow: /push-subscribe
Disallow: /mrp-proxy
Disallow: /timegram-template
Disallow: /*-backup-*
Disallow: /*-bk-*
Disallow: /*-patch-*
Disallow: /_deploy_chunks

# Throttle aggressive SEO crawlers (not AI)
User-agent: AhrefsBot
Crawl-delay: 5
User-agent: SemrushBot
Crawl-delay: 5

Sitemap: https://racer.wiki/sitemap.xml
Sitemap: https://racer.wiki/racer-sitemap.xml