# Default: allow all crawlers, block private/auth surfaces
User-agent: *
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /history
Disallow: /tokens
Disallow: /billing
Disallow: /auth

# Explicit allows for major AI crawlers (search + training)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# SEO-tool crawlers, and PetalBot (Huawei's Petal Search, blocked by choice
# for its crawl volume), kept out (2026-10-14). They read the whole catalogue
# after every deploy, and each page they open there is a fresh Vercel ISR
# write. Every AI and LLM crawler above stays allowed. These obey robots.txt;
# ones that do not are the Vercel Firewall's job.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: PetalBot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: Barkrowler
User-agent: serpstatbot
Disallow: /

Sitemap: https://setlist.id/sitemap.xml
# The blog's posts. While the magazine is off on production
# (ENABLE_MAGAZINE, lib/feature-flags.ts) they live under /blog; when it goes
# on, this line becomes https://setlist.id/magazine/sitemap.xml.
Sitemap: https://setlist.id/blog/sitemap.xml
# The public set pages went live on production 2026-09-10
# (ENABLE_PUBLIC_TRACKLISTS, lib/feature-flags.ts). This file is static and
# cannot follow the flag: if those pages are ever taken back off, comment the
# line below out again or it advertises a sitemap that 404s.
# The /artist and /label pages are behind a SECOND flag
# (ENABLE_PUBLIC_RECORD_PAGES, on production since 2026-09-18) and have no
# sitemap of their own -- their entries live inside /tracklist/sitemap.xml,
# which drops them while that flag is off, so nothing here changes with it.
Sitemap: https://setlist.id/tracklist/sitemap.xml