# https://www.robotstxt.org/robotstxt.html
#
# Note: This robots.txt is for the main domain (bankstatemently.com).
# The clerk subdomain (clerk.bankstatemently.com) is managed by Clerk
# and serves authentication pages only (protected by auth, not publicly crawlable).
#
# Auth-gated and private pages (/sign-in, /sign-up, /billing/success, /results)
# are NOT blocked here for search crawlers. They use <meta name="robots"
# content="noindex, nofollow" /> instead — blocking in robots.txt would prevent
# crawlers from seeing the noindex directive, risking phantom indexing.
# AI crawlers are still blocked since they don't respect meta robots tags.

# AI crawlers — block auth-gated/private pages
User-agent: GPTBot
Disallow: /sign-in
Disallow: /sign-up
Disallow: /billing/success
Disallow: /results
Disallow: /impersonate
Allow: /

User-agent: ClaudeBot
Disallow: /sign-in
Disallow: /sign-up
Disallow: /billing/success
Disallow: /results
Disallow: /impersonate
Allow: /

User-agent: anthropic-ai
Disallow: /sign-in
Disallow: /sign-up
Disallow: /billing/success
Disallow: /results
Disallow: /impersonate
Allow: /

# OpenAI's answer-engine crawler (ChatGPT search citations)
User-agent: OAI-SearchBot
Disallow: /sign-in
Disallow: /sign-up
Disallow: /billing/success
Disallow: /results
Disallow: /impersonate
Allow: /

# Perplexity's answer-engine crawler
User-agent: PerplexityBot
Disallow: /sign-in
Disallow: /sign-up
Disallow: /billing/success
Disallow: /results
Disallow: /impersonate
Allow: /

# Common Crawl (feeds many downstream AI training/answer-engine datasets)
User-agent: CCBot
Disallow: /sign-in
Disallow: /sign-up
Disallow: /billing/success
Disallow: /results
Disallow: /impersonate
Allow: /

# Google AI training crawler — block private pages
User-agent: Google-Extended
Disallow: /sign-in
Disallow: /sign-up
Disallow: /billing/success
Disallow: /results
Disallow: /impersonate
Allow: /

# Search crawlers — allow everything, noindex meta tags handle exclusion
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: *
Allow: /

# Sitemap and AI discoverability
Sitemap: https://bankstatemently.com/sitemap.xml

# Content Signals (https://contentsignals.org/)
# ai-input: agents may send content here for processing (that's the product)
# ai-train: do not use site content to train AI models
# search: allow search indexing
Content-Signal: ai-input=yes, ai-train=no, search=yes