# InboxGuard robots.txt
#
# Purpose-based crawl signals (Cloudflare Content Signals proposal):
#   search   = indexing for search results            -> yes
#   ai-input = retrieval-augmented / on-demand answers -> yes
#   ai-train = training foundation models on our text  -> no
# Applies site-wide unless a specific user-agent block overrides it.
Content-Signal: search=yes, ai-input=yes, ai-train=no

# Default crawler rules. Block authenticated app surfaces; allow everything
# else. Sitemap + schemamap pointers at the bottom are the canonical inventories.
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

# ------------------------------------------------------------------
# AI / LLM crawlers.
#
# We ALLOW search + on-demand-answer crawlers (they cite us to users) on the
# same public surface as humans, and BLOCK training-only crawlers (they take
# our text into a model with no attribution or referral). The Content-Signal
# above states the same intent for crawlers that honor it.
# ------------------------------------------------------------------

# OpenAI — ChatGPT search + on-demand browsing (allowed).
User-agent: GPTBot
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

# Anthropic — Claude search + on-demand browsing (allowed).
User-agent: ClaudeBot
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

User-agent: Claude-User
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

# Perplexity (allowed).
User-agent: PerplexityBot
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

User-agent: Perplexity-User
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

# Google AI / Gemini grounding (allowed — search + grounded answers).
User-agent: Google-Extended
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

# Apple Intelligence (allowed).
User-agent: Applebot-Extended
Allow: /
Disallow: /dashboard
Disallow: /domains
Disallow: /alerts
Disallow: /settings
Disallow: /admin
Disallow: /onboarding
Disallow: /billing
Disallow: /postmaster
Disallow: /invite
Disallow: /auth
Disallow: /r/

# ------------------------------------------------------------------
# Training-only crawlers — DISALLOWED. These ingest text into model
# training corpora with no search referral or citation back to us.
# ------------------------------------------------------------------

# Common Crawl — bootstraps many open-source training sets.
User-agent: CCBot
Disallow: /

# ByteDance / TikTok training crawler.
User-agent: Bytespider
Disallow: /

# Meta training crawler.
User-agent: meta-externalagent
Disallow: /

# Sitemap (regenerated on every guide publish) and NLWeb Schema Map.
Sitemap: https://inboxguard.io/sitemap.xml
Schemamap: https://inboxguard.io/schema-map.xml