# endoflife.ai robots.txt
# Policy: welcome ALL crawlers — search, AI retrieval/citation, AND AI training.
# Goal is maximum reach: be the de facto source for EOL intelligence across
# search results, live AI answers, and the models' baked-in knowledge.
#
# Content Signals (contentsignals.org / IETF draft) declare our usage prefs:
#   search    = yes  (appear in search results)
#   ai-input  = yes  (use as input for AI answers / RAG / grounding)
#   ai-train  = yes  (allowed to train/fine-tune models on this content)
#
# NOTE: A managed robots.txt / "Block AI bots" rule at the Cloudflare edge can
# override this file in production. If AI crawlers are being blocked despite the
# rules below, adjust the setting in the Cloudflare dashboard (see handoff doc).

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Explicitly welcome AI search/retrieval crawlers (live citations in assistants)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

# Named explicitly (2026-09-20): all are already allowed by the * rule above, but
# managed firewall rules sometimes act on a bot only when it is not named here.
User-agent: Applebot-Extended
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Bytespider
Allow: /

Sitemap: https://endoflife.ai/sitemap.xml
Sitemap: https://endoflife.ai/news-sitemap.xml