# Default — open to all crawlers; styleguide is internal-only, and /api/* are
# POST-only Worker endpoints (a crawl gets 405, which Search Console filed as
# "blocked due to other 4xx" in 2026-09 — nothing to index there).
#
# Content Signals (contentsignals.org, CC0; Cloudflare Content Signals Policy,
# 2025-09) live in this same group: Entogo's stance is open to indexing,
# training and AI-answer input — the site exists to be cited. Tighten by
# switching the relevant token to `=no` if policy changes, and mirror the
# change in worker/index.ts (the `Content-Signal` header on Markdown
# responses MUST say the same thing). Kept in ONE `User-agent: *` record (not
# a separate block) so strict first-match parsers apply both the signal and
# the Disallow. Advisory only: Google honours just user-agent / allow /
# disallow / sitemap (RFC 9309) and ignores this line — harmless. The IETF
# AIPREF `Content-Usage:` line is deliberately NOT added until the draft
# (draft-ietf-aipref-attach) is an RFC with a settled vocabulary.
User-agent: *
Content-Signal: search=yes, ai-train=yes, ai-input=yes
Allow: /
Disallow: /styleguide/
Disallow: /api/

# ---------------------------------------------------------------------------
# Explicit allowlist for AI / LLM crawlers (GEO). Listing them by name is a
# stronger, auditable signal than relying on the wildcard above. Tokens are
# the ones each vendor documents (reviewed 2026-09-05); user-triggered
# fetchers (ChatGPT-User, Claude-User, Perplexity-User, Google-Agent, …)
# generally ignore robots.txt anyway — listed for intent, not control.
# Several user-agents per record is valid per RFC 9309 §2.2.1.
# ---------------------------------------------------------------------------

# Google — AI grounding / training token, Vertex AI agent crawls, R&D crawler.
# (Googlebot itself is covered by `*`; Google-Extended only affects Gemini
# training + grounding, never Search / AI Overviews / AI Mode.)
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
Allow: /
Disallow: /styleguide/
Disallow: /api/

# OpenAI — training (GPTBot), ChatGPT search index (OAI-SearchBot), user
# fetches (ChatGPT-User), ad landing-page validation (OAI-AdsBot).
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: OAI-AdsBot
Allow: /
Disallow: /styleguide/
Disallow: /api/

# Anthropic — training (ClaudeBot), Claude search index (Claude-SearchBot),
# user fetches (Claude-User). `Claude-Web` / `anthropic-ai` are legacy tokens
# no longer in Anthropic's documentation; kept because they are harmless.
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /
Disallow: /styleguide/
Disallow: /api/

# Perplexity — answer-engine index + user fetches.
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /styleguide/
Disallow: /api/

# Microsoft — Bing index feeds Copilot, Bing AI, DuckDuckGo and Yahoo
# (bingbot is covered by `*`; no separate Copilot crawler is documented).

# Apple — Siri / Spotlight / Safari search (Applebot) and the training
# opt-out token (Applebot-Extended); allowing both keeps Apple Intelligence
# answers open.
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /
Disallow: /styleguide/
Disallow: /api/

# Amazon — Alexa+ / Rufus search index (Amzn-SearchBot), user fetches
# (Amzn-User) and the general crawler (Amazonbot, may train Amazon models).
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
Allow: /
Disallow: /styleguide/
Disallow: /api/

# Meta AI — training + indexing (meta-externalagent), Meta AI search index
# (meta-webindexer), user fetches (meta-externalfetcher), link previews.
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: meta-externalfetcher
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
Allow: /
Disallow: /styleguide/
Disallow: /api/

# Mistral — search index, user fetches (Le Chat / Vibe), training.
User-agent: MistralAI-Index
User-agent: MistralAI-User
User-agent: MistralAI-Training
Allow: /
Disallow: /styleguide/
Disallow: /api/

# DuckDuckGo DuckAssist, You.com, Kagi, Cohere, Diffbot knowledge graph,
# Allen Institute (open LMs) — long tail but cheap to list.
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: Kagibot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: AI2Bot
Allow: /
Disallow: /styleguide/
Disallow: /api/

# Common Crawl — feeds many LLM training corpora.
User-agent: CCBot
Allow: /
Disallow: /styleguide/
Disallow: /api/

# ByteDance / Doubao / Volcengine AI.
User-agent: Bytespider
Allow: /
Disallow: /styleguide/
Disallow: /api/

Sitemap: https://entogo.ca/sitemap-index.xml
# Per-language sitemaps (also listed in the index above): named here too so
# crawlers that read robots.txt before the index (Yandex, Bing) find each one.
Sitemap: https://entogo.ca/sitemap-en-0.xml
Sitemap: https://entogo.ca/sitemap-fr-0.xml
Sitemap: https://entogo.ca/sitemap-es-0.xml
Sitemap: https://entogo.ca/sitemap-ar-0.xml
Sitemap: https://entogo.ca/sitemap-pl-0.xml
Sitemap: https://entogo.ca/sitemap-ro-0.xml
Sitemap: https://entogo.ca/sitemap-ru-0.xml