# robots.txt for https://lite.marketgrep.com/
#
# Crawler policy is split into three lanes:
#   1) AI search / grounding crawlers — actively allow. These decide
#      whether our content can appear in AI search results (ChatGPT
#      search, Claude citations, Perplexity, Google AI Overviews,
#      Apple Intelligence). Blocking these means vanishing from AI
#      search, not just AI training.
#   2) AI training crawlers — allow but kept on their own User-agent
#      blocks so the policy is one edit away from being flipped.
#   3) Default — search engines and everyone else. Public data
#      endpoints are allowed; internal POST refresh endpoints are
#      not.
#
# NOTE: Cloudflare's "AI Audit / Content Signals" feature can override
# this file at the edge with a managed robots.txt. If the live response
# at /robots.txt begins with `# BEGIN Cloudflare Managed content`, the
# Cloudflare override is in effect and this file is ignored — disable
# AI Audit in the Cloudflare dashboard or this file will not take
# effect.

# --- AI search / grounding crawlers — actively allow ---

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

# --- AI training crawlers — allow, kept separate so they're easy to flip ---

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: omgilibot
Allow: /

User-agent: ImagesiftBot
Allow: /

# --- Default for everyone else (Googlebot, Bingbot, etc.) ---
#
# Public data endpoints are allowed so that Dataset.distribution
# contentUrl entries in our JSON-LD validate. POST refresh endpoints
# are disallowed because they trigger expensive recomputation and
# should never be invoked by a crawler.

User-agent: *
Allow: /
Allow: /api/summary
Allow: /api/agent-context
Allow: /api/market-overview
Allow: /api/global-overview
Allow: /api/most-active
Allow: /api/sentiment-report
Allow: /api/turbulence
Allow: /api/ticker-detail/
Allow: /api/version
Allow: /api/health
Allow: /openapi.json
Disallow: /api/refresh
Disallow: /api/refresh-sentiment
Disallow: /api/refresh-turbulence
Disallow: /v1/api/refresh
Disallow: /v1/api/refresh-sentiment
Disallow: /v1/api/refresh-turbulence
Disallow: /healthz

Sitemap: https://lite.marketgrep.com/sitemap.xml