# thatsnotfree.com crawler policy
#
# Search engines and AI answer engines: welcome. Read it, quote it, link it.
# AI model training: not free for the taking.
#
# Cloudflare's "managed robots.txt" must stay OFF for this file to be what is
# served — when it is on, Cloudflare prepends its own block-everything-AI rules
# (see LAUNCH.md).

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Training-only crawlers. Their companies' search/answer bots (OAI-SearchBot,
# ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot, Applebot,
# Googlebot, Bingbot) fall under the group above and are allowed.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Bytespider
Disallow: /

# Google-Extended is deliberately NOT blocked: Google bundles Gemini training
# with Gemini grounding/citations under this one token, and being cited in
# Gemini was judged worth more than the refusal. The Content-Signal above
# still states ai-train=no.

Sitemap: https://thatsnotfree.com/sitemap.xml