# robots.txt for unrequested.info
#
# Search engines: welcome.
# AI *search* bots that fetch a page to cite it live: welcome.
# AI *training* / dataset crawlers: not welcome. See /terms.
#
# The split is deliberate. Being citable in ChatGPT, Claude, and Perplexity is a
# traffic source and does not itself put anything in a training set. Crawling
# this site is still not a license to train on it — /terms reserves that right
# regardless of what is allowed below.
#
# This file is a request, not a wall. Well-behaved crawlers honor it; scrapers
# that ignore it are handled at the network layer (Cloudflare bot rules) and by
# not publishing originals. See docs/ai-scraper-defense.md.
#
# The bot list below goes stale — vendors add and rename agents constantly.
# Re-check every few months against:
#   https://darkvisitors.com/agents
#   https://github.com/ai-robots-txt/ai.robots.txt

# =====================================================================
# ALLOWED — AI search and user-initiated fetches (citation traffic)
# =====================================================================
# These retrieve a page so an assistant can answer with it and link back.
# Vendors document them as separate from their training crawlers, which is
# why they are allowed while the training agents above/below are not.

# OpenAI — ChatGPT Search index, and fetches made when a user pastes a link.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic — Claude's search index and user-initiated fetches.
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity. Note: Perplexity has been credibly reported crawling from
# undeclared user agents and rotating IPs, so treat this entry as goodwill,
# not as the thing keeping them honest. Cloudflare's bot rules are.
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# DuckDuckGo's AI answers.
User-agent: DuckAssistBot
Allow: /

# =====================================================================
# BLOCKED — training and dataset crawlers
# =====================================================================

# --- OpenAI (model training) ---
User-agent: GPTBot
Disallow: /

# --- Anthropic (model training; Claude-Web and anthropic-ai are legacy names) ---
User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

# --- Google (Gemini training; does NOT affect Google Search indexing) ---
User-agent: Google-Extended
Disallow: /

# --- Apple (Apple Intelligence training; does NOT affect Siri/Spotlight indexing) ---
User-agent: Applebot-Extended
Disallow: /

# --- Meta. No clean training/search split is published, so both stay blocked. ---
User-agent: meta-externalagent
Disallow: /

User-agent: meta-externalfetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

# --- ByteDance / TikTok ---
User-agent: Bytespider
Disallow: /

User-agent: TikTokSpider
Disallow: /

# --- Common Crawl (feeds most open training sets) ---
User-agent: CCBot
Disallow: /

# --- Amazon ---
User-agent: Amazonbot
Disallow: /

# --- Image-specific harvesters ---
User-agent: ImagesiftBot
Disallow: /

User-agent: img2dataset
Disallow: /

# --- Other AI crawlers, scrapers, and dataset builders ---
User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FirecrawlAgent
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /

User-agent: Sidetrade indexer bot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: iaskspider/2.0
Disallow: /

# --- Everyone else (Googlebot, Bingbot, feed readers, browsers) ---
User-agent: *
Allow: /
# The form endpoint answers POST only, /contact/thanks/ is a dead end you reach
# after sending, and /og-card/ is only there to be screenshotted into the social
# card at /og.png. None of the three belongs in an index. /og.png itself stays
# crawlable — a card nobody may fetch is not a card.
Disallow: /api/
Disallow: /contact/thanks/
Disallow: /og-card/

Sitemap: https://unrequested.info/sitemap-index.xml