# https://www.robotstxt.org/robotstxt.html
# FEDLIN LLC - https://fedlin.com
#
# Generative Engine Optimization (GEO): AI answer, search, and model crawlers
# are explicitly welcome. FEDLIN wants to be discoverable and citable in AI
# search and answer engines (ChatGPT, Claude, Perplexity, Gemini, etc.).
# Bing/Copilot is fed by Bingbot, which is handled by the default wildcard group.
# The named groups below make that intent unambiguous and keep access even if the
# wildcard group is tightened later. To opt out of model *training* while keeping
# answer/search visibility, remove GPTBot, CCBot, Google-Extended, and
# Applebot-Extended from the AI group and leave the rest.

# --- Default: traditional web crawlers (Googlebot, Bingbot, etc.) ---
User-agent: *
Allow: /
Disallow: /api/
Disallow: /_archive/

# --- AI answer / search / model crawlers (GEO) ---
# A structured site summary for LLMs is published at https://fedlin.com/llms.txt
# OpenAI
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
# Anthropic
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Google (Gemini / Vertex grounding + training)
User-agent: Google-Extended
# Apple (Apple Intelligence + Siri/Spotlight)
User-agent: Applebot
User-agent: Applebot-Extended
# Others feeding AI answer engines
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: MistralAI-User
User-agent: CCBot
# ByteDance (Doubao)
User-agent: Bytespider
# Cohere
User-agent: cohere-ai
# Diffbot (feeds several answer products)
User-agent: Diffbot
# Timpi
User-agent: Timpibot
Allow: /
Disallow: /api/
Disallow: /_archive/

# Sitemap
Sitemap: https://fedlin.com/sitemap-index.xml