# ---------------------------------------------------------------------
# The Fabricant — LLM-Friendly robots.txt
# ---------------------------------------------------------------------
# Purpose: Allow AI crawlers (LLMs) and search engines to index
# all public-facing content, while blocking internal or private paths.
# ---------------------------------------------------------------------

# -------------------------
# MAIN AI CRAWLER ALLOW LIST
# -------------------------

# OpenAI GPTBot
User-agent: GPTBot
Allow: /

# OpenAI ChatGPT-User crawler
User-agent: ChatGPT-User
Allow: /

# Anthropic ClaudeBot
User-agent: ClaudeBot
Allow: /

# Anthropic (legacy)
User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google Extended (AI training)
User-agent: Google-Extended
Allow: /

# Google Generative Crawler
User-agent: GoogleOther
Allow: /

# Meta AI Crawler
User-agent: MetaAI
Allow: /

# AppleBot (Siri + Apple Intelligence)
User-agent: Applebot
Allow: /

# xAI (Grok)
User-agent: Xbot
Allow: /

# -------------------------
# GENERAL SEARCH ENGINES
# -------------------------
User-agent: *
Disallow: /?*
Disallow: /search
Disallow: /admin
Disallow: /404
Disallow: /500
Disallow: /api/
Disallow: /private/
Disallow: /internal/
Disallow: /uploads/private/
Disallow: /cms-preview/
Disallow: /scripts/
Disallow: /ajax/

# Allow everything else
Allow: /

# -------------------------
# llms.txt Access
# -------------------------
Allow: /llms.txt

# -------------------------
# Sitemap
# -------------------------
Sitemap: https://www.thefabricant.com/sitemap.xml


Sitemap: https://www.thefabricant.com/sitemap.xml