# Robots.txt for erst-recht.com
# Updated: 2026-04-03

User-agent: *
Allow: /

# Block private/authenticated routes
Disallow: /dashboard
Disallow: /profile
Disallow: /new-consultation
Disallow: /consultation/
Disallow: /payment-success
Disallow: /payment-cancelled
Disallow: /auth/
Disallow: /lawyer/
Disallow: /admin/
Disallow: /test-components
Disallow: /mietvertrag-check/upload
Disallow: /mietvertrag-check/*/analyzing
Disallow: /mietvertrag-check/*/results

# Block suspicious paths (spam/hack attempts)
Disallow: /search/
Disallow: /*.php
Disallow: /reset-password

# AI Search / retrieval crawlers - ALLOWED (these fetch pages to CITE us in
# live AI answers). Blocking these is the single most expensive GEO mistake:
# a Rutgers/Wharton study (Dec 2025) measured -23% traffic for sites that
# blocked AI bots vs peers that let them crawl. Named per waiser.dev GEO
# evidence review (2026-07): OAI-SearchBot, Claude-SearchBot, Claude-User,
# PerplexityBot are the *retrieval* agents (distinct from training crawlers).
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google-Extended controls inclusion in Gemini-app grounding (a growing AI
# answer surface). It does NOT affect classic Google Search ranking, so
# allowing it only opens an additional AI-citation channel. Flipped to Allow
# 2026-07-08 for AI visibility.
User-agent: Google-Extended
Allow: /

# Applebot-Extended: gates Apple Intelligence / Siri answer grounding.
User-agent: Applebot-Extended
Allow: /

# Microsoft Copilot / Bing AI. ChatGPT live search runs mostly on the Bing
# index, so keeping Bing's crawlers happy is high-leverage for GEO.
User-agent: bingbot
Allow: /

# AI *training-only* crawlers - BLOCKED (no bulk model-training scraping;
# these do not drive live citations, so blocking them costs no AI visibility)
User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# Note: FacebookBot is NOT blocked — needed for social link previews

# Sitemap location (canonical domain)
Sitemap: https://erst-recht.com/sitemap.xml

# LLM-specific content
# See https://erst-recht.com/llms.txt for AI-friendly content