# robots.txt - Bot-Zugriff auf ~100 Seiten/Tag begrenzt
# Platzieren Sie diese Datei im Root-Verzeichnis Ihrer Website

# AI Training Bots
User-agent: ClaudeBot
Crawl-delay: 864
Disallow: /admin/
Disallow: /private/

User-agent: ChatGPT-User
Crawl-delay: 864
Disallow: /admin/
Disallow: /private/

User-agent: GPTBot
Crawl-delay: 864
Disallow: /admin/
Disallow: /private/

User-agent: Google-Extended
Crawl-delay: 864
Disallow: /admin/
Disallow: /private/

User-agent: PerplexityBot
Crawl-delay: 864
Disallow: /admin/
Disallow: /private/

User-agent: YouBot
Crawl-delay: 864
Disallow: /admin/
Disallow: /private/

User-agent: Meta-ExternalAgent
Crawl-delay: 864
Disallow: /admin/
Disallow: /private/

User-agent: FacebookBot
Crawl-delay: 864
Disallow: /admin/
Disallow: /private/

# Search Engine Bots (weniger restriktiv)
User-agent: Googlebot
Crawl-delay: 10
Disallow: /admin/
Disallow: /private/

User-agent: Bingbot
Crawl-delay: 20
Disallow: /admin/
Disallow: /private/

User-agent: YandexBot
Crawl-delay: 30
Disallow: /admin/
Disallow: /private/

User-agent: DuckDuckBot
Crawl-delay: 30
Disallow: /admin/
Disallow: /private/

# Social Media Bots
User-agent: Twitterbot
Crawl-delay: 60
Disallow: /admin/
Disallow: /private/

User-agent: LinkedInBot
Crawl-delay: 60
Disallow: /admin/
Disallow: /private/

User-agent: facebookexternalhit
Crawl-delay: 60
Disallow: /admin/
Disallow: /private/

# SEO/Analytics Bots
User-agent: AhrefsBot
Crawl-delay: 300
Disallow: /admin/
Disallow: /private/

User-agent: SemrushBot
Crawl-delay: 300
Disallow: /admin/
Disallow: /private/

User-agent: MozBot
Crawl-delay: 300
Disallow: /admin/
Disallow: /private/

User-agent: MJ12bot
Crawl-delay: 600
Disallow: /admin/
Disallow: /private/

User-agent: DotBot
Crawl-delay: 600
Disallow: /admin/
Disallow: /private/

# Archiving Bots
User-agent: ia_archiver
Crawl-delay: 1200
Disallow: /admin/
Disallow: /private/

User-agent: archive.org_bot
Crawl-delay: 1200
Disallow: /admin/
Disallow: /private/

# Aggressive/Problematic Bots (stark eingeschränkt)
User-agent: MegaIndex
Crawl-delay: 3600
Disallow: /

User-agent: SeznamBot
Crawl-delay: 1800
Disallow: /admin/
Disallow: /private/

User-agent: BLEXBot
Crawl-delay: 1800
Disallow: /admin/
Disallow: /private/

# Catch-all für unbekannte Bots
User-agent: *
Crawl-delay: 60
Disallow: /admin/
Disallow: /private/
Disallow: /*.php$
Disallow: /*?*
Disallow: /search/
Disallow: /tmp/

# Erlaubte Bereiche für alle
Allow: /robots.txt
Allow: /favicon.ico
Allow: /.well-known/