102 lines
2.0 KiB
Plaintext
102 lines
2.0 KiB
Plaintext
# ── Default: all bots ────────────────────────────────────────────
|
|
User-agent: *
|
|
Allow: /
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# ── AI crawlers: same restrictions ───────────────────────────────
|
|
# Personal data (Impressum, Datenschutz) must not be indexed — DSGVO.
|
|
# Business content and llms.txt are explicitly available.
|
|
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: GoogleOther
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: Applebot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: YouBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# OpenAI search bot (separate from GPTBot)
|
|
User-agent: OAI-SearchBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# Meta AI
|
|
User-agent: Meta-ExternalAgent
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# ByteDance / TikTok AI
|
|
User-agent: Bytespider
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# DuckDuckGo AI
|
|
User-agent: DuckAssistBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# Bing (explicit — used by Copilot / ChatGPT via Bing index)
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
Sitemap: https://www.superfice.de/sitemap.xml
|