104 lines
1.8 KiB
Plaintext
104 lines
1.8 KiB
Plaintext
Sitemap: https://www.superfice.de/sitemap.xml
|
|
|
|
# Default: all bots
|
|
User-agent: *
|
|
Allow: /
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# AI crawlers: same restrictions
|
|
# Personal data (Impressum, Datenschutz) must not be indexed - DSGVO.
|
|
# Business content and llms.txt are explicitly available.
|
|
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: GoogleOther
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: Applebot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: YouBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# OpenAI search bot (separate from GPTBot)
|
|
User-agent: OAI-SearchBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# Meta AI
|
|
User-agent: Meta-ExternalAgent
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# ByteDance / TikTok AI
|
|
User-agent: Bytespider
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# DuckDuckGo AI
|
|
User-agent: DuckAssistBot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
# Bing (explicit - used by Copilot / ChatGPT via Bing index)
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /impressum
|
|
Disallow: /datenschutz
|
|
Disallow: /api/
|
|
|
|
Sitemap: https://www.superfice.de/sitemap.xml
|