// Generates a robots.txt. Two modes: // 'new' — full file (user has no robots.txt at all) // 'diff' — only the bot blocks the user is missing import { AI_BOTS } from './extract.js' function botBlock(bot) { return `User-agent: ${bot}\nAllow: /\nAllow: /llms.txt\n` } export function generateRobotsTxt(siteData) { const { url = '', existingRobots = '', existingAiBots = [] } = siteData const hasRobots = existingRobots.trim().length > 0 const sitemap = url ? `${url.replace(/\/$/, '')}/sitemap.xml` : '[Bitte ergänzen: https://deine-domain.de/sitemap.xml]' if (!hasRobots) { const header = `# robots.txt — generated by VISIGINE # ${url || 'https://deine-domain.de'} User-agent: * Allow: / # AI search engines and language model crawlers ` const blocks = AI_BOTS.map(botBlock).join('\n') return { content: `${header}${blocks}\nSitemap: ${sitemap}\n`, mode: 'new', } } const missing = AI_BOTS.filter((b) => !existingAiBots.includes(b)) if (missing.length === 0) { return { content: '# Deine robots.txt deckt bereits alle relevanten KI-Bots ab. Keine Änderungen nötig.\n', mode: 'diff', } } const header = `# Folgende Blöcke zu deiner bestehenden robots.txt hinzufügen # (am Ende der Datei, vor der Sitemap-Zeile falls vorhanden) ` return { content: header + missing.map(botBlock).join('\n'), mode: 'diff', } }