Initial commit: Visigine (Vite client + Express/SQLite backend)
Container-ready via docker/ compose (frontend nginx + backend Node). Compose adjusted for Coolify on the prod server: frontend uses expose:80 (no host binding — host 8080 is taken by the Coolify proxy; Traefik routes visigine.de), backend ALLOWED_ORIGINS=https://visigine.de. Secrets stay in server/.env (git-ignored); see server/.env.example. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
59
server/lib/activity.js
Normal file
59
server/lib/activity.js
Normal file
@@ -0,0 +1,59 @@
|
||||
// Rolling buffer of recent analyses, resets on restart.
|
||||
// Metadata-only by design — no PII, no full response bodies, no extracted siteData.
|
||||
const MAX_ENTRIES = 50
|
||||
const log = []
|
||||
|
||||
export function recordAnalysis(entry) {
|
||||
log.unshift({
|
||||
ts: new Date().toISOString(),
|
||||
requestId: entry.requestId,
|
||||
host: entry.host || null,
|
||||
score: entry.score ?? null,
|
||||
issuesCount: entry.issuesCount ?? null,
|
||||
failedCheckIds: entry.failedCheckIds || [],
|
||||
cacheHit: Boolean(entry.cacheHit),
|
||||
ms: entry.ms ?? 0,
|
||||
status: entry.status === 'ok' ? 'ok' : 'err',
|
||||
code: entry.code || null,
|
||||
admin: Boolean(entry.admin),
|
||||
})
|
||||
if (log.length > MAX_ENTRIES) log.pop()
|
||||
}
|
||||
|
||||
export function recentAnalyses() {
|
||||
return [...log]
|
||||
}
|
||||
|
||||
export function computeStats() {
|
||||
const succeeded = log.filter((e) => e.status === 'ok' && typeof e.score === 'number')
|
||||
if (!succeeded.length) {
|
||||
return { total: log.length, succeeded: 0, avgScore: null, topFails: [], topHosts: [] }
|
||||
}
|
||||
const avgScore = succeeded.reduce((s, e) => s + e.score, 0) / succeeded.length
|
||||
|
||||
const failCounts = {}
|
||||
for (const e of succeeded) {
|
||||
for (const id of e.failedCheckIds) failCounts[id] = (failCounts[id] || 0) + 1
|
||||
}
|
||||
const topFails = Object.entries(failCounts)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 5)
|
||||
.map(([id, count]) => ({ id, count }))
|
||||
|
||||
const hostCounts = {}
|
||||
for (const e of log) {
|
||||
if (e.host) hostCounts[e.host] = (hostCounts[e.host] || 0) + 1
|
||||
}
|
||||
const topHosts = Object.entries(hostCounts)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([host, count]) => ({ host, count }))
|
||||
|
||||
return {
|
||||
total: log.length,
|
||||
succeeded: succeeded.length,
|
||||
avgScore: Number(avgScore.toFixed(1)),
|
||||
topFails,
|
||||
topHosts,
|
||||
}
|
||||
}
|
||||
192
server/lib/autofix/extract.js
Normal file
192
server/lib/autofix/extract.js
Normal file
@@ -0,0 +1,192 @@
|
||||
// Extracts user-facing siteData from the analyze context for the three generators.
|
||||
// Every field is optional; generators fall back to German `[Bitte ergänzen: ...]` placeholders.
|
||||
|
||||
// Kept in sync with checks/ai-bots.js. Order matters — used as canonical
|
||||
// ordering for generated robots.txt.
|
||||
export const AI_BOTS = [
|
||||
'GPTBot', 'ClaudeBot', 'OAI-SearchBot', 'PerplexityBot', 'Bingbot',
|
||||
'Google-Extended', 'GoogleOther', 'Applebot-Extended', 'Meta-ExternalAgent',
|
||||
'CCBot', 'Bytespider', 'DuckAssistBot', 'ChatGPT-User',
|
||||
]
|
||||
|
||||
const SEPARATORS = /\s+[–|—\-·•|]\s+/
|
||||
const PLACEHOLDER_EMAILS = new Set([
|
||||
'name@example.com', 'test@test.de', 'test@example.com',
|
||||
'mail@example.com', 'info@example.com',
|
||||
])
|
||||
const PLACEHOLDER_PHONES = new Set(['+49 0', '+49000', '0000000', '1234567'])
|
||||
|
||||
function cleanTitle(title) {
|
||||
if (!title) return null
|
||||
const parts = title.split(SEPARATORS).map((s) => s.trim()).filter(Boolean)
|
||||
if (!parts.length) return null
|
||||
const longest = parts.reduce((a, b) => (a.length >= b.length ? a : b))
|
||||
return longest.length >= 3 ? longest : null
|
||||
}
|
||||
|
||||
function decodeEntities(s) {
|
||||
if (!s) return s
|
||||
return s
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/ /g, ' ')
|
||||
}
|
||||
|
||||
function metaContent(headHtml, attr, value) {
|
||||
const re = new RegExp(
|
||||
`<meta[^>]*${attr}=["']${value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}["'][^>]*content=["']([^"']*)["']`,
|
||||
'i'
|
||||
)
|
||||
const m = headHtml.match(re)
|
||||
if (m) return decodeEntities(m[1].trim())
|
||||
// Try attribute order swapped.
|
||||
const re2 = new RegExp(
|
||||
`<meta[^>]*content=["']([^"']*)["'][^>]*${attr}=["']${value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}["']`,
|
||||
'i'
|
||||
)
|
||||
const m2 = headHtml.match(re2)
|
||||
return m2 ? decodeEntities(m2[1].trim()) : null
|
||||
}
|
||||
|
||||
function linkHref(headHtml, rel) {
|
||||
const re = new RegExp(`<link[^>]*rel=["']${rel}["'][^>]*href=["']([^"']+)["']`, 'i')
|
||||
const m = headHtml.match(re)
|
||||
if (m) return m[1].trim()
|
||||
const re2 = new RegExp(`<link[^>]*href=["']([^"']+)["'][^>]*rel=["']${rel}["']`, 'i')
|
||||
const m2 = headHtml.match(re2)
|
||||
return m2 ? m2[1].trim() : null
|
||||
}
|
||||
|
||||
function parseJsonLdBlocks(blocks) {
|
||||
const parsed = []
|
||||
for (const block of blocks || []) {
|
||||
try {
|
||||
const v = JSON.parse(block)
|
||||
if (Array.isArray(v)) parsed.push(...v)
|
||||
else parsed.push(v)
|
||||
} catch {
|
||||
// skip malformed
|
||||
}
|
||||
}
|
||||
// Flatten @graph members so consumers can iterate flat list.
|
||||
const flat = []
|
||||
for (const node of parsed) {
|
||||
if (node && typeof node === 'object' && Array.isArray(node['@graph'])) {
|
||||
flat.push(...node['@graph'])
|
||||
} else if (node) {
|
||||
flat.push(node)
|
||||
}
|
||||
}
|
||||
return flat
|
||||
}
|
||||
|
||||
function pickType(node) {
|
||||
const t = node?.['@type']
|
||||
if (Array.isArray(t)) return t[0]
|
||||
return t
|
||||
}
|
||||
|
||||
function findNode(nodes, types) {
|
||||
const set = new Set(types)
|
||||
return nodes.find((n) => set.has(pickType(n))) || null
|
||||
}
|
||||
|
||||
function firstEmail(html) {
|
||||
const m = (html || '').match(/mailto:([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})/i)
|
||||
if (!m) return null
|
||||
const email = m[1].toLowerCase()
|
||||
return PLACEHOLDER_EMAILS.has(email) ? null : email
|
||||
}
|
||||
|
||||
function firstPhone(html) {
|
||||
const m = (html || '').match(/tel:(\+?[0-9 \-()]{6,})/i)
|
||||
if (!m) return null
|
||||
const phone = m[1].trim()
|
||||
return PLACEHOLDER_PHONES.has(phone) ? null : phone
|
||||
}
|
||||
|
||||
function detectExistingAiBots(robotsTxt) {
|
||||
if (!robotsTxt) return []
|
||||
return AI_BOTS.filter((bot) => robotsTxt.includes(bot))
|
||||
}
|
||||
|
||||
export function extractSiteData(context) {
|
||||
const { headHtml = '', html = '', jsonLdBlocks = [], robotsTxt = '', llmsTxt = '', baseUrl = '' } = context
|
||||
const nodes = parseJsonLdBlocks(jsonLdBlocks)
|
||||
const org = findNode(nodes, ['Organization', 'LocalBusiness', 'Corporation', 'NewsMediaOrganization'])
|
||||
const website = findNode(nodes, ['WebSite'])
|
||||
|
||||
const ogSiteName = metaContent(headHtml, 'property', 'og:site_name')
|
||||
const ogTitle = metaContent(headHtml, 'property', 'og:title')
|
||||
const ogDesc = metaContent(headHtml, 'property', 'og:description')
|
||||
const ogLocale = metaContent(headHtml, 'property', 'og:locale')
|
||||
const metaDesc = metaContent(headHtml, 'name', 'description')
|
||||
|
||||
const titleRaw = (headHtml.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '').trim()
|
||||
const titleClean = cleanTitle(decodeEntities(titleRaw))
|
||||
|
||||
const langMatch = html.match(/<html[^>]*\slang=["']([^"']+)["']/i)
|
||||
const language = (langMatch?.[1] || ogLocale || 'de').split(/[-_]/)[0].toLowerCase()
|
||||
|
||||
const canonical = linkHref(headHtml, 'canonical')
|
||||
const url = canonical || baseUrl || ''
|
||||
const hostname = (() => {
|
||||
try { return new URL(url).hostname } catch { return '' }
|
||||
})()
|
||||
|
||||
const name =
|
||||
ogSiteName ||
|
||||
(typeof org?.name === 'string' ? org.name : null) ||
|
||||
(typeof website?.name === 'string' ? website.name : null) ||
|
||||
titleClean ||
|
||||
hostname ||
|
||||
null
|
||||
|
||||
const description =
|
||||
metaDesc ||
|
||||
ogDesc ||
|
||||
(typeof org?.description === 'string' ? org.description : null) ||
|
||||
null
|
||||
|
||||
const email = firstEmail(html) || (typeof org?.email === 'string' ? org.email : null) || null
|
||||
|
||||
const phone =
|
||||
firstPhone(html) ||
|
||||
(typeof org?.telephone === 'string' ? org.telephone : null) ||
|
||||
null
|
||||
|
||||
let address = null
|
||||
const addrRaw = org?.address
|
||||
if (addrRaw && typeof addrRaw === 'object') {
|
||||
address = {
|
||||
streetAddress: addrRaw.streetAddress || null,
|
||||
postalCode: addrRaw.postalCode || null,
|
||||
addressLocality: addrRaw.addressLocality || null,
|
||||
addressCountry: addrRaw.addressCountry || null,
|
||||
}
|
||||
}
|
||||
|
||||
let sameAs = []
|
||||
if (Array.isArray(org?.sameAs)) {
|
||||
sameAs = org.sameAs.filter((s) => typeof s === 'string' && /^https?:\/\//.test(s))
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
description,
|
||||
url,
|
||||
language,
|
||||
hostname,
|
||||
email,
|
||||
phone,
|
||||
address,
|
||||
sameAs,
|
||||
existingRobots: robotsTxt || '',
|
||||
existingAiBots: detectExistingAiBots(robotsTxt),
|
||||
hasLlmsTxt: Boolean(llmsTxt && llmsTxt.length > 0),
|
||||
hasOrgJsonLd: Boolean(org),
|
||||
}
|
||||
}
|
||||
19
server/lib/autofix/index.js
Normal file
19
server/lib/autofix/index.js
Normal file
@@ -0,0 +1,19 @@
|
||||
import { extractSiteData } from './extract.js'
|
||||
import { generateLlmsTxt } from './llms-txt.js'
|
||||
import { generateRobotsTxt } from './robots-txt.js'
|
||||
import { generateJsonLd } from './json-ld.js'
|
||||
import { buildReadme } from './readme.js'
|
||||
|
||||
// Returns an autofix bundle. `_siteData` is included for debug-mode payloads;
|
||||
// analyze.js strips it from the public response.
|
||||
export function generateAutofix(context) {
|
||||
const siteData = extractSiteData(context)
|
||||
return {
|
||||
llmsTxt: generateLlmsTxt(siteData),
|
||||
robotsTxt: generateRobotsTxt(siteData),
|
||||
jsonLd: generateJsonLd(siteData),
|
||||
_siteData: siteData,
|
||||
}
|
||||
}
|
||||
|
||||
export { buildReadme, extractSiteData }
|
||||
79
server/lib/autofix/json-ld.js
Normal file
79
server/lib/autofix/json-ld.js
Normal file
@@ -0,0 +1,79 @@
|
||||
// Generates a JSON-LD skeleton: Organization (or LocalBusiness if address/phone),
|
||||
// WebSite, and FAQPage — three highest-impact AI signals.
|
||||
|
||||
const ph = (s) => `[Bitte ergänzen: ${s}]`
|
||||
|
||||
function buildOrganizationNode(siteData) {
|
||||
const { name, url, description, email, phone, address, sameAs = [] } = siteData
|
||||
const useLocalBusiness = Boolean(address || phone)
|
||||
|
||||
const node = {
|
||||
'@type': useLocalBusiness ? 'LocalBusiness' : 'Organization',
|
||||
'@id': `${(url || ph('https://deine-domain.de')).replace(/\/+$/, '')}/#organization`,
|
||||
name: name || ph('Name deines Unternehmens'),
|
||||
url: url || ph('https://deine-domain.de'),
|
||||
description: description || ph('Ein-Satz-Beschreibung'),
|
||||
}
|
||||
|
||||
if (email) node.email = email
|
||||
if (phone) node.telephone = phone
|
||||
|
||||
node.address = {
|
||||
'@type': 'PostalAddress',
|
||||
addressCountry: address?.addressCountry || ph('DE/AT/CH'),
|
||||
addressLocality: address?.addressLocality || ph('Stadt'),
|
||||
postalCode: address?.postalCode || ph('PLZ'),
|
||||
streetAddress: address?.streetAddress || ph('Straße + Nr.'),
|
||||
}
|
||||
|
||||
node.sameAs = sameAs.length > 0 ? sameAs : [ph('https://www.linkedin.com/company/...')]
|
||||
|
||||
return node
|
||||
}
|
||||
|
||||
function buildWebSiteNode(siteData) {
|
||||
const { name, url, language = 'de' } = siteData
|
||||
return {
|
||||
'@type': 'WebSite',
|
||||
'@id': `${(url || ph('https://deine-domain.de')).replace(/\/+$/, '')}/#website`,
|
||||
url: url || ph('https://deine-domain.de'),
|
||||
name: name || ph('Name deines Unternehmens'),
|
||||
inLanguage: `${language}-DE`,
|
||||
publisher: { '@id': `${(url || ph('https://deine-domain.de')).replace(/\/+$/, '')}/#organization` },
|
||||
}
|
||||
}
|
||||
|
||||
function buildFaqNode() {
|
||||
return {
|
||||
'@type': 'FAQPage',
|
||||
mainEntity: [
|
||||
{
|
||||
'@type': 'Question',
|
||||
name: ph('häufige Frage'),
|
||||
acceptedAnswer: { '@type': 'Answer', text: ph('1-2 Sätze') },
|
||||
},
|
||||
{
|
||||
'@type': 'Question',
|
||||
name: ph('weitere Frage'),
|
||||
acceptedAnswer: { '@type': 'Answer', text: ph('1-2 Sätze') },
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
export function generateJsonLd(siteData) {
|
||||
const payload = {
|
||||
'@context': 'https://schema.org',
|
||||
'@graph': [
|
||||
buildOrganizationNode(siteData),
|
||||
buildWebSiteNode(siteData),
|
||||
buildFaqNode(),
|
||||
],
|
||||
}
|
||||
const pretty = JSON.stringify(payload, null, 2)
|
||||
const content = `<script type="application/ld+json">\n${pretty}\n</script>\n`
|
||||
return {
|
||||
content,
|
||||
mode: siteData.hasOrgJsonLd ? 'enhance' : 'new',
|
||||
}
|
||||
}
|
||||
67
server/lib/autofix/llms-txt.js
Normal file
67
server/lib/autofix/llms-txt.js
Normal file
@@ -0,0 +1,67 @@
|
||||
// Generates a personalized llms.txt template in German.
|
||||
// Placeholders use the `[Bitte ergänzen: ...]` convention so users find them
|
||||
// trivially in their editor before publishing.
|
||||
|
||||
const ph = (s) => `[Bitte ergänzen: ${s}]`
|
||||
|
||||
function emailLine(email) {
|
||||
return email ? email : ph('hello@deine-domain.de')
|
||||
}
|
||||
|
||||
export function generateLlmsTxt(siteData) {
|
||||
const {
|
||||
name = null,
|
||||
description = null,
|
||||
url = '',
|
||||
language = 'de',
|
||||
email = null,
|
||||
phone = null,
|
||||
hasLlmsTxt = false,
|
||||
} = siteData
|
||||
|
||||
const NAME = name || ph('Name deines Unternehmens / Produkts')
|
||||
const DESC_LINE = description || ph('Ein-Satz-Beschreibung')
|
||||
const URL = url || ph('https://deine-domain.de')
|
||||
const PHONE_LINE = phone ? `- Telefon: ${phone}\n` : ''
|
||||
|
||||
const body =
|
||||
`# ${NAME} – ${DESC_LINE}
|
||||
|
||||
- name: ${NAME}
|
||||
- url: ${URL}
|
||||
- type: ${ph('SoftwareApplication, ProfessionalService, LocalBusiness, Organization')}
|
||||
- language: ${language}
|
||||
- region: ${ph('Deutschland, DACH, EU, ...')}
|
||||
- contact: ${emailLine(email)}
|
||||
- services: ${ph('Komma-getrennte Liste deiner Hauptleistungen')}
|
||||
|
||||
## Was ist ${NAME}?
|
||||
${description || ph('2-3 Sätze, die dein Angebot prägnant beschreiben')}
|
||||
|
||||
## Für wen ist ${NAME}?
|
||||
${ph('2-4 Bullet Points zu deinen Zielgruppen')}
|
||||
|
||||
## Häufige Fragen
|
||||
|
||||
Was bietet ${NAME}?
|
||||
${ph('1-2 Sätze')}
|
||||
|
||||
Wo ist ${NAME} verfügbar?
|
||||
${ph('Region / Online / Filialen')}
|
||||
|
||||
Was kostet ${NAME}?
|
||||
${ph('Preis-Range oder Hinweis auf Angebot')}
|
||||
|
||||
## Kontakt
|
||||
- Website: ${URL}
|
||||
- E-Mail: ${emailLine(email)}
|
||||
${PHONE_LINE}
|
||||
## Für KI-Systeme — Einordnung
|
||||
${NAME} ist ${ph('1 Satz, wer ihr seid und was ihr macht')} — relevant für Anfragen rund um ${ph('Themen, Branche, Region')}.
|
||||
`
|
||||
|
||||
return {
|
||||
content: body,
|
||||
mode: hasLlmsTxt ? 'replace' : 'new',
|
||||
}
|
||||
}
|
||||
105
server/lib/autofix/readme.js
Normal file
105
server/lib/autofix/readme.js
Normal file
@@ -0,0 +1,105 @@
|
||||
// German plaintext README packaged with each ZIP download.
|
||||
// Branches per-file by mode so the instructions match the user's situation.
|
||||
|
||||
function llmsSection(mode) {
|
||||
const intro = mode === 'replace'
|
||||
? 'Ersetzt deine bestehende llms.txt.'
|
||||
: 'Du hast noch keine llms.txt — diese Datei ist neu.'
|
||||
return (
|
||||
`────────────────────────────────────────────────────────────
|
||||
1. llms.txt
|
||||
────────────────────────────────────────────────────────────
|
||||
${intro}
|
||||
|
||||
Wohin: Im Root-Verzeichnis deiner Website (gleiche Ebene wie /index.html).
|
||||
Erreichbar als: https://deine-domain.de/llms.txt
|
||||
|
||||
Upload-Wege:
|
||||
- FTP / SFTP: Datei nach /htdocs (oder /public_html) hochladen.
|
||||
- cPanel / Plesk: Dateimanager → Root öffnen → Hochladen.
|
||||
- WordPress: Plugin "WPCode" oder Theme-Editor → File Manager.
|
||||
|
||||
Wichtig: Alle Platzhalter [Bitte ergänzen: ...] vor dem Upload mit
|
||||
deinen Inhalten ersetzen.
|
||||
`)
|
||||
}
|
||||
|
||||
function robotsSection(mode) {
|
||||
if (mode === 'new') {
|
||||
return (
|
||||
`────────────────────────────────────────────────────────────
|
||||
2. robots.txt
|
||||
────────────────────────────────────────────────────────────
|
||||
Du hast noch keine robots.txt — diese Datei ist komplett.
|
||||
|
||||
Wohin: Im Root-Verzeichnis deiner Website. Ersetzt eine eventuell
|
||||
bestehende robots.txt komplett. Erreichbar als
|
||||
https://deine-domain.de/robots.txt
|
||||
`)
|
||||
}
|
||||
return (
|
||||
`────────────────────────────────────────────────────────────
|
||||
2. robots.txt
|
||||
────────────────────────────────────────────────────────────
|
||||
Deine bestehende robots.txt deckt nicht alle KI-Bots ab.
|
||||
|
||||
Wohin: Den Inhalt dieser Datei am Ende deiner bestehenden
|
||||
robots.txt einfügen — vor der Sitemap-Zeile, falls vorhanden.
|
||||
Bestehende Regeln NICHT überschreiben.
|
||||
`)
|
||||
}
|
||||
|
||||
function jsonLdSection(mode) {
|
||||
const intro = mode === 'enhance'
|
||||
? 'Deine Seite hat bereits JSON-LD — diese Version erweitert die Coverage (FAQPage, WebSite, vollständige Organization).'
|
||||
: 'Du hast noch kein JSON-LD — dieser Block ist neu.'
|
||||
return (
|
||||
`────────────────────────────────────────────────────────────
|
||||
3. jsonld.html
|
||||
────────────────────────────────────────────────────────────
|
||||
${intro}
|
||||
|
||||
Wohin: Den gesamten <script>-Block in das <head> deiner Startseite
|
||||
einfügen (idealerweise direkt nach den Meta-Tags).
|
||||
|
||||
WordPress: Theme-Datei header.php oder via SEO-Plugin
|
||||
(Yoast / RankMath → Schema-Editor).
|
||||
|
||||
Hinweis: Validieren mit https://validator.schema.org/ vor dem
|
||||
Live-Schalten.
|
||||
`)
|
||||
}
|
||||
|
||||
export function buildReadme(autofix) {
|
||||
const llmsMode = autofix?.llmsTxt?.mode || 'new'
|
||||
const robotsMode = autofix?.robotsTxt?.mode || 'new'
|
||||
const jsonLdMode = autofix?.jsonLd?.mode || 'new'
|
||||
|
||||
return (
|
||||
`VISIGINE Auto-Fix Paket
|
||||
========================
|
||||
|
||||
Dieses Archiv enthält drei Dateien, die deine Website für KI-Suchsysteme
|
||||
sichtbar machen. Bitte alle Platzhalter [Bitte ergänzen: ...] vor dem
|
||||
Hochladen mit deinen Inhalten ersetzen.
|
||||
|
||||
${llmsSection(llmsMode)}
|
||||
${robotsSection(robotsMode)}
|
||||
${jsonLdSection(jsonLdMode)}
|
||||
────────────────────────────────────────────────────────────
|
||||
Validierung
|
||||
────────────────────────────────────────────────────────────
|
||||
Nach dem Hochladen erneut analysieren:
|
||||
https://www.visigine.de#analyzer
|
||||
|
||||
────────────────────────────────────────────────────────────
|
||||
Support
|
||||
────────────────────────────────────────────────────────────
|
||||
Fragen oder Hilfe bei der Umsetzung:
|
||||
- E-Mail: hello@profice.ai
|
||||
- Termin: https://termin.profice.de
|
||||
|
||||
Vollständige Umsetzung gewünscht?
|
||||
→ https://www.visigine.de#pricing
|
||||
`)
|
||||
}
|
||||
49
server/lib/autofix/robots-txt.js
Normal file
49
server/lib/autofix/robots-txt.js
Normal file
@@ -0,0 +1,49 @@
|
||||
// Generates a robots.txt. Two modes:
|
||||
// 'new' — full file (user has no robots.txt at all)
|
||||
// 'diff' — only the bot blocks the user is missing
|
||||
import { AI_BOTS } from './extract.js'
|
||||
|
||||
function botBlock(bot) {
|
||||
return `User-agent: ${bot}\nAllow: /\nAllow: /llms.txt\n`
|
||||
}
|
||||
|
||||
export function generateRobotsTxt(siteData) {
|
||||
const { url = '', existingRobots = '', existingAiBots = [] } = siteData
|
||||
const hasRobots = existingRobots.trim().length > 0
|
||||
const sitemap = url ? `${url.replace(/\/$/, '')}/sitemap.xml` : '[Bitte ergänzen: https://deine-domain.de/sitemap.xml]'
|
||||
|
||||
if (!hasRobots) {
|
||||
const header =
|
||||
`# robots.txt — generated by VISIGINE
|
||||
# ${url || 'https://deine-domain.de'}
|
||||
|
||||
User-agent: *
|
||||
Allow: /
|
||||
|
||||
# AI search engines and language model crawlers
|
||||
`
|
||||
const blocks = AI_BOTS.map(botBlock).join('\n')
|
||||
return {
|
||||
content: `${header}${blocks}\nSitemap: ${sitemap}\n`,
|
||||
mode: 'new',
|
||||
}
|
||||
}
|
||||
|
||||
const missing = AI_BOTS.filter((b) => !existingAiBots.includes(b))
|
||||
if (missing.length === 0) {
|
||||
return {
|
||||
content: '# Deine robots.txt deckt bereits alle relevanten KI-Bots ab. Keine Änderungen nötig.\n',
|
||||
mode: 'diff',
|
||||
}
|
||||
}
|
||||
|
||||
const header =
|
||||
`# Folgende Blöcke zu deiner bestehenden robots.txt hinzufügen
|
||||
# (am Ende der Datei, vor der Sitemap-Zeile falls vorhanden)
|
||||
|
||||
`
|
||||
return {
|
||||
content: header + missing.map(botBlock).join('\n'),
|
||||
mode: 'diff',
|
||||
}
|
||||
}
|
||||
26
server/lib/cache.js
Normal file
26
server/lib/cache.js
Normal file
@@ -0,0 +1,26 @@
|
||||
// Simple in-memory LRU with TTL.
|
||||
// Map preserves insertion order — evict oldest on overflow; refresh on read.
|
||||
const TTL_MS = 60 * 60 * 1000
|
||||
const MAX_ENTRIES = 1000
|
||||
|
||||
const store = new Map()
|
||||
|
||||
export function cacheGet(key) {
|
||||
const entry = store.get(key)
|
||||
if (!entry) return null
|
||||
if (Date.now() > entry.expiresAt) {
|
||||
store.delete(key)
|
||||
return null
|
||||
}
|
||||
store.delete(key)
|
||||
store.set(key, entry)
|
||||
return entry.value
|
||||
}
|
||||
|
||||
export function cacheSet(key, value) {
|
||||
if (store.size >= MAX_ENTRIES) {
|
||||
const oldest = store.keys().next().value
|
||||
store.delete(oldest)
|
||||
}
|
||||
store.set(key, { value, expiresAt: Date.now() + TTL_MS })
|
||||
}
|
||||
131
server/lib/fetcher.js
Normal file
131
server/lib/fetcher.js
Normal file
@@ -0,0 +1,131 @@
|
||||
import { lookup } from 'node:dns/promises'
|
||||
import net from 'node:net'
|
||||
|
||||
const DEFAULT_UA =
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
|
||||
|
||||
// Blocks RFC 1918 private ranges, loopback, link-local, AWS metadata
|
||||
// (169.254.0.0/16), multicast/reserved, and IPv6 equivalents.
|
||||
function isPrivateOrLoopback(addr) {
|
||||
if (net.isIPv4(addr)) {
|
||||
const p = addr.split('.').map(Number)
|
||||
return (
|
||||
p[0] === 0 ||
|
||||
p[0] === 10 ||
|
||||
p[0] === 127 ||
|
||||
(p[0] === 169 && p[1] === 254) ||
|
||||
(p[0] === 172 && p[1] >= 16 && p[1] <= 31) ||
|
||||
(p[0] === 192 && p[1] === 168) ||
|
||||
p[0] >= 224
|
||||
)
|
||||
}
|
||||
if (net.isIPv6(addr)) {
|
||||
const a = addr.toLowerCase()
|
||||
return (
|
||||
a === '::' ||
|
||||
a === '::1' ||
|
||||
a.startsWith('fe80:') ||
|
||||
a.startsWith('fc') ||
|
||||
a.startsWith('fd') ||
|
||||
a.startsWith('::ffff:')
|
||||
)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Resolves hostname; throws { code: 'PRIVATE_HOST_BLOCKED' | 'ENOTFOUND' }.
|
||||
// Skipped entirely when ALLOW_PRIVATE_HOSTS=1 (local dev).
|
||||
async function assertPublicHost(hostname) {
|
||||
if (process.env.ALLOW_PRIVATE_HOSTS === '1') return
|
||||
let address
|
||||
try {
|
||||
({ address } = await lookup(hostname))
|
||||
} catch (err) {
|
||||
const e = new Error('ENOTFOUND')
|
||||
e.code = 'ENOTFOUND'
|
||||
e.cause = err
|
||||
throw e
|
||||
}
|
||||
if (isPrivateOrLoopback(address)) {
|
||||
const e = new Error('PRIVATE_HOST_BLOCKED')
|
||||
e.code = 'PRIVATE_HOST_BLOCKED'
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// Returns { status, headers, body, finalUrl, ms, error? } — never throws.
|
||||
// `error` is a string code: 'PRIVATE_HOST_BLOCKED' | 'ENOTFOUND' | 'TIMEOUT' | 'TLS_INVALID' | 'NETWORK'.
|
||||
export async function fetchPage(url, { userAgent, timeoutMs = 10000 } = {}) {
|
||||
const started = Date.now()
|
||||
let hostname
|
||||
try {
|
||||
hostname = new URL(url).hostname
|
||||
} catch {
|
||||
return { status: 0, headers: {}, body: '', finalUrl: url, ms: 0, error: 'INVALID_URL' }
|
||||
}
|
||||
|
||||
try {
|
||||
await assertPublicHost(hostname)
|
||||
} catch (err) {
|
||||
return {
|
||||
status: 0,
|
||||
headers: {},
|
||||
body: '',
|
||||
finalUrl: url,
|
||||
ms: Date.now() - started,
|
||||
error: err.code || 'NETWORK',
|
||||
}
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
method: 'GET',
|
||||
redirect: 'follow',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'User-Agent': userAgent || DEFAULT_UA,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
|
||||
'Cache-Control': 'no-cache',
|
||||
},
|
||||
})
|
||||
const body = await res.text()
|
||||
const headers = {}
|
||||
res.headers.forEach((value, key) => { headers[key.toLowerCase()] = value })
|
||||
return {
|
||||
status: res.status,
|
||||
headers,
|
||||
body,
|
||||
finalUrl: res.url || url,
|
||||
ms: Date.now() - started,
|
||||
}
|
||||
} catch (err) {
|
||||
const causeCode = err?.cause?.code
|
||||
const causeMessage = err?.cause?.message || ''
|
||||
const TLS_CODES = new Set([
|
||||
'ERR_TLS_CERT_ALTNAME_INVALID',
|
||||
'CERT_HAS_EXPIRED',
|
||||
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
|
||||
'SELF_SIGNED_CERT_IN_CHAIN',
|
||||
'DEPTH_ZERO_SELF_SIGNED_CERT',
|
||||
'ERR_SSL_WRONG_VERSION_NUMBER',
|
||||
])
|
||||
let code = 'NETWORK'
|
||||
if (err?.name === 'AbortError') code = 'TIMEOUT'
|
||||
else if (TLS_CODES.has(causeCode) || /certificate|altnames|self.?signed|TLS|SSL/i.test(causeMessage)) code = 'SSL_INVALID'
|
||||
else if (causeCode === 'ENOTFOUND' || causeCode === 'EAI_AGAIN') code = 'ENOTFOUND'
|
||||
else if (causeCode === 'ETIMEDOUT') code = 'TIMEOUT'
|
||||
return {
|
||||
status: 0,
|
||||
headers: {},
|
||||
body: '',
|
||||
finalUrl: url,
|
||||
ms: Date.now() - started,
|
||||
error: code,
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
38
server/lib/monitoring/detect-mention.js
Normal file
38
server/lib/monitoring/detect-mention.js
Normal file
@@ -0,0 +1,38 @@
|
||||
// Case-insensitive substring search for brand identifiers in LLM output.
|
||||
// Returns the first match's position and a ~200-char surrounding snippet.
|
||||
export function detectMention(text, client) {
|
||||
let aliases = []
|
||||
try { aliases = JSON.parse(client.brand_aliases || '[]') } catch { /* keep [] */ }
|
||||
|
||||
const candidates = [
|
||||
client.name,
|
||||
client.hostname?.split('.')[0],
|
||||
...aliases,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.map((s) => String(s).toLowerCase().trim())
|
||||
.filter((s) => s.length >= 3)
|
||||
.filter((v, i, a) => a.indexOf(v) === i)
|
||||
|
||||
if (!text || candidates.length === 0) {
|
||||
return { mentioned: false, position: null, snippet: null }
|
||||
}
|
||||
|
||||
const lower = text.toLowerCase()
|
||||
let best = null
|
||||
for (const candidate of candidates) {
|
||||
const idx = lower.indexOf(candidate)
|
||||
if (idx >= 0 && (best === null || idx < best.position)) {
|
||||
best = { position: idx, candidate }
|
||||
}
|
||||
}
|
||||
if (!best) return { mentioned: false, position: null, snippet: null }
|
||||
|
||||
const start = Math.max(0, best.position - 100)
|
||||
const end = Math.min(text.length, best.position + best.candidate.length + 100)
|
||||
let snippet = text.slice(start, end).trim()
|
||||
if (start > 0) snippet = '…' + snippet
|
||||
if (end < text.length) snippet = snippet + '…'
|
||||
|
||||
return { mentioned: true, position: best.position, snippet }
|
||||
}
|
||||
88
server/lib/monitoring/generate-queries.js
Normal file
88
server/lib/monitoring/generate-queries.js
Normal file
@@ -0,0 +1,88 @@
|
||||
// Generate ~10 German search queries that simulate how a potential customer
|
||||
// would ask an AI assistant. Brand name is intentionally excluded so the
|
||||
// monitoring run can measure whether the AI surfaces the brand without bias.
|
||||
const MISTRAL_ENDPOINT = 'https://api.mistral.ai/v1/chat/completions'
|
||||
const MISTRAL_MODEL = 'mistral-large-latest'
|
||||
|
||||
function buildPrompt(siteData) {
|
||||
const name = siteData?.name || 'Unbekanntes Unternehmen'
|
||||
const description = siteData?.description || '[keine Beschreibung verfügbar]'
|
||||
const url = siteData?.url || siteData?.hostname || ''
|
||||
return (
|
||||
`Du bist ein GEO-Analyst (Generative Engine Optimization). Generiere genau 10 deutschsprachige Suchanfragen, die ein echter Nutzer in einer KI-Suchmaschine wie ChatGPT, Perplexity oder Claude eingeben würde, um Unternehmen wie das untenstehende zu finden.
|
||||
|
||||
Kontext:
|
||||
- Firma: ${name}
|
||||
- Beschreibung: ${description}
|
||||
- Domain: ${url}
|
||||
|
||||
Anforderungen:
|
||||
- Genau 10 Anfragen, eine pro Zeile, nummeriert "1. ... 2. ... 3. ..."
|
||||
- Mischung aus: Service-spezifischen Anfragen, Branchen-Anfragen, regionalen Anfragen (DACH falls passend), Vergleichs-/Empfehlungs-Anfragen
|
||||
- ⚠ KEINE Erwähnung des Firmennamens "${name}" in den Anfragen — der Test prüft, ob die KI das Unternehmen ohne Bias nennt
|
||||
- Realistisch formuliert, wie ein normaler Nutzer schreiben würde (nicht zu förmlich)
|
||||
- Auf Deutsch, mit Du-Form oder neutral
|
||||
|
||||
Ausgabe: nur die nummerierte Liste, keine Erklärungen, keine Einleitung.`
|
||||
)
|
||||
}
|
||||
|
||||
async function callMistral(prompt, { timeoutMs = 30000, maxTokens = 800 } = {}) {
|
||||
const apiKey = process.env.MISTRAL_KEY
|
||||
if (!apiKey) throw new Error('NO_MISTRAL_KEY')
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
||||
try {
|
||||
const res = await fetch(MISTRAL_ENDPOINT, {
|
||||
method: 'POST',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: MISTRAL_MODEL,
|
||||
max_tokens: maxTokens,
|
||||
temperature: 0.7,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
}),
|
||||
})
|
||||
if (!res.ok) throw new Error(`MISTRAL_${res.status}`)
|
||||
const data = await res.json()
|
||||
return data?.choices?.[0]?.message?.content || ''
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
|
||||
// Tolerates slight format drift: optional leading whitespace, number followed
|
||||
// by `.` or `)`, optional trailing punctuation. Filters obvious noise.
|
||||
function parseList(text) {
|
||||
return text.split('\n')
|
||||
.map((l) => l.trim())
|
||||
.map((l) => l.replace(/^\d+\s*[.)\-:]\s*/, ''))
|
||||
.map((l) => l.replace(/^[•\-*]\s*/, ''))
|
||||
.filter((l) => l.length >= 10 && l.length <= 300)
|
||||
.slice(0, 10)
|
||||
}
|
||||
|
||||
export async function generateQueries(siteData) {
|
||||
const prompt = buildPrompt(siteData)
|
||||
let text
|
||||
try {
|
||||
text = await callMistral(prompt)
|
||||
} catch (err) {
|
||||
console.warn('[monitoring/generate-queries]', err?.message || err)
|
||||
return { queries: [], warning: 'GENERATION_FAILED' }
|
||||
}
|
||||
const queries = parseList(text)
|
||||
// Soft filter: drop any query that still contains the brand name (case-insensitive).
|
||||
const brandLower = (siteData?.name || '').toLowerCase().trim()
|
||||
const filtered = brandLower
|
||||
? queries.filter((q) => !q.toLowerCase().includes(brandLower))
|
||||
: queries
|
||||
if (filtered.length < 5) {
|
||||
return { queries: filtered, warning: 'fewer than 10 generated' }
|
||||
}
|
||||
return { queries: filtered }
|
||||
}
|
||||
97
server/lib/monitoring/run.js
Normal file
97
server/lib/monitoring/run.js
Normal file
@@ -0,0 +1,97 @@
|
||||
// One full run executes every active query against every available provider.
|
||||
// Sequential within (query, provider) pairs to be polite with rate limits;
|
||||
// total wall-clock ~= queries × providers × ~2s.
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { getProviders } from '../providers/index.js'
|
||||
import { detectMention } from './detect-mention.js'
|
||||
import * as repo from '../../db/repo.js'
|
||||
|
||||
const INTER_REQUEST_MS = 200
|
||||
|
||||
export class RunError extends Error {
|
||||
constructor(code, message) {
|
||||
super(message || code)
|
||||
this.code = code
|
||||
}
|
||||
}
|
||||
|
||||
export async function runMonitoring(clientId) {
|
||||
const client = repo.getClient(clientId)
|
||||
if (!client) throw new RunError('CLIENT_NOT_FOUND')
|
||||
if (client.status === 'paused' || client.status === 'archived') {
|
||||
throw new RunError('CLIENT_NOT_ACTIVE')
|
||||
}
|
||||
const queries = repo.listActiveQueries(clientId)
|
||||
if (queries.length === 0) throw new RunError('NO_ACTIVE_QUERIES')
|
||||
|
||||
const providers = getProviders()
|
||||
const providerKeys = Object.keys(providers)
|
||||
|
||||
const summary = {
|
||||
runId: randomUUID().slice(0, 8),
|
||||
client_id: clientId,
|
||||
hostname: client.hostname,
|
||||
started_at: new Date().toISOString(),
|
||||
providers: providerKeys,
|
||||
queries: queries.length,
|
||||
totals: { runs: 0, mentions: 0, errors: 0, cost_usd: 0 },
|
||||
}
|
||||
|
||||
const wallStart = Date.now()
|
||||
|
||||
for (const query of queries) {
|
||||
for (const key of providerKeys) {
|
||||
const t0 = Date.now()
|
||||
let result
|
||||
try {
|
||||
result = await providers[key].query(query.text, { brandHint: client.name })
|
||||
} catch (e) {
|
||||
result = {
|
||||
provider: key,
|
||||
content: '',
|
||||
ms: Date.now() - t0,
|
||||
costUsd: 0,
|
||||
error: e?.code || e?.message || 'UNKNOWN',
|
||||
}
|
||||
}
|
||||
|
||||
const mention = result.error
|
||||
? { mentioned: false, position: null, snippet: null }
|
||||
: detectMention(result.content, client)
|
||||
|
||||
repo.insertRun({
|
||||
client_id: clientId,
|
||||
query_id: query.id,
|
||||
provider: result.provider || key,
|
||||
mentioned: mention.mentioned,
|
||||
position: mention.position,
|
||||
snippet: mention.snippet,
|
||||
response_full: result.content || null,
|
||||
ms: result.ms || (Date.now() - t0),
|
||||
cost_usd: result.costUsd || 0,
|
||||
error: result.error || null,
|
||||
})
|
||||
|
||||
summary.totals.runs++
|
||||
if (mention.mentioned) summary.totals.mentions++
|
||||
if (result.error) summary.totals.errors++
|
||||
summary.totals.cost_usd += result.costUsd || 0
|
||||
|
||||
await new Promise((r) => setTimeout(r, INTER_REQUEST_MS))
|
||||
}
|
||||
}
|
||||
|
||||
repo.touchClientRun(clientId)
|
||||
summary.finished_at = new Date().toISOString()
|
||||
summary.totals.cost_usd = Number(summary.totals.cost_usd.toFixed(6))
|
||||
|
||||
const ms = Date.now() - wallStart
|
||||
console.log(
|
||||
`[monitoring] runId=${summary.runId} client=${client.hostname} queries=${queries.length} ` +
|
||||
`providers=${providerKeys.length} runs=${summary.totals.runs} ` +
|
||||
`mentions=${summary.totals.mentions} errors=${summary.totals.errors} ` +
|
||||
`cost_usd=${summary.totals.cost_usd} ms=${ms}`
|
||||
)
|
||||
summary.ms = ms
|
||||
return summary
|
||||
}
|
||||
16
server/lib/parser.js
Normal file
16
server/lib/parser.js
Normal file
@@ -0,0 +1,16 @@
|
||||
// Extracts the parts of the HTML response that downstream checks need.
|
||||
export function parseHtml(html) {
|
||||
const safe = html || ''
|
||||
const headMatch = safe.match(/<head[\s\S]*?<\/head>/i)
|
||||
const headHtml = headMatch ? headMatch[0] : safe.slice(0, 4000)
|
||||
|
||||
const jsonLdBlocks = [
|
||||
...safe.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi),
|
||||
]
|
||||
.map((m) => m[1].trim())
|
||||
.filter(Boolean)
|
||||
|
||||
const jsonLdJoined = jsonLdBlocks.join('\n---\n')
|
||||
|
||||
return { headHtml, jsonLdBlocks, jsonLdJoined }
|
||||
}
|
||||
191
server/lib/pipeline.js
Normal file
191
server/lib/pipeline.js
Normal file
@@ -0,0 +1,191 @@
|
||||
import { fetchPage } from './fetcher.js'
|
||||
import { parseHtml } from './parser.js'
|
||||
import { computeScore } from './scoring.js'
|
||||
import { generateSummary } from './summary.js'
|
||||
import { runAllChecks } from '../checks/index.js'
|
||||
import { generateAutofix } from './autofix/index.js'
|
||||
|
||||
const SEVERITY_ORDER = { high: 0, medium: 1, low: 2 }
|
||||
const MAX_ISSUES = 8
|
||||
const MAX_URL_LENGTH = 2048
|
||||
|
||||
// Map internal fetch error codes → { httpStatus, germanMessage }.
|
||||
export const FETCH_ERROR_MAP = {
|
||||
PRIVATE_HOST_BLOCKED: { http: 400, msg: 'Diese Adresse kann nicht analysiert werden.' },
|
||||
ENOTFOUND: { http: 502, msg: 'Die Website ist nicht erreichbar. Bitte URL prüfen.' },
|
||||
TIMEOUT: { http: 504, msg: 'Die Analyse hat zu lange gedauert. Bitte später erneut versuchen.' },
|
||||
SSL_INVALID: { http: 502, msg: 'SSL-Zertifikat der Website ist ungültig. Bitte den Hosting-Anbieter kontaktieren.' },
|
||||
NETWORK: { http: 502, msg: 'Die Website ist nicht erreichbar. Bitte URL prüfen.' },
|
||||
}
|
||||
|
||||
// HTML payload masquerading as text/plain (SPA fallback, custom 404) → treat as empty.
|
||||
function looksLikeText(body) {
|
||||
if (!body) return false
|
||||
return !body.trimStart().startsWith('<')
|
||||
}
|
||||
|
||||
export function validateUrl(rawInput) {
|
||||
if (typeof rawInput !== 'string' || rawInput.trim().length === 0) {
|
||||
return { error: { http: 400, code: 'EMPTY_URL', msg: 'Bitte eine URL eingeben.' } }
|
||||
}
|
||||
let raw = rawInput.trim()
|
||||
if (raw.length > MAX_URL_LENGTH) {
|
||||
return { error: { http: 400, code: 'URL_TOO_LONG', msg: 'URL ist zu lang.' } }
|
||||
}
|
||||
if (/^http:\/\//i.test(raw)) {
|
||||
return { error: { http: 400, code: 'HTTP_NOT_SUPPORTED', msg: 'HTTP wird nicht unterstützt. Bitte eine HTTPS-URL verwenden.' } }
|
||||
}
|
||||
raw = raw.replace(/^https?:\/\//i, '').replace(/\/+$/, '')
|
||||
const targetUrl = 'https://' + raw
|
||||
try {
|
||||
const u = new URL(targetUrl)
|
||||
return { targetUrl, host: u.hostname, originalOrigin: u.origin }
|
||||
} catch {
|
||||
return { error: { http: 400, code: 'INVALID_URL', msg: 'Ungültige URL. Bitte eine gültige Domain eingeben.' } }
|
||||
}
|
||||
}
|
||||
|
||||
// Runs the full analysis pipeline against an already-validated targetUrl.
|
||||
// Returns either `{ error: { http, code, msg } }` for hard fetch failures,
|
||||
// or `{ data: { score, summary, issues, autofix }, debugPayload, mainStatus }`.
|
||||
export async function runAnalysisPipeline(targetUrl, { debugMode = false } = {}) {
|
||||
const main = await fetchPage(targetUrl)
|
||||
|
||||
if (main.status === 0 && main.error && FETCH_ERROR_MAP[main.error]) {
|
||||
const mapping = FETCH_ERROR_MAP[main.error]
|
||||
return { error: { http: mapping.http, code: main.error, msg: mapping.msg } }
|
||||
}
|
||||
|
||||
const finalUrl = main.finalUrl || targetUrl
|
||||
let finalOrigin
|
||||
try {
|
||||
finalOrigin = new URL(finalUrl).origin
|
||||
} catch {
|
||||
finalOrigin = new URL(targetUrl).origin
|
||||
}
|
||||
|
||||
const [robotsRes, llmsRes] = await Promise.all([
|
||||
fetchPage(`${finalOrigin}/robots.txt`),
|
||||
fetchPage(`${finalOrigin}/llms.txt`),
|
||||
])
|
||||
|
||||
const robotsTxt = robotsRes.status === 200 && looksLikeText(robotsRes.body) ? robotsRes.body : ''
|
||||
const llmsTxt = llmsRes.status === 200 && looksLikeText(llmsRes.body) ? llmsRes.body : ''
|
||||
const llmsStatusEffective = llmsTxt ? llmsRes.status : 0
|
||||
|
||||
const { headHtml, jsonLdBlocks, jsonLdJoined } = parseHtml(main.body)
|
||||
|
||||
const probeRecords = {}
|
||||
const recordingFetch = async (url, opts) => {
|
||||
const r = await fetchPage(url, opts)
|
||||
const key = opts?.userAgent
|
||||
? (/Claude/i.test(opts.userAgent) ? 'uaClaudeBot'
|
||||
: /GPT/i.test(opts.userAgent) ? 'uaGptBot'
|
||||
: 'uaCustom')
|
||||
: (url.endsWith('/sitemap.xml') ? 'sitemap'
|
||||
: url.endsWith('/llms-full.txt') ? 'llmsFull'
|
||||
: 'extra')
|
||||
probeRecords[key] = { status: r.status, finalUrl: r.finalUrl, bodyLength: (r.body || '').length, ms: r.ms }
|
||||
return r
|
||||
}
|
||||
|
||||
const context = {
|
||||
baseUrl: finalOrigin,
|
||||
targetUrl: finalUrl,
|
||||
html: main.body || '',
|
||||
headHtml,
|
||||
jsonLdBlocks,
|
||||
jsonLdJoined,
|
||||
robotsTxt,
|
||||
llmsTxt,
|
||||
llmsStatus: llmsStatusEffective,
|
||||
mainStatus: main.status,
|
||||
responseHeaders: main.headers || {},
|
||||
fetchPage: recordingFetch,
|
||||
}
|
||||
|
||||
const results = await runAllChecks(context)
|
||||
const score = computeScore(results)
|
||||
|
||||
const failed = results
|
||||
.filter((r) => r.passed === false)
|
||||
.sort((a, b) => (SEVERITY_ORDER[a.severity] ?? 9) - (SEVERITY_ORDER[b.severity] ?? 9))
|
||||
|
||||
const summary = failed.length === 0
|
||||
? 'Alle GEO- und SEO-Signale sind vorhanden. Die Website ist optimal für KI-Suche konfiguriert.'
|
||||
: await generateSummary(failed.map((r) => r.title), finalUrl)
|
||||
|
||||
const issues = failed.slice(0, MAX_ISSUES).map((r) => ({
|
||||
title: r.title,
|
||||
severity: r.severity,
|
||||
}))
|
||||
|
||||
// Generate autofix from the same context the checks ran on.
|
||||
const autofixFull = generateAutofix(context)
|
||||
const { _siteData, ...autofixPublic } = autofixFull
|
||||
|
||||
const data = { score, summary, issues, autofix: autofixPublic }
|
||||
|
||||
// Surface failed-check IDs for activity logging only. Never returned to client.
|
||||
const failedCheckIds = failed.map((r) => r.id)
|
||||
|
||||
const debugPayload = debugMode
|
||||
? {
|
||||
requestedUrl: targetUrl,
|
||||
finalUrl,
|
||||
finalOrigin,
|
||||
fetches: {
|
||||
main: { status: main.status, finalUrl: main.finalUrl, bodyLength: (main.body || '').length, ms: main.ms, error: main.error },
|
||||
robots: { status: robotsRes.status, finalUrl: robotsRes.finalUrl, bodyLength: (robotsRes.body || '').length, ms: robotsRes.ms, error: robotsRes.error },
|
||||
llms: { status: llmsRes.status, finalUrl: llmsRes.finalUrl, bodyLength: (llmsRes.body || '').length, ms: llmsRes.ms, error: llmsRes.error },
|
||||
...probeRecords,
|
||||
},
|
||||
checks: results.map((r) => ({ id: r.id, passed: r.passed, severity: r.severity })),
|
||||
siteData: _siteData,
|
||||
}
|
||||
: null
|
||||
|
||||
return { data, debugPayload, mainStatus: main.status, _siteData, failedCheckIds }
|
||||
}
|
||||
|
||||
// Strips actionable content from the full pipeline output before sending to
|
||||
// unauthenticated visitors. The full data stays in the cache so admin and
|
||||
// future paid flows can still read it.
|
||||
function lineCount(s) {
|
||||
return (s || '').trim().split('\n').filter(Boolean).length
|
||||
}
|
||||
|
||||
function publicLabelFor(file) {
|
||||
switch (file?.mode) {
|
||||
case 'new': return 'neue Datei'
|
||||
case 'replace': return 'bestehende Datei ersetzen'
|
||||
case 'diff': return `${lineCount(file.content)} Zeilen ergänzen`
|
||||
case 'enhance': return 'erweitern'
|
||||
default: return ''
|
||||
}
|
||||
}
|
||||
|
||||
export function toPublicResponse(data) {
|
||||
const issues = data?.issues || []
|
||||
const issueCounts = issues.reduce(
|
||||
(acc, i) => { acc[i.severity] = (acc[i.severity] || 0) + 1; return acc },
|
||||
{ high: 0, medium: 0, low: 0 }
|
||||
)
|
||||
|
||||
const body = {
|
||||
score: data.score,
|
||||
summary: data.summary,
|
||||
issueCounts,
|
||||
}
|
||||
|
||||
// Score-10 sites get nothing to fix — omit the teaser entirely.
|
||||
const af = data.autofix
|
||||
if (af && issues.length > 0) {
|
||||
body.autofix = {
|
||||
llmsTxt: { mode: af.llmsTxt.mode, label: publicLabelFor(af.llmsTxt) },
|
||||
robotsTxt: { mode: af.robotsTxt.mode, label: publicLabelFor(af.robotsTxt) },
|
||||
jsonLd: { mode: af.jsonLd.mode, label: publicLabelFor(af.jsonLd) },
|
||||
}
|
||||
}
|
||||
return body
|
||||
}
|
||||
56
server/lib/providers/anthropic.js
Normal file
56
server/lib/providers/anthropic.js
Normal file
@@ -0,0 +1,56 @@
|
||||
// Anthropic Messages API. Cheapest current Haiku model.
|
||||
const ENDPOINT = 'https://api.anthropic.com/v1/messages'
|
||||
const MODEL = 'claude-haiku-4-5-20251001'
|
||||
const TIMEOUT_MS = 30_000
|
||||
const SYSTEM_PROMPT =
|
||||
'You are a helpful AI search assistant. Answer the user\'s question concisely, in German if the question is in German. Recommend specific companies, services, or products when relevant.'
|
||||
|
||||
// Approximate per-1M token rates in USD for Haiku 4.5.
|
||||
const COST = { inputPer1M: 1.00, outputPer1M: 5.00 }
|
||||
|
||||
async function once(prompt, signal) {
|
||||
return fetch(ENDPOINT, {
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-api-key': process.env.ANTHROPIC_KEY,
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: MODEL,
|
||||
max_tokens: 600,
|
||||
temperature: 0.3,
|
||||
system: SYSTEM_PROMPT,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
export async function query(prompt) {
|
||||
const t0 = Date.now()
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS)
|
||||
try {
|
||||
let res = await once(prompt, controller.signal)
|
||||
if (res.status >= 500) {
|
||||
res = await once(prompt, controller.signal)
|
||||
}
|
||||
if (!res.ok) {
|
||||
return { provider: 'anthropic', content: '', ms: Date.now() - t0, costUsd: 0, error: `ANTHROPIC_${res.status}` }
|
||||
}
|
||||
const data = await res.json()
|
||||
// Claude returns content as an array of blocks. Concatenate `text` blocks.
|
||||
const blocks = Array.isArray(data?.content) ? data.content : []
|
||||
const content = blocks.map((b) => b?.text || '').join('')
|
||||
const inTok = data?.usage?.input_tokens || 0
|
||||
const outTok = data?.usage?.output_tokens || 0
|
||||
const costUsd = (inTok * COST.inputPer1M + outTok * COST.outputPer1M) / 1_000_000
|
||||
return { provider: 'anthropic', content, ms: Date.now() - t0, costUsd, error: null }
|
||||
} catch (e) {
|
||||
const code = e?.name === 'AbortError' ? 'TIMEOUT' : 'NETWORK'
|
||||
return { provider: 'anthropic', content: '', ms: Date.now() - t0, costUsd: 0, error: code }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
32
server/lib/providers/index.js
Normal file
32
server/lib/providers/index.js
Normal file
@@ -0,0 +1,32 @@
|
||||
// Registry. If a provider's API key is missing, fall back to mock but keep
|
||||
// the original label (so the UI displays "openai (mock)" etc.).
|
||||
import * as openai from './openai.js'
|
||||
import * as perplexity from './perplexity.js'
|
||||
import * as anthropic from './anthropic.js'
|
||||
import * as mock from './mock.js'
|
||||
|
||||
function wrapAsMock(label) {
|
||||
return {
|
||||
query: async (prompt, opts) => {
|
||||
const r = await mock.query(prompt, opts)
|
||||
return { ...r, provider: `${label} (mock)` }
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function getProviders() {
|
||||
return {
|
||||
openai: process.env.OPENAI_KEY ? openai : wrapAsMock('openai'),
|
||||
perplexity: process.env.PERPLEXITY_KEY ? perplexity : wrapAsMock('perplexity'),
|
||||
anthropic: process.env.ANTHROPIC_KEY ? anthropic : wrapAsMock('anthropic'),
|
||||
}
|
||||
}
|
||||
|
||||
// Returns labels for the UI: { openai: 'real' | 'mock', ... }.
|
||||
export function getProviderModes() {
|
||||
return {
|
||||
openai: process.env.OPENAI_KEY ? 'real' : 'mock',
|
||||
perplexity: process.env.PERPLEXITY_KEY ? 'real' : 'mock',
|
||||
anthropic: process.env.ANTHROPIC_KEY ? 'real' : 'mock',
|
||||
}
|
||||
}
|
||||
11
server/lib/providers/mock.js
Normal file
11
server/lib/providers/mock.js
Normal file
@@ -0,0 +1,11 @@
|
||||
// Deterministic fake provider for tests and zero-key dev. The returned content
|
||||
// sometimes includes the brand name (hash-driven) to exercise mention detection.
|
||||
export async function query(prompt, { brandHint } = {}) {
|
||||
await new Promise((r) => setTimeout(r, 80))
|
||||
const hash = String(prompt).split('').reduce((a, c) => a + c.charCodeAt(0), 0)
|
||||
const willMention = brandHint && (hash % 2 === 0)
|
||||
const content = willMention
|
||||
? `Eine Option wäre ${brandHint}. Es gibt auch andere Anbieter wie Beispiel-AG und Muster GmbH.`
|
||||
: `Mögliche Anbieter in diesem Bereich sind unter anderem Beispiel-AG, Muster GmbH und Test Solutions.`
|
||||
return { provider: 'mock', content, ms: 80, costUsd: 0, error: null }
|
||||
}
|
||||
56
server/lib/providers/openai.js
Normal file
56
server/lib/providers/openai.js
Normal file
@@ -0,0 +1,56 @@
|
||||
// OpenAI chat completions provider. Uses the cheapest GPT-4-class model.
|
||||
const ENDPOINT = 'https://api.openai.com/v1/chat/completions'
|
||||
const MODEL = 'gpt-4o-mini'
|
||||
const TIMEOUT_MS = 30_000
|
||||
const SYSTEM_PROMPT =
|
||||
'You are a helpful AI search assistant. Answer the user\'s question concisely, in German if the question is in German. Recommend specific companies, services, or products when relevant.'
|
||||
|
||||
// Approximate per-1M token rates in USD for gpt-4o-mini.
|
||||
const COST = { inputPer1M: 0.15, outputPer1M: 0.60 }
|
||||
|
||||
async function once(prompt, signal) {
|
||||
const res = await fetch(ENDPOINT, {
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${process.env.OPENAI_KEY}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: MODEL,
|
||||
temperature: 0.3,
|
||||
max_tokens: 600,
|
||||
messages: [
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: prompt },
|
||||
],
|
||||
}),
|
||||
})
|
||||
return res
|
||||
}
|
||||
|
||||
export async function query(prompt) {
|
||||
const t0 = Date.now()
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS)
|
||||
try {
|
||||
let res = await once(prompt, controller.signal)
|
||||
if (res.status >= 500) {
|
||||
res = await once(prompt, controller.signal)
|
||||
}
|
||||
if (!res.ok) {
|
||||
return { provider: 'openai', content: '', ms: Date.now() - t0, costUsd: 0, error: `OPENAI_${res.status}` }
|
||||
}
|
||||
const data = await res.json()
|
||||
const content = data?.choices?.[0]?.message?.content || ''
|
||||
const inTok = data?.usage?.prompt_tokens || 0
|
||||
const outTok = data?.usage?.completion_tokens || 0
|
||||
const costUsd = (inTok * COST.inputPer1M + outTok * COST.outputPer1M) / 1_000_000
|
||||
return { provider: 'openai', content, ms: Date.now() - t0, costUsd, error: null }
|
||||
} catch (e) {
|
||||
const code = e?.name === 'AbortError' ? 'TIMEOUT' : 'NETWORK'
|
||||
return { provider: 'openai', content: '', ms: Date.now() - t0, costUsd: 0, error: code }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
64
server/lib/providers/perplexity.js
Normal file
64
server/lib/providers/perplexity.js
Normal file
@@ -0,0 +1,64 @@
|
||||
// Perplexity is the highest-priority provider — it is an actual AI search
|
||||
// engine, where real users go for recommendations. OpenAI-compatible schema.
|
||||
const ENDPOINT = 'https://api.perplexity.ai/chat/completions'
|
||||
const MODEL = 'sonar'
|
||||
const TIMEOUT_MS = 30_000
|
||||
const SYSTEM_PROMPT =
|
||||
'You are a helpful AI search assistant. Answer the user\'s question concisely, in German if the question is in German. Recommend specific companies, services, or products when relevant.'
|
||||
|
||||
// Approximate per-1M token rates in USD for `sonar`.
|
||||
const COST = { inputPer1M: 1.00, outputPer1M: 1.00 }
|
||||
|
||||
async function once(prompt, signal) {
|
||||
return fetch(ENDPOINT, {
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${process.env.PERPLEXITY_KEY}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: MODEL,
|
||||
temperature: 0.3,
|
||||
max_tokens: 600,
|
||||
messages: [
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: prompt },
|
||||
],
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
export async function query(prompt) {
|
||||
const t0 = Date.now()
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS)
|
||||
try {
|
||||
let res = await once(prompt, controller.signal)
|
||||
if (res.status >= 500) {
|
||||
res = await once(prompt, controller.signal)
|
||||
}
|
||||
if (!res.ok) {
|
||||
return { provider: 'perplexity', content: '', ms: Date.now() - t0, costUsd: 0, error: `PERPLEXITY_${res.status}` }
|
||||
}
|
||||
const data = await res.json()
|
||||
const content = data?.choices?.[0]?.message?.content || ''
|
||||
const inTok = data?.usage?.prompt_tokens || 0
|
||||
const outTok = data?.usage?.completion_tokens || 0
|
||||
// Fall back to a character-based rough estimate if usage is absent.
|
||||
let costUsd
|
||||
if (inTok || outTok) {
|
||||
costUsd = (inTok * COST.inputPer1M + outTok * COST.outputPer1M) / 1_000_000
|
||||
} else {
|
||||
const approxIn = Math.ceil((prompt.length + SYSTEM_PROMPT.length) / 4)
|
||||
const approxOut = Math.ceil(content.length / 4)
|
||||
costUsd = (approxIn * COST.inputPer1M + approxOut * COST.outputPer1M) / 1_000_000
|
||||
}
|
||||
return { provider: 'perplexity', content, ms: Date.now() - t0, costUsd, error: null }
|
||||
} catch (e) {
|
||||
const code = e?.name === 'AbortError' ? 'TIMEOUT' : 'NETWORK'
|
||||
return { provider: 'perplexity', content: '', ms: Date.now() - t0, costUsd: 0, error: code }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
15
server/lib/scoring.js
Normal file
15
server/lib/scoring.js
Normal file
@@ -0,0 +1,15 @@
|
||||
const WEIGHTS = { high: 3, medium: 2, low: 1 }
|
||||
|
||||
export function computeScore(results) {
|
||||
if (!Array.isArray(results) || results.length === 0) return 1
|
||||
let earned = 0
|
||||
let max = 0
|
||||
for (const r of results) {
|
||||
const w = WEIGHTS[r.severity] || 1
|
||||
max += w
|
||||
if (r.passed) earned += w
|
||||
}
|
||||
if (max === 0) return 1
|
||||
const raw = 1 + 9 * (earned / max)
|
||||
return Math.max(1, Math.min(10, Math.round(raw)))
|
||||
}
|
||||
42
server/lib/summary.js
Normal file
42
server/lib/summary.js
Normal file
@@ -0,0 +1,42 @@
|
||||
const SUMMARY_PROMPT = `You are a GEO/SEO auditor. The following checks were run on a website. Some FAILED.
|
||||
Write a summary in German: 1-2 sentences describing what is missing and why it matters for AI visibility.
|
||||
Be specific about which signals are missing. Do not invent checks — only describe the ones listed.
|
||||
Return ONLY the summary text, no JSON, no markdown.`
|
||||
|
||||
function fallback(failedTitles) {
|
||||
return `${failedTitles.length} GEO/SEO-Signale fehlen. Behebe die aufgelisteten Probleme für bessere KI-Sichtbarkeit.`
|
||||
}
|
||||
|
||||
export async function generateSummary(failedTitles, targetUrl) {
|
||||
const apiKey = process.env.MISTRAL_KEY
|
||||
if (!apiKey || failedTitles.length === 0) return fallback(failedTitles)
|
||||
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), 10000)
|
||||
try {
|
||||
const res = await fetch('https://api.mistral.ai/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'mistral-large-latest',
|
||||
max_tokens: 150,
|
||||
messages: [
|
||||
{ role: 'system', content: SUMMARY_PROMPT },
|
||||
{ role: 'user', content: `Website: ${targetUrl}\nFehlende Signale: ${failedTitles.join(', ')}` },
|
||||
],
|
||||
}),
|
||||
})
|
||||
if (!res.ok) return fallback(failedTitles)
|
||||
const data = await res.json()
|
||||
const text = (data?.choices?.[0]?.message?.content || '').trim()
|
||||
return text || fallback(failedTitles)
|
||||
} catch {
|
||||
return fallback(failedTitles)
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user