Initial commit: Visigine (Vite client + Express/SQLite backend)
Container-ready via docker/ compose (frontend nginx + backend Node). Compose adjusted for Coolify on the prod server: frontend uses expose:80 (no host binding — host 8080 is taken by the Coolify proxy; Traefik routes visigine.de), backend ALLOWED_ORIGINS=https://visigine.de. Secrets stay in server/.env (git-ignored); see server/.env.example. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
30
server/.env.example
Normal file
30
server/.env.example
Normal file
@@ -0,0 +1,30 @@
|
||||
# Mistral API key — server-only, never exposed to the browser.
|
||||
MISTRAL_KEY=
|
||||
|
||||
# Port the Express server binds to.
|
||||
PORT=3001
|
||||
|
||||
# Comma-separated CORS whitelist. Empty/missing = deny all browser origins.
|
||||
ALLOWED_ORIGINS=http://localhost:5173
|
||||
|
||||
# 'development' enables ?debug=1; 'production' strips _debug unconditionally.
|
||||
NODE_ENV=development
|
||||
|
||||
# SSRF guard. Set to 1 ONLY in local dev if you want to scan localhost services.
|
||||
ALLOW_PRIVATE_HOSTS=0
|
||||
|
||||
# Required for /admin to function. Leave empty to disable admin endpoints entirely.
|
||||
# Generate with: node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
|
||||
ADMIN_TOKEN=
|
||||
|
||||
# Monitoring providers — optional. When a key is missing, that provider
|
||||
# transparently falls back to the deterministic `mock` provider, labelled
|
||||
# e.g. "openai (mock)" in the UI. Zero-key dev works end-to-end.
|
||||
OPENAI_KEY=
|
||||
PERPLEXITY_KEY=
|
||||
ANTHROPIC_KEY=
|
||||
|
||||
# SQLite path. Defaults to server/data/visigine.db. Override for tests:
|
||||
# DB_PATH=/tmp/visigine-test.db
|
||||
DB_PATH=
|
||||
|
||||
27
server/checks/ai-bots.js
Normal file
27
server/checks/ai-bots.js
Normal file
@@ -0,0 +1,27 @@
|
||||
// Group A — AI bots referenced in robots.txt. Substring match is intentional:
|
||||
// canonical bot tokens are case-sensitive and stable.
|
||||
const BOTS = [
|
||||
{ id: 'ai-bots.gptbot', token: 'GPTBot', severity: 'high', title: 'GPTBot nicht in robots.txt' },
|
||||
{ id: 'ai-bots.claudebot', token: 'ClaudeBot', severity: 'high', title: 'ClaudeBot nicht in robots.txt' },
|
||||
{ id: 'ai-bots.oai-searchbot', token: 'OAI-SearchBot', severity: 'high', title: 'OAI-SearchBot nicht in robots.txt' },
|
||||
{ id: 'ai-bots.perplexitybot', token: 'PerplexityBot', severity: 'high', title: 'PerplexityBot nicht in robots.txt' },
|
||||
{ id: 'ai-bots.bingbot', token: 'Bingbot', severity: 'high', title: 'Bingbot nicht in robots.txt' },
|
||||
{ id: 'ai-bots.google-extended', token: 'Google-Extended', severity: 'medium', title: 'Google-Extended nicht in robots.txt' },
|
||||
{ id: 'ai-bots.googleother', token: 'GoogleOther', severity: 'medium', title: 'GoogleOther nicht in robots.txt' },
|
||||
{ id: 'ai-bots.applebot-extended', token: 'Applebot-Extended', severity: 'medium', title: 'Applebot-Extended nicht in robots.txt' },
|
||||
{ id: 'ai-bots.meta-externalagent', token: 'Meta-ExternalAgent', severity: 'medium', title: 'Meta-ExternalAgent nicht in robots.txt' },
|
||||
{ id: 'ai-bots.ccbot', token: 'CCBot', severity: 'medium', title: 'CCBot (Common Crawl) nicht in robots.txt' },
|
||||
{ id: 'ai-bots.bytespider', token: 'Bytespider', severity: 'low', title: 'Bytespider nicht in robots.txt' },
|
||||
{ id: 'ai-bots.duckassistbot', token: 'DuckAssistBot', severity: 'low', title: 'DuckAssistBot nicht in robots.txt' },
|
||||
{ id: 'ai-bots.chatgpt-user', token: 'ChatGPT-User', severity: 'low', title: 'ChatGPT-User nicht in robots.txt' },
|
||||
]
|
||||
|
||||
export function runAiBotsChecks({ robotsTxt }) {
|
||||
const rb = robotsTxt || ''
|
||||
return BOTS.map(({ id, token, severity, title }) => ({
|
||||
id,
|
||||
title,
|
||||
severity,
|
||||
passed: rb.includes(token),
|
||||
}))
|
||||
}
|
||||
43
server/checks/ai-reachability.js
Normal file
43
server/checks/ai-reachability.js
Normal file
@@ -0,0 +1,43 @@
|
||||
// Group F — probe the target with AI bot user agents to detect WAF/CDN blocking.
|
||||
const BLOCKED_STATUSES = new Set([403, 429, 451, 503, 0])
|
||||
|
||||
const PROBES = [
|
||||
{
|
||||
id: 'ai-reach.claudebot',
|
||||
title: 'ClaudeBot wird blockiert (Cloudflare oder Firewall)',
|
||||
userAgent: 'ClaudeBot/1.0 (+https://www.anthropic.com)',
|
||||
},
|
||||
{
|
||||
id: 'ai-reach.gptbot',
|
||||
title: 'GPTBot wird blockiert (Cloudflare oder Firewall)',
|
||||
userAgent: 'GPTBot/1.0 (+https://openai.com/gptbot)',
|
||||
},
|
||||
]
|
||||
|
||||
export async function runAiReachabilityChecks({ targetUrl, mainStatus, fetchPage }) {
|
||||
// If the baseline fetch already failed, the probes would be misleading —
|
||||
// mark both as failed but skip the network calls.
|
||||
if (mainStatus !== 200) {
|
||||
return PROBES.map((p) => ({
|
||||
id: p.id,
|
||||
title: p.title,
|
||||
severity: 'high',
|
||||
passed: false,
|
||||
}))
|
||||
}
|
||||
|
||||
const probeResults = await Promise.all(
|
||||
PROBES.map((p) => fetchPage(targetUrl, { userAgent: p.userAgent }))
|
||||
)
|
||||
|
||||
return PROBES.map((p, i) => {
|
||||
const status = probeResults[i].status
|
||||
const blocked = BLOCKED_STATUSES.has(status)
|
||||
return {
|
||||
id: p.id,
|
||||
title: p.title,
|
||||
severity: 'high',
|
||||
passed: !blocked,
|
||||
}
|
||||
})
|
||||
}
|
||||
23
server/checks/index.js
Normal file
23
server/checks/index.js
Normal file
@@ -0,0 +1,23 @@
|
||||
// Orchestrates all check groups. Async groups run in parallel; sync groups append.
|
||||
import { runAiBotsChecks } from './ai-bots.js'
|
||||
import { runLlmsChecks } from './llms-txt.js'
|
||||
import { runJsonLdChecks } from './json-ld.js'
|
||||
import { runMetaChecks } from './meta-tags.js'
|
||||
import { runTechnicalChecks } from './technical.js'
|
||||
import { runAiReachabilityChecks } from './ai-reachability.js'
|
||||
|
||||
export async function runAllChecks(context) {
|
||||
const [llms, technical, aiReach] = await Promise.all([
|
||||
runLlmsChecks(context),
|
||||
runTechnicalChecks(context),
|
||||
runAiReachabilityChecks(context),
|
||||
])
|
||||
return [
|
||||
...runAiBotsChecks(context),
|
||||
...llms,
|
||||
...runJsonLdChecks(context),
|
||||
...runMetaChecks(context),
|
||||
...technical,
|
||||
...aiReach,
|
||||
]
|
||||
}
|
||||
82
server/checks/json-ld.js
Normal file
82
server/checks/json-ld.js
Normal file
@@ -0,0 +1,82 @@
|
||||
// Group C — schema.org / JSON-LD coverage. Validity is checked block-by-block;
|
||||
// the remaining matches run against the concatenated raw string for resilience.
|
||||
export function runJsonLdChecks({ jsonLdBlocks, jsonLdJoined }) {
|
||||
const blocks = jsonLdBlocks || []
|
||||
const jl = jsonLdJoined || ''
|
||||
|
||||
let validity = true
|
||||
if (blocks.length > 0) {
|
||||
validity = blocks.some((b) => {
|
||||
try { JSON.parse(b); return true } catch { return false }
|
||||
})
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
id: 'jsonld.valid',
|
||||
title: 'JSON-LD: ungültiges JSON',
|
||||
severity: 'high',
|
||||
passed: validity,
|
||||
},
|
||||
{
|
||||
id: 'jsonld.organization',
|
||||
title: 'JSON-LD: kein LocalBusiness/Organization',
|
||||
severity: 'high',
|
||||
passed: /"@type"\s*:\s*"?(LocalBusiness|Organization)/i.test(jl),
|
||||
},
|
||||
{
|
||||
id: 'jsonld.faqpage',
|
||||
title: 'JSON-LD: kein FAQPage-Schema',
|
||||
severity: 'medium',
|
||||
passed: jl.includes('FAQPage'),
|
||||
},
|
||||
{
|
||||
id: 'jsonld.sameas',
|
||||
title: 'JSON-LD: kein sameAs mit externen URLs',
|
||||
severity: 'medium',
|
||||
passed: /"sameAs"/.test(jl) && /https?:\/\//.test(jl),
|
||||
},
|
||||
{
|
||||
id: 'jsonld.openinghours',
|
||||
title: 'JSON-LD: keine Öffnungszeiten',
|
||||
severity: 'medium',
|
||||
passed: jl.includes('openingHours'),
|
||||
},
|
||||
{
|
||||
id: 'jsonld.breadcrumb',
|
||||
title: 'JSON-LD: keine BreadcrumbList',
|
||||
severity: 'low',
|
||||
passed: jl.includes('BreadcrumbList'),
|
||||
},
|
||||
{
|
||||
id: 'jsonld.website',
|
||||
title: 'JSON-LD: kein WebSite-Schema',
|
||||
severity: 'low',
|
||||
passed: /"@type"\s*:\s*"?WebSite/.test(jl),
|
||||
},
|
||||
{
|
||||
id: 'jsonld.address',
|
||||
title: 'JSON-LD: keine PostalAddress',
|
||||
severity: 'medium',
|
||||
passed: jl.includes('PostalAddress'),
|
||||
},
|
||||
{
|
||||
id: 'jsonld.telephone',
|
||||
title: 'JSON-LD: keine Telefonnummer im Schema',
|
||||
severity: 'low',
|
||||
passed: /"telephone"/.test(jl),
|
||||
},
|
||||
{
|
||||
id: 'jsonld.service-product',
|
||||
title: 'JSON-LD: kein Service- oder Product-Schema',
|
||||
severity: 'low',
|
||||
passed: /"@type"\s*:\s*"?(Service|Product)/i.test(jl),
|
||||
},
|
||||
{
|
||||
id: 'jsonld.article',
|
||||
title: 'JSON-LD: kein Article/BlogPosting-Schema',
|
||||
severity: 'low',
|
||||
passed: /"@type"\s*:\s*"?(Article|BlogPosting)/i.test(jl),
|
||||
},
|
||||
]
|
||||
}
|
||||
50
server/checks/llms-txt.js
Normal file
50
server/checks/llms-txt.js
Normal file
@@ -0,0 +1,50 @@
|
||||
// Group B — llms.txt presence, structure, and accessibility.
|
||||
export async function runLlmsChecks({ baseUrl, llmsTxt, llmsStatus, robotsTxt, fetchPage }) {
|
||||
const ll = llmsTxt || ''
|
||||
const rb = robotsTxt || ''
|
||||
const present = (llmsStatus === 200) && ll.length > 0
|
||||
|
||||
const results = [
|
||||
{
|
||||
id: 'llms.present',
|
||||
title: 'llms.txt fehlt',
|
||||
severity: 'high',
|
||||
passed: present,
|
||||
},
|
||||
{
|
||||
id: 'llms.structured',
|
||||
title: 'llms.txt: keine strukturierten Metadaten',
|
||||
severity: 'high',
|
||||
passed: /^[-*]\s*\w+:/m.test(ll),
|
||||
},
|
||||
{
|
||||
id: 'llms.substantial',
|
||||
title: 'llms.txt zu kurz (unter 500 Zeichen)',
|
||||
severity: 'medium',
|
||||
passed: ll.length >= 500,
|
||||
},
|
||||
{
|
||||
id: 'llms.not-disallowed',
|
||||
title: 'llms.txt ist in robots.txt blockiert',
|
||||
severity: 'high',
|
||||
passed: !/Disallow:\s*\/llms\.txt/i.test(rb),
|
||||
},
|
||||
]
|
||||
|
||||
// llms-full.txt is fetched lazily; only report fail when the base file is reachable.
|
||||
let fullPassed = false
|
||||
try {
|
||||
const full = await fetchPage(`${baseUrl}/llms-full.txt`)
|
||||
fullPassed = full.status === 200 && (full.body || '').length > 0
|
||||
} catch {
|
||||
fullPassed = false
|
||||
}
|
||||
results.push({
|
||||
id: 'llms.full-version',
|
||||
title: 'llms-full.txt fehlt (erweiterte Version)',
|
||||
severity: 'low',
|
||||
passed: fullPassed,
|
||||
})
|
||||
|
||||
return results
|
||||
}
|
||||
68
server/checks/meta-tags.js
Normal file
68
server/checks/meta-tags.js
Normal file
@@ -0,0 +1,68 @@
|
||||
// Group D — head / meta / open graph / twitter / canonical / lang.
|
||||
export function runMetaChecks({ html, headHtml }) {
|
||||
const hh = headHtml || ''
|
||||
const full = html || ''
|
||||
|
||||
const descMatch = hh.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["']/i)
|
||||
const descContent = descMatch ? descMatch[1].trim() : ''
|
||||
|
||||
const titleMatch = hh.match(/<title[^>]*>([\s\S]*?)<\/title>/i)
|
||||
const titleText = titleMatch ? titleMatch[1].trim() : ''
|
||||
|
||||
return [
|
||||
{
|
||||
id: 'meta.canonical',
|
||||
title: 'Canonical-Tag fehlt',
|
||||
severity: 'low',
|
||||
passed: /rel=["']canonical["']/i.test(hh),
|
||||
},
|
||||
{
|
||||
id: 'meta.og-title',
|
||||
title: 'og:title fehlt',
|
||||
severity: 'low',
|
||||
passed: /property=["']og:title["']/i.test(hh),
|
||||
},
|
||||
{
|
||||
id: 'meta.og-image',
|
||||
title: 'og:image fehlt',
|
||||
severity: 'low',
|
||||
passed: /property=["']og:image["']/i.test(hh),
|
||||
},
|
||||
{
|
||||
id: 'meta.og-description',
|
||||
title: 'og:description fehlt',
|
||||
severity: 'low',
|
||||
passed: /property=["']og:description["']/i.test(hh),
|
||||
},
|
||||
{
|
||||
id: 'meta.og-type',
|
||||
title: 'og:type fehlt',
|
||||
severity: 'low',
|
||||
passed: /property=["']og:type["']/i.test(hh),
|
||||
},
|
||||
{
|
||||
id: 'meta.twitter-card',
|
||||
title: 'twitter:card fehlt',
|
||||
severity: 'low',
|
||||
passed: /name=["']twitter:card["']/i.test(hh),
|
||||
},
|
||||
{
|
||||
id: 'meta.description',
|
||||
title: 'Meta-Description fehlt oder zu kurz',
|
||||
severity: 'medium',
|
||||
passed: descContent.length >= 50,
|
||||
},
|
||||
{
|
||||
id: 'meta.title-length',
|
||||
title: 'Title fehlt oder ungeeignete Länge',
|
||||
severity: 'medium',
|
||||
passed: titleText.length >= 20 && titleText.length <= 70,
|
||||
},
|
||||
{
|
||||
id: 'meta.lang-attribute',
|
||||
title: '<html lang="..."> Attribut fehlt',
|
||||
severity: 'medium',
|
||||
passed: /<html[^>]*\slang=/i.test(full),
|
||||
},
|
||||
]
|
||||
}
|
||||
51
server/checks/technical.js
Normal file
51
server/checks/technical.js
Normal file
@@ -0,0 +1,51 @@
|
||||
// Group E — technical signals: sitemap, HSTS, viewport, H1.
|
||||
export async function runTechnicalChecks({ baseUrl, html, headHtml, robotsTxt, responseHeaders, fetchPage }) {
|
||||
const rb = robotsTxt || ''
|
||||
const hh = headHtml || ''
|
||||
const full = html || ''
|
||||
const headers = responseHeaders || {}
|
||||
|
||||
let sitemapReachable = false
|
||||
try {
|
||||
const sm = await fetchPage(`${baseUrl}/sitemap.xml`)
|
||||
const body = (sm.body || '').trimStart()
|
||||
sitemapReachable =
|
||||
sm.status === 200 &&
|
||||
(body.startsWith('<?xml') || body.startsWith('<urlset') || body.startsWith('<sitemapindex'))
|
||||
} catch {
|
||||
sitemapReachable = false
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
id: 'tech.sitemap-referenced',
|
||||
title: 'Sitemap in robots.txt nicht referenziert',
|
||||
severity: 'low',
|
||||
passed: /sitemap:/i.test(rb),
|
||||
},
|
||||
{
|
||||
id: 'tech.sitemap-reachable',
|
||||
title: 'Sitemap.xml nicht erreichbar',
|
||||
severity: 'medium',
|
||||
passed: sitemapReachable,
|
||||
},
|
||||
{
|
||||
id: 'tech.hsts',
|
||||
title: 'HSTS-Header fehlt',
|
||||
severity: 'low',
|
||||
passed: Boolean(headers['strict-transport-security']),
|
||||
},
|
||||
{
|
||||
id: 'tech.viewport',
|
||||
title: 'Viewport-Meta-Tag fehlt (Mobile)',
|
||||
severity: 'medium',
|
||||
passed: /name=["']viewport["']/i.test(hh),
|
||||
},
|
||||
{
|
||||
id: 'tech.h1',
|
||||
title: 'H1-Überschrift fehlt',
|
||||
severity: 'medium',
|
||||
passed: /<h1[\s>]/i.test(full),
|
||||
},
|
||||
]
|
||||
}
|
||||
21
server/db/index.js
Normal file
21
server/db/index.js
Normal file
@@ -0,0 +1,21 @@
|
||||
// Synchronous SQLite initialization. On startup ensure the data directory
|
||||
// exists, open the DB, and run schema.sql (idempotent CREATE IF NOT EXISTS).
|
||||
import Database from 'better-sqlite3'
|
||||
import { readFileSync, mkdirSync } from 'node:fs'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
const defaultPath = resolve(here, '..', 'data', 'visigine.db')
|
||||
const dbPath = process.env.DB_PATH ? resolve(process.env.DB_PATH) : defaultPath
|
||||
|
||||
mkdirSync(dirname(dbPath), { recursive: true })
|
||||
|
||||
export const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL') // better concurrent reads
|
||||
db.pragma('foreign_keys = ON') // enforce ON DELETE CASCADE
|
||||
|
||||
const schema = readFileSync(new URL('./schema.sql', import.meta.url), 'utf8')
|
||||
db.exec(schema)
|
||||
|
||||
console.log(`[db] opened ${dbPath}`)
|
||||
177
server/db/repo.js
Normal file
177
server/db/repo.js
Normal file
@@ -0,0 +1,177 @@
|
||||
// Repository layer. Every exported function is one prepared statement or a
|
||||
// small transaction. No ORM — plain SQL behind named functions.
|
||||
import { db } from './index.js'
|
||||
|
||||
const stmts = {
|
||||
listClients: db.prepare(`
|
||||
SELECT
|
||||
c.*,
|
||||
(SELECT COUNT(*) FROM queries q WHERE q.client_id = c.id) AS queries_count,
|
||||
(SELECT COUNT(*) FROM runs r
|
||||
WHERE r.client_id = c.id
|
||||
AND r.ran_at >= datetime('now', '-30 days')
|
||||
AND r.error IS NULL) AS runs_30d,
|
||||
(SELECT COUNT(*) FROM runs r
|
||||
WHERE r.client_id = c.id
|
||||
AND r.ran_at >= datetime('now', '-30 days')
|
||||
AND r.error IS NULL
|
||||
AND r.mentioned = 1) AS mentions_30d
|
||||
FROM clients c
|
||||
ORDER BY c.hostname
|
||||
`),
|
||||
getClient: db.prepare(`SELECT * FROM clients WHERE id = ?`),
|
||||
getClientByHost: db.prepare(`SELECT * FROM clients WHERE hostname = ?`),
|
||||
insertClient: db.prepare(`
|
||||
INSERT INTO clients (hostname, url, name, description, brand_aliases, language)
|
||||
VALUES (@hostname, @url, @name, @description, @brand_aliases, @language)
|
||||
`),
|
||||
updateClient: db.prepare(`
|
||||
UPDATE clients
|
||||
SET name = COALESCE(@name, name),
|
||||
description = COALESCE(@description, description),
|
||||
brand_aliases = COALESCE(@brand_aliases, brand_aliases),
|
||||
status = COALESCE(@status, status)
|
||||
WHERE id = @id
|
||||
`),
|
||||
deleteClient: db.prepare(`DELETE FROM clients WHERE id = ?`),
|
||||
touchClientRun: db.prepare(`UPDATE clients SET last_run_at = datetime('now') WHERE id = ?`),
|
||||
|
||||
listActiveQueries: db.prepare(`SELECT * FROM queries WHERE client_id = ? AND active = 1 ORDER BY id`),
|
||||
listAllQueries: db.prepare(`SELECT * FROM queries WHERE client_id = ? ORDER BY id`),
|
||||
getQuery: db.prepare(`SELECT * FROM queries WHERE id = ?`),
|
||||
insertQuery: db.prepare(`INSERT INTO queries (client_id, text) VALUES (?, ?)`),
|
||||
updateQuery: db.prepare(`
|
||||
UPDATE queries
|
||||
SET text = COALESCE(@text, text),
|
||||
active = COALESCE(@active, active)
|
||||
WHERE id = @id
|
||||
`),
|
||||
deleteQuery: db.prepare(`DELETE FROM queries WHERE id = ?`),
|
||||
deleteAllQueries: db.prepare(`DELETE FROM queries WHERE client_id = ?`),
|
||||
|
||||
insertRun: db.prepare(`
|
||||
INSERT INTO runs
|
||||
(client_id, query_id, provider, mentioned, position, snippet, response_full, ms, cost_usd, error)
|
||||
VALUES
|
||||
(@client_id, @query_id, @provider, @mentioned, @position, @snippet, @response_full, @ms, @cost_usd, @error)
|
||||
`),
|
||||
getRun: db.prepare(`
|
||||
SELECT r.*, q.text AS query_text FROM runs r
|
||||
JOIN queries q ON q.id = r.query_id
|
||||
WHERE r.id = ?
|
||||
`),
|
||||
recentRuns: db.prepare(`
|
||||
SELECT r.*, q.text AS query_text FROM runs r
|
||||
JOIN queries q ON q.id = r.query_id
|
||||
WHERE r.client_id = ?
|
||||
ORDER BY r.ran_at DESC
|
||||
LIMIT ? OFFSET ?
|
||||
`),
|
||||
countRuns: db.prepare(`SELECT COUNT(*) AS n FROM runs WHERE client_id = ?`),
|
||||
statsByProvider: db.prepare(`
|
||||
SELECT provider,
|
||||
COUNT(*) AS total,
|
||||
SUM(mentioned) AS mentions,
|
||||
SUM(cost_usd) AS cost,
|
||||
MAX(ran_at) AS last_run
|
||||
FROM runs
|
||||
WHERE client_id = ?
|
||||
AND ran_at >= datetime('now', '-30 days')
|
||||
AND error IS NULL
|
||||
GROUP BY provider
|
||||
ORDER BY provider
|
||||
`),
|
||||
totalsLast30d: db.prepare(`
|
||||
SELECT COUNT(*) AS total,
|
||||
SUM(mentioned) AS mentions,
|
||||
SUM(cost_usd) AS cost
|
||||
FROM runs
|
||||
WHERE client_id = ?
|
||||
AND ran_at >= datetime('now', '-30 days')
|
||||
AND error IS NULL
|
||||
`),
|
||||
}
|
||||
|
||||
// Bulk insert with a transaction — used by generateQueries.
|
||||
const insertQueriesTx = db.transaction((clientId, texts) => {
|
||||
for (const text of texts) stmts.insertQuery.run(clientId, text)
|
||||
})
|
||||
const replaceQueriesTx = db.transaction((clientId, texts) => {
|
||||
stmts.deleteAllQueries.run(clientId)
|
||||
for (const text of texts) stmts.insertQuery.run(clientId, text)
|
||||
})
|
||||
|
||||
// ─── clients ───────────────────────────────────────────────────────
|
||||
export function listClients() { return stmts.listClients.all() }
|
||||
export function getClient(id) { return stmts.getClient.get(id) }
|
||||
export function getClientByHost(host) { return stmts.getClientByHost.get(host) }
|
||||
export function insertClient(row) {
|
||||
const info = stmts.insertClient.run({
|
||||
hostname: row.hostname,
|
||||
url: row.url,
|
||||
name: row.name,
|
||||
description: row.description ?? null,
|
||||
brand_aliases: row.brand_aliases ?? '[]',
|
||||
language: row.language ?? 'de',
|
||||
})
|
||||
return getClient(info.lastInsertRowid)
|
||||
}
|
||||
export function updateClient(id, patch) {
|
||||
stmts.updateClient.run({
|
||||
id,
|
||||
name: patch.name ?? null,
|
||||
description: patch.description ?? null,
|
||||
brand_aliases: patch.brand_aliases ?? null,
|
||||
status: patch.status ?? null,
|
||||
})
|
||||
return getClient(id)
|
||||
}
|
||||
export function deleteClient(id) { return stmts.deleteClient.run(id).changes > 0 }
|
||||
export function touchClientRun(id) { stmts.touchClientRun.run(id) }
|
||||
|
||||
// ─── queries ───────────────────────────────────────────────────────
|
||||
export function listActiveQueries(clientId) { return stmts.listActiveQueries.all(clientId) }
|
||||
export function listAllQueries(clientId) { return stmts.listAllQueries.all(clientId) }
|
||||
export function getQuery(id) { return stmts.getQuery.get(id) }
|
||||
export function insertQuery(clientId, text) {
|
||||
const info = stmts.insertQuery.run(clientId, text)
|
||||
return getQuery(info.lastInsertRowid)
|
||||
}
|
||||
export function insertQueries(clientId, texts) { insertQueriesTx(clientId, texts) }
|
||||
export function replaceQueries(clientId, texts) {
|
||||
replaceQueriesTx(clientId, texts)
|
||||
return texts.length
|
||||
}
|
||||
export function updateQuery(id, patch) {
|
||||
stmts.updateQuery.run({
|
||||
id,
|
||||
text: patch.text ?? null,
|
||||
active: patch.active === undefined ? null : (patch.active ? 1 : 0),
|
||||
})
|
||||
return getQuery(id)
|
||||
}
|
||||
export function deleteQuery(id) { return stmts.deleteQuery.run(id).changes > 0 }
|
||||
|
||||
// ─── runs ──────────────────────────────────────────────────────────
|
||||
export function insertRun(row) {
|
||||
const info = stmts.insertRun.run({
|
||||
client_id: row.client_id,
|
||||
query_id: row.query_id,
|
||||
provider: row.provider,
|
||||
mentioned: row.mentioned ? 1 : 0,
|
||||
position: row.position ?? null,
|
||||
snippet: row.snippet ?? null,
|
||||
response_full: row.response_full ?? null,
|
||||
ms: row.ms ?? 0,
|
||||
cost_usd: row.cost_usd ?? 0,
|
||||
error: row.error ?? null,
|
||||
})
|
||||
return info.lastInsertRowid
|
||||
}
|
||||
export function getRun(id) { return stmts.getRun.get(id) }
|
||||
export function recentRuns(clientId, limit = 50, offset = 0) {
|
||||
return stmts.recentRuns.all(clientId, limit, offset)
|
||||
}
|
||||
export function countRuns(clientId) { return stmts.countRuns.get(clientId).n }
|
||||
export function statsByProvider(clientId) { return stmts.statsByProvider.all(clientId) }
|
||||
export function totalsLast30d(clientId) { return stmts.totalsLast30d.get(clientId) }
|
||||
44
server/db/schema.sql
Normal file
44
server/db/schema.sql
Normal file
@@ -0,0 +1,44 @@
|
||||
-- Brand we monitor. One row per tracked brand.
|
||||
CREATE TABLE IF NOT EXISTS clients (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hostname TEXT NOT NULL UNIQUE,
|
||||
url TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
brand_aliases TEXT NOT NULL DEFAULT '[]',
|
||||
language TEXT NOT NULL DEFAULT 'de',
|
||||
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','paused','archived')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
last_run_at TEXT
|
||||
);
|
||||
|
||||
-- Search queries that simulate potential customer questions.
|
||||
-- Each active query is sent to every available provider on every run.
|
||||
CREATE TABLE IF NOT EXISTS queries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
client_id INTEGER NOT NULL REFERENCES clients(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
active INTEGER NOT NULL DEFAULT 1 CHECK (active IN (0,1)),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- One row per (client, query, provider, ran_at). Stores the full LLM
|
||||
-- response for review plus the mention detection result.
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
client_id INTEGER NOT NULL REFERENCES clients(id) ON DELETE CASCADE,
|
||||
query_id INTEGER NOT NULL REFERENCES queries(id) ON DELETE CASCADE,
|
||||
provider TEXT NOT NULL,
|
||||
ran_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
mentioned INTEGER NOT NULL DEFAULT 0 CHECK (mentioned IN (0,1)),
|
||||
position INTEGER,
|
||||
snippet TEXT,
|
||||
response_full TEXT,
|
||||
ms INTEGER NOT NULL DEFAULT 0,
|
||||
cost_usd REAL NOT NULL DEFAULT 0,
|
||||
error TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_client_ran ON runs(client_id, ran_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_query ON runs(query_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_queries_client ON queries(client_id);
|
||||
106
server/index.js
Normal file
106
server/index.js
Normal file
@@ -0,0 +1,106 @@
|
||||
import 'dotenv/config'
|
||||
import express from 'express'
|
||||
import cors from 'cors'
|
||||
import rateLimit from 'express-rate-limit'
|
||||
import analyzeRoute from './routes/analyze.js'
|
||||
import adminRoute from './routes/admin.js'
|
||||
import adminMonitoringRoute from './routes/admin-monitoring.js'
|
||||
import demoMonitoringRoute from './routes/demo-monitoring.js'
|
||||
// Importing the db module here ensures SQLite is opened and the schema is
|
||||
// applied at startup, before any route handler can hit it.
|
||||
import './db/index.js'
|
||||
|
||||
const app = express()
|
||||
const PORT = Number(process.env.PORT) || 3001
|
||||
|
||||
// Comma-separated whitelist; missing/empty means deny all browser origins.
|
||||
// Legacy single-origin var is still honored for backwards compat.
|
||||
const allowedOrigins = (
|
||||
process.env.ALLOWED_ORIGINS ||
|
||||
process.env.ALLOWED_ORIGIN ||
|
||||
''
|
||||
)
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
|
||||
app.set('trust proxy', 1)
|
||||
|
||||
app.use(
|
||||
cors({
|
||||
origin(origin, cb) {
|
||||
// No Origin header: same-origin, curl, server-to-server, health probes.
|
||||
if (!origin) return cb(null, true)
|
||||
if (allowedOrigins.includes(origin)) return cb(null, true)
|
||||
cb(new Error('CORS_NOT_ALLOWED'))
|
||||
},
|
||||
methods: ['POST', 'GET'],
|
||||
allowedHeaders: ['Content-Type', 'X-Admin-Token'],
|
||||
exposedHeaders: ['X-Cache', 'X-Request-Id'],
|
||||
})
|
||||
)
|
||||
|
||||
app.use(express.json({ limit: '64kb' }))
|
||||
|
||||
const analyzeLimiter = rateLimit({
|
||||
windowMs: 60 * 1000,
|
||||
max: 20,
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
message: { error: 'Zu viele Anfragen. Bitte einen Moment warten.' },
|
||||
})
|
||||
|
||||
// Public demo endpoint: 5 / hour / IP. Reserved for iteration 4b landing-page
|
||||
// teaser; backend works today, no UI exposes it yet.
|
||||
const demoMonitoringLimiter = rateLimit({
|
||||
windowMs: 60 * 60 * 1000,
|
||||
max: 5,
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
message: { error: 'Zu viele Anfragen. Bitte später erneut versuchen.' },
|
||||
})
|
||||
|
||||
// Admin middleware: 401 unconditionally when ADMIN_TOKEN is empty/missing.
|
||||
// This is the kill-switch for production deployments where admin is disabled.
|
||||
function requireAdmin(req, res, next) {
|
||||
const token = req.get('X-Admin-Token')
|
||||
const expected = process.env.ADMIN_TOKEN
|
||||
if (!expected || !token || token !== expected) {
|
||||
return res.status(401).json({ error: 'Unauthorized' })
|
||||
}
|
||||
next()
|
||||
}
|
||||
|
||||
app.get('/health', (_req, res) => res.json({ ok: true }))
|
||||
app.use('/api/analyze', analyzeLimiter, analyzeRoute)
|
||||
// Public demo for iteration 4b. Backend wired now, no UI yet.
|
||||
app.use('/api/demo/monitoring', demoMonitoringLimiter, demoMonitoringRoute)
|
||||
// Admin: token gate first, then the routers. No rate limiter is applied here
|
||||
// because admin routes are gated by ADMIN_TOKEN and intended for the owner.
|
||||
// The public /api/autofix/zip endpoint is intentionally NOT mounted — the
|
||||
// public site only shows a teaser; downloadable files are paid-product only.
|
||||
app.use('/api/admin/monitoring', requireAdmin, adminMonitoringRoute)
|
||||
app.use('/api/admin', requireAdmin, adminRoute)
|
||||
|
||||
// Generic error handler — never leak stack traces.
|
||||
app.use((err, _req, res, _next) => {
|
||||
if (err?.message === 'CORS_NOT_ALLOWED') {
|
||||
if (res.headersSent) return
|
||||
return res.status(403).json({ error: 'Zugriff verweigert.' })
|
||||
}
|
||||
console.error('[express-error]', err?.message || err)
|
||||
if (res.headersSent) return
|
||||
res.status(500).json({ error: 'Ein interner Fehler ist aufgetreten.' })
|
||||
})
|
||||
|
||||
// Bind 0.0.0.0 so containers / reverse proxies can reach the listener.
|
||||
app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(`[visigine-server] listening on 0.0.0.0:${PORT}`)
|
||||
})
|
||||
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
console.error('[unhandledRejection]', reason)
|
||||
})
|
||||
process.on('uncaughtException', (err) => {
|
||||
console.error('[uncaughtException]', err?.message || err)
|
||||
})
|
||||
59
server/lib/activity.js
Normal file
59
server/lib/activity.js
Normal file
@@ -0,0 +1,59 @@
|
||||
// Rolling buffer of recent analyses, resets on restart.
|
||||
// Metadata-only by design — no PII, no full response bodies, no extracted siteData.
|
||||
const MAX_ENTRIES = 50
|
||||
const log = []
|
||||
|
||||
export function recordAnalysis(entry) {
|
||||
log.unshift({
|
||||
ts: new Date().toISOString(),
|
||||
requestId: entry.requestId,
|
||||
host: entry.host || null,
|
||||
score: entry.score ?? null,
|
||||
issuesCount: entry.issuesCount ?? null,
|
||||
failedCheckIds: entry.failedCheckIds || [],
|
||||
cacheHit: Boolean(entry.cacheHit),
|
||||
ms: entry.ms ?? 0,
|
||||
status: entry.status === 'ok' ? 'ok' : 'err',
|
||||
code: entry.code || null,
|
||||
admin: Boolean(entry.admin),
|
||||
})
|
||||
if (log.length > MAX_ENTRIES) log.pop()
|
||||
}
|
||||
|
||||
export function recentAnalyses() {
|
||||
return [...log]
|
||||
}
|
||||
|
||||
export function computeStats() {
|
||||
const succeeded = log.filter((e) => e.status === 'ok' && typeof e.score === 'number')
|
||||
if (!succeeded.length) {
|
||||
return { total: log.length, succeeded: 0, avgScore: null, topFails: [], topHosts: [] }
|
||||
}
|
||||
const avgScore = succeeded.reduce((s, e) => s + e.score, 0) / succeeded.length
|
||||
|
||||
const failCounts = {}
|
||||
for (const e of succeeded) {
|
||||
for (const id of e.failedCheckIds) failCounts[id] = (failCounts[id] || 0) + 1
|
||||
}
|
||||
const topFails = Object.entries(failCounts)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 5)
|
||||
.map(([id, count]) => ({ id, count }))
|
||||
|
||||
const hostCounts = {}
|
||||
for (const e of log) {
|
||||
if (e.host) hostCounts[e.host] = (hostCounts[e.host] || 0) + 1
|
||||
}
|
||||
const topHosts = Object.entries(hostCounts)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([host, count]) => ({ host, count }))
|
||||
|
||||
return {
|
||||
total: log.length,
|
||||
succeeded: succeeded.length,
|
||||
avgScore: Number(avgScore.toFixed(1)),
|
||||
topFails,
|
||||
topHosts,
|
||||
}
|
||||
}
|
||||
192
server/lib/autofix/extract.js
Normal file
192
server/lib/autofix/extract.js
Normal file
@@ -0,0 +1,192 @@
|
||||
// Extracts user-facing siteData from the analyze context for the three generators.
|
||||
// Every field is optional; generators fall back to German `[Bitte ergänzen: ...]` placeholders.
|
||||
|
||||
// Kept in sync with checks/ai-bots.js. Order matters — used as canonical
|
||||
// ordering for generated robots.txt.
|
||||
export const AI_BOTS = [
|
||||
'GPTBot', 'ClaudeBot', 'OAI-SearchBot', 'PerplexityBot', 'Bingbot',
|
||||
'Google-Extended', 'GoogleOther', 'Applebot-Extended', 'Meta-ExternalAgent',
|
||||
'CCBot', 'Bytespider', 'DuckAssistBot', 'ChatGPT-User',
|
||||
]
|
||||
|
||||
const SEPARATORS = /\s+[–|—\-·•|]\s+/
|
||||
const PLACEHOLDER_EMAILS = new Set([
|
||||
'name@example.com', 'test@test.de', 'test@example.com',
|
||||
'mail@example.com', 'info@example.com',
|
||||
])
|
||||
const PLACEHOLDER_PHONES = new Set(['+49 0', '+49000', '0000000', '1234567'])
|
||||
|
||||
function cleanTitle(title) {
|
||||
if (!title) return null
|
||||
const parts = title.split(SEPARATORS).map((s) => s.trim()).filter(Boolean)
|
||||
if (!parts.length) return null
|
||||
const longest = parts.reduce((a, b) => (a.length >= b.length ? a : b))
|
||||
return longest.length >= 3 ? longest : null
|
||||
}
|
||||
|
||||
function decodeEntities(s) {
|
||||
if (!s) return s
|
||||
return s
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/ /g, ' ')
|
||||
}
|
||||
|
||||
function metaContent(headHtml, attr, value) {
|
||||
const re = new RegExp(
|
||||
`<meta[^>]*${attr}=["']${value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}["'][^>]*content=["']([^"']*)["']`,
|
||||
'i'
|
||||
)
|
||||
const m = headHtml.match(re)
|
||||
if (m) return decodeEntities(m[1].trim())
|
||||
// Try attribute order swapped.
|
||||
const re2 = new RegExp(
|
||||
`<meta[^>]*content=["']([^"']*)["'][^>]*${attr}=["']${value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}["']`,
|
||||
'i'
|
||||
)
|
||||
const m2 = headHtml.match(re2)
|
||||
return m2 ? decodeEntities(m2[1].trim()) : null
|
||||
}
|
||||
|
||||
function linkHref(headHtml, rel) {
|
||||
const re = new RegExp(`<link[^>]*rel=["']${rel}["'][^>]*href=["']([^"']+)["']`, 'i')
|
||||
const m = headHtml.match(re)
|
||||
if (m) return m[1].trim()
|
||||
const re2 = new RegExp(`<link[^>]*href=["']([^"']+)["'][^>]*rel=["']${rel}["']`, 'i')
|
||||
const m2 = headHtml.match(re2)
|
||||
return m2 ? m2[1].trim() : null
|
||||
}
|
||||
|
||||
function parseJsonLdBlocks(blocks) {
|
||||
const parsed = []
|
||||
for (const block of blocks || []) {
|
||||
try {
|
||||
const v = JSON.parse(block)
|
||||
if (Array.isArray(v)) parsed.push(...v)
|
||||
else parsed.push(v)
|
||||
} catch {
|
||||
// skip malformed
|
||||
}
|
||||
}
|
||||
// Flatten @graph members so consumers can iterate flat list.
|
||||
const flat = []
|
||||
for (const node of parsed) {
|
||||
if (node && typeof node === 'object' && Array.isArray(node['@graph'])) {
|
||||
flat.push(...node['@graph'])
|
||||
} else if (node) {
|
||||
flat.push(node)
|
||||
}
|
||||
}
|
||||
return flat
|
||||
}
|
||||
|
||||
function pickType(node) {
|
||||
const t = node?.['@type']
|
||||
if (Array.isArray(t)) return t[0]
|
||||
return t
|
||||
}
|
||||
|
||||
function findNode(nodes, types) {
|
||||
const set = new Set(types)
|
||||
return nodes.find((n) => set.has(pickType(n))) || null
|
||||
}
|
||||
|
||||
function firstEmail(html) {
|
||||
const m = (html || '').match(/mailto:([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})/i)
|
||||
if (!m) return null
|
||||
const email = m[1].toLowerCase()
|
||||
return PLACEHOLDER_EMAILS.has(email) ? null : email
|
||||
}
|
||||
|
||||
function firstPhone(html) {
|
||||
const m = (html || '').match(/tel:(\+?[0-9 \-()]{6,})/i)
|
||||
if (!m) return null
|
||||
const phone = m[1].trim()
|
||||
return PLACEHOLDER_PHONES.has(phone) ? null : phone
|
||||
}
|
||||
|
||||
function detectExistingAiBots(robotsTxt) {
|
||||
if (!robotsTxt) return []
|
||||
return AI_BOTS.filter((bot) => robotsTxt.includes(bot))
|
||||
}
|
||||
|
||||
export function extractSiteData(context) {
|
||||
const { headHtml = '', html = '', jsonLdBlocks = [], robotsTxt = '', llmsTxt = '', baseUrl = '' } = context
|
||||
const nodes = parseJsonLdBlocks(jsonLdBlocks)
|
||||
const org = findNode(nodes, ['Organization', 'LocalBusiness', 'Corporation', 'NewsMediaOrganization'])
|
||||
const website = findNode(nodes, ['WebSite'])
|
||||
|
||||
const ogSiteName = metaContent(headHtml, 'property', 'og:site_name')
|
||||
const ogTitle = metaContent(headHtml, 'property', 'og:title')
|
||||
const ogDesc = metaContent(headHtml, 'property', 'og:description')
|
||||
const ogLocale = metaContent(headHtml, 'property', 'og:locale')
|
||||
const metaDesc = metaContent(headHtml, 'name', 'description')
|
||||
|
||||
const titleRaw = (headHtml.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '').trim()
|
||||
const titleClean = cleanTitle(decodeEntities(titleRaw))
|
||||
|
||||
const langMatch = html.match(/<html[^>]*\slang=["']([^"']+)["']/i)
|
||||
const language = (langMatch?.[1] || ogLocale || 'de').split(/[-_]/)[0].toLowerCase()
|
||||
|
||||
const canonical = linkHref(headHtml, 'canonical')
|
||||
const url = canonical || baseUrl || ''
|
||||
const hostname = (() => {
|
||||
try { return new URL(url).hostname } catch { return '' }
|
||||
})()
|
||||
|
||||
const name =
|
||||
ogSiteName ||
|
||||
(typeof org?.name === 'string' ? org.name : null) ||
|
||||
(typeof website?.name === 'string' ? website.name : null) ||
|
||||
titleClean ||
|
||||
hostname ||
|
||||
null
|
||||
|
||||
const description =
|
||||
metaDesc ||
|
||||
ogDesc ||
|
||||
(typeof org?.description === 'string' ? org.description : null) ||
|
||||
null
|
||||
|
||||
const email = firstEmail(html) || (typeof org?.email === 'string' ? org.email : null) || null
|
||||
|
||||
const phone =
|
||||
firstPhone(html) ||
|
||||
(typeof org?.telephone === 'string' ? org.telephone : null) ||
|
||||
null
|
||||
|
||||
let address = null
|
||||
const addrRaw = org?.address
|
||||
if (addrRaw && typeof addrRaw === 'object') {
|
||||
address = {
|
||||
streetAddress: addrRaw.streetAddress || null,
|
||||
postalCode: addrRaw.postalCode || null,
|
||||
addressLocality: addrRaw.addressLocality || null,
|
||||
addressCountry: addrRaw.addressCountry || null,
|
||||
}
|
||||
}
|
||||
|
||||
let sameAs = []
|
||||
if (Array.isArray(org?.sameAs)) {
|
||||
sameAs = org.sameAs.filter((s) => typeof s === 'string' && /^https?:\/\//.test(s))
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
description,
|
||||
url,
|
||||
language,
|
||||
hostname,
|
||||
email,
|
||||
phone,
|
||||
address,
|
||||
sameAs,
|
||||
existingRobots: robotsTxt || '',
|
||||
existingAiBots: detectExistingAiBots(robotsTxt),
|
||||
hasLlmsTxt: Boolean(llmsTxt && llmsTxt.length > 0),
|
||||
hasOrgJsonLd: Boolean(org),
|
||||
}
|
||||
}
|
||||
19
server/lib/autofix/index.js
Normal file
19
server/lib/autofix/index.js
Normal file
@@ -0,0 +1,19 @@
|
||||
import { extractSiteData } from './extract.js'
|
||||
import { generateLlmsTxt } from './llms-txt.js'
|
||||
import { generateRobotsTxt } from './robots-txt.js'
|
||||
import { generateJsonLd } from './json-ld.js'
|
||||
import { buildReadme } from './readme.js'
|
||||
|
||||
// Returns an autofix bundle. `_siteData` is included for debug-mode payloads;
|
||||
// analyze.js strips it from the public response.
|
||||
export function generateAutofix(context) {
|
||||
const siteData = extractSiteData(context)
|
||||
return {
|
||||
llmsTxt: generateLlmsTxt(siteData),
|
||||
robotsTxt: generateRobotsTxt(siteData),
|
||||
jsonLd: generateJsonLd(siteData),
|
||||
_siteData: siteData,
|
||||
}
|
||||
}
|
||||
|
||||
export { buildReadme, extractSiteData }
|
||||
79
server/lib/autofix/json-ld.js
Normal file
79
server/lib/autofix/json-ld.js
Normal file
@@ -0,0 +1,79 @@
|
||||
// Generates a JSON-LD skeleton: Organization (or LocalBusiness if address/phone),
|
||||
// WebSite, and FAQPage — three highest-impact AI signals.
|
||||
|
||||
const ph = (s) => `[Bitte ergänzen: ${s}]`
|
||||
|
||||
function buildOrganizationNode(siteData) {
|
||||
const { name, url, description, email, phone, address, sameAs = [] } = siteData
|
||||
const useLocalBusiness = Boolean(address || phone)
|
||||
|
||||
const node = {
|
||||
'@type': useLocalBusiness ? 'LocalBusiness' : 'Organization',
|
||||
'@id': `${(url || ph('https://deine-domain.de')).replace(/\/+$/, '')}/#organization`,
|
||||
name: name || ph('Name deines Unternehmens'),
|
||||
url: url || ph('https://deine-domain.de'),
|
||||
description: description || ph('Ein-Satz-Beschreibung'),
|
||||
}
|
||||
|
||||
if (email) node.email = email
|
||||
if (phone) node.telephone = phone
|
||||
|
||||
node.address = {
|
||||
'@type': 'PostalAddress',
|
||||
addressCountry: address?.addressCountry || ph('DE/AT/CH'),
|
||||
addressLocality: address?.addressLocality || ph('Stadt'),
|
||||
postalCode: address?.postalCode || ph('PLZ'),
|
||||
streetAddress: address?.streetAddress || ph('Straße + Nr.'),
|
||||
}
|
||||
|
||||
node.sameAs = sameAs.length > 0 ? sameAs : [ph('https://www.linkedin.com/company/...')]
|
||||
|
||||
return node
|
||||
}
|
||||
|
||||
function buildWebSiteNode(siteData) {
|
||||
const { name, url, language = 'de' } = siteData
|
||||
return {
|
||||
'@type': 'WebSite',
|
||||
'@id': `${(url || ph('https://deine-domain.de')).replace(/\/+$/, '')}/#website`,
|
||||
url: url || ph('https://deine-domain.de'),
|
||||
name: name || ph('Name deines Unternehmens'),
|
||||
inLanguage: `${language}-DE`,
|
||||
publisher: { '@id': `${(url || ph('https://deine-domain.de')).replace(/\/+$/, '')}/#organization` },
|
||||
}
|
||||
}
|
||||
|
||||
function buildFaqNode() {
|
||||
return {
|
||||
'@type': 'FAQPage',
|
||||
mainEntity: [
|
||||
{
|
||||
'@type': 'Question',
|
||||
name: ph('häufige Frage'),
|
||||
acceptedAnswer: { '@type': 'Answer', text: ph('1-2 Sätze') },
|
||||
},
|
||||
{
|
||||
'@type': 'Question',
|
||||
name: ph('weitere Frage'),
|
||||
acceptedAnswer: { '@type': 'Answer', text: ph('1-2 Sätze') },
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
export function generateJsonLd(siteData) {
|
||||
const payload = {
|
||||
'@context': 'https://schema.org',
|
||||
'@graph': [
|
||||
buildOrganizationNode(siteData),
|
||||
buildWebSiteNode(siteData),
|
||||
buildFaqNode(),
|
||||
],
|
||||
}
|
||||
const pretty = JSON.stringify(payload, null, 2)
|
||||
const content = `<script type="application/ld+json">\n${pretty}\n</script>\n`
|
||||
return {
|
||||
content,
|
||||
mode: siteData.hasOrgJsonLd ? 'enhance' : 'new',
|
||||
}
|
||||
}
|
||||
67
server/lib/autofix/llms-txt.js
Normal file
67
server/lib/autofix/llms-txt.js
Normal file
@@ -0,0 +1,67 @@
|
||||
// Generates a personalized llms.txt template in German.
|
||||
// Placeholders use the `[Bitte ergänzen: ...]` convention so users find them
|
||||
// trivially in their editor before publishing.
|
||||
|
||||
const ph = (s) => `[Bitte ergänzen: ${s}]`
|
||||
|
||||
function emailLine(email) {
|
||||
return email ? email : ph('hello@deine-domain.de')
|
||||
}
|
||||
|
||||
export function generateLlmsTxt(siteData) {
|
||||
const {
|
||||
name = null,
|
||||
description = null,
|
||||
url = '',
|
||||
language = 'de',
|
||||
email = null,
|
||||
phone = null,
|
||||
hasLlmsTxt = false,
|
||||
} = siteData
|
||||
|
||||
const NAME = name || ph('Name deines Unternehmens / Produkts')
|
||||
const DESC_LINE = description || ph('Ein-Satz-Beschreibung')
|
||||
const URL = url || ph('https://deine-domain.de')
|
||||
const PHONE_LINE = phone ? `- Telefon: ${phone}\n` : ''
|
||||
|
||||
const body =
|
||||
`# ${NAME} – ${DESC_LINE}
|
||||
|
||||
- name: ${NAME}
|
||||
- url: ${URL}
|
||||
- type: ${ph('SoftwareApplication, ProfessionalService, LocalBusiness, Organization')}
|
||||
- language: ${language}
|
||||
- region: ${ph('Deutschland, DACH, EU, ...')}
|
||||
- contact: ${emailLine(email)}
|
||||
- services: ${ph('Komma-getrennte Liste deiner Hauptleistungen')}
|
||||
|
||||
## Was ist ${NAME}?
|
||||
${description || ph('2-3 Sätze, die dein Angebot prägnant beschreiben')}
|
||||
|
||||
## Für wen ist ${NAME}?
|
||||
${ph('2-4 Bullet Points zu deinen Zielgruppen')}
|
||||
|
||||
## Häufige Fragen
|
||||
|
||||
Was bietet ${NAME}?
|
||||
${ph('1-2 Sätze')}
|
||||
|
||||
Wo ist ${NAME} verfügbar?
|
||||
${ph('Region / Online / Filialen')}
|
||||
|
||||
Was kostet ${NAME}?
|
||||
${ph('Preis-Range oder Hinweis auf Angebot')}
|
||||
|
||||
## Kontakt
|
||||
- Website: ${URL}
|
||||
- E-Mail: ${emailLine(email)}
|
||||
${PHONE_LINE}
|
||||
## Für KI-Systeme — Einordnung
|
||||
${NAME} ist ${ph('1 Satz, wer ihr seid und was ihr macht')} — relevant für Anfragen rund um ${ph('Themen, Branche, Region')}.
|
||||
`
|
||||
|
||||
return {
|
||||
content: body,
|
||||
mode: hasLlmsTxt ? 'replace' : 'new',
|
||||
}
|
||||
}
|
||||
105
server/lib/autofix/readme.js
Normal file
105
server/lib/autofix/readme.js
Normal file
@@ -0,0 +1,105 @@
|
||||
// German plaintext README packaged with each ZIP download.
|
||||
// Branches per-file by mode so the instructions match the user's situation.
|
||||
|
||||
function llmsSection(mode) {
|
||||
const intro = mode === 'replace'
|
||||
? 'Ersetzt deine bestehende llms.txt.'
|
||||
: 'Du hast noch keine llms.txt — diese Datei ist neu.'
|
||||
return (
|
||||
`────────────────────────────────────────────────────────────
|
||||
1. llms.txt
|
||||
────────────────────────────────────────────────────────────
|
||||
${intro}
|
||||
|
||||
Wohin: Im Root-Verzeichnis deiner Website (gleiche Ebene wie /index.html).
|
||||
Erreichbar als: https://deine-domain.de/llms.txt
|
||||
|
||||
Upload-Wege:
|
||||
- FTP / SFTP: Datei nach /htdocs (oder /public_html) hochladen.
|
||||
- cPanel / Plesk: Dateimanager → Root öffnen → Hochladen.
|
||||
- WordPress: Plugin "WPCode" oder Theme-Editor → File Manager.
|
||||
|
||||
Wichtig: Alle Platzhalter [Bitte ergänzen: ...] vor dem Upload mit
|
||||
deinen Inhalten ersetzen.
|
||||
`)
|
||||
}
|
||||
|
||||
function robotsSection(mode) {
|
||||
if (mode === 'new') {
|
||||
return (
|
||||
`────────────────────────────────────────────────────────────
|
||||
2. robots.txt
|
||||
────────────────────────────────────────────────────────────
|
||||
Du hast noch keine robots.txt — diese Datei ist komplett.
|
||||
|
||||
Wohin: Im Root-Verzeichnis deiner Website. Ersetzt eine eventuell
|
||||
bestehende robots.txt komplett. Erreichbar als
|
||||
https://deine-domain.de/robots.txt
|
||||
`)
|
||||
}
|
||||
return (
|
||||
`────────────────────────────────────────────────────────────
|
||||
2. robots.txt
|
||||
────────────────────────────────────────────────────────────
|
||||
Deine bestehende robots.txt deckt nicht alle KI-Bots ab.
|
||||
|
||||
Wohin: Den Inhalt dieser Datei am Ende deiner bestehenden
|
||||
robots.txt einfügen — vor der Sitemap-Zeile, falls vorhanden.
|
||||
Bestehende Regeln NICHT überschreiben.
|
||||
`)
|
||||
}
|
||||
|
||||
function jsonLdSection(mode) {
|
||||
const intro = mode === 'enhance'
|
||||
? 'Deine Seite hat bereits JSON-LD — diese Version erweitert die Coverage (FAQPage, WebSite, vollständige Organization).'
|
||||
: 'Du hast noch kein JSON-LD — dieser Block ist neu.'
|
||||
return (
|
||||
`────────────────────────────────────────────────────────────
|
||||
3. jsonld.html
|
||||
────────────────────────────────────────────────────────────
|
||||
${intro}
|
||||
|
||||
Wohin: Den gesamten <script>-Block in das <head> deiner Startseite
|
||||
einfügen (idealerweise direkt nach den Meta-Tags).
|
||||
|
||||
WordPress: Theme-Datei header.php oder via SEO-Plugin
|
||||
(Yoast / RankMath → Schema-Editor).
|
||||
|
||||
Hinweis: Validieren mit https://validator.schema.org/ vor dem
|
||||
Live-Schalten.
|
||||
`)
|
||||
}
|
||||
|
||||
export function buildReadme(autofix) {
|
||||
const llmsMode = autofix?.llmsTxt?.mode || 'new'
|
||||
const robotsMode = autofix?.robotsTxt?.mode || 'new'
|
||||
const jsonLdMode = autofix?.jsonLd?.mode || 'new'
|
||||
|
||||
return (
|
||||
`VISIGINE Auto-Fix Paket
|
||||
========================
|
||||
|
||||
Dieses Archiv enthält drei Dateien, die deine Website für KI-Suchsysteme
|
||||
sichtbar machen. Bitte alle Platzhalter [Bitte ergänzen: ...] vor dem
|
||||
Hochladen mit deinen Inhalten ersetzen.
|
||||
|
||||
${llmsSection(llmsMode)}
|
||||
${robotsSection(robotsMode)}
|
||||
${jsonLdSection(jsonLdMode)}
|
||||
────────────────────────────────────────────────────────────
|
||||
Validierung
|
||||
────────────────────────────────────────────────────────────
|
||||
Nach dem Hochladen erneut analysieren:
|
||||
https://www.visigine.de#analyzer
|
||||
|
||||
────────────────────────────────────────────────────────────
|
||||
Support
|
||||
────────────────────────────────────────────────────────────
|
||||
Fragen oder Hilfe bei der Umsetzung:
|
||||
- E-Mail: hello@profice.ai
|
||||
- Termin: https://termin.profice.de
|
||||
|
||||
Vollständige Umsetzung gewünscht?
|
||||
→ https://www.visigine.de#pricing
|
||||
`)
|
||||
}
|
||||
49
server/lib/autofix/robots-txt.js
Normal file
49
server/lib/autofix/robots-txt.js
Normal file
@@ -0,0 +1,49 @@
|
||||
// Generates a robots.txt. Two modes:
|
||||
// 'new' — full file (user has no robots.txt at all)
|
||||
// 'diff' — only the bot blocks the user is missing
|
||||
import { AI_BOTS } from './extract.js'
|
||||
|
||||
function botBlock(bot) {
|
||||
return `User-agent: ${bot}\nAllow: /\nAllow: /llms.txt\n`
|
||||
}
|
||||
|
||||
export function generateRobotsTxt(siteData) {
|
||||
const { url = '', existingRobots = '', existingAiBots = [] } = siteData
|
||||
const hasRobots = existingRobots.trim().length > 0
|
||||
const sitemap = url ? `${url.replace(/\/$/, '')}/sitemap.xml` : '[Bitte ergänzen: https://deine-domain.de/sitemap.xml]'
|
||||
|
||||
if (!hasRobots) {
|
||||
const header =
|
||||
`# robots.txt — generated by VISIGINE
|
||||
# ${url || 'https://deine-domain.de'}
|
||||
|
||||
User-agent: *
|
||||
Allow: /
|
||||
|
||||
# AI search engines and language model crawlers
|
||||
`
|
||||
const blocks = AI_BOTS.map(botBlock).join('\n')
|
||||
return {
|
||||
content: `${header}${blocks}\nSitemap: ${sitemap}\n`,
|
||||
mode: 'new',
|
||||
}
|
||||
}
|
||||
|
||||
const missing = AI_BOTS.filter((b) => !existingAiBots.includes(b))
|
||||
if (missing.length === 0) {
|
||||
return {
|
||||
content: '# Deine robots.txt deckt bereits alle relevanten KI-Bots ab. Keine Änderungen nötig.\n',
|
||||
mode: 'diff',
|
||||
}
|
||||
}
|
||||
|
||||
const header =
|
||||
`# Folgende Blöcke zu deiner bestehenden robots.txt hinzufügen
|
||||
# (am Ende der Datei, vor der Sitemap-Zeile falls vorhanden)
|
||||
|
||||
`
|
||||
return {
|
||||
content: header + missing.map(botBlock).join('\n'),
|
||||
mode: 'diff',
|
||||
}
|
||||
}
|
||||
26
server/lib/cache.js
Normal file
26
server/lib/cache.js
Normal file
@@ -0,0 +1,26 @@
|
||||
// Simple in-memory LRU with TTL.
|
||||
// Map preserves insertion order — evict oldest on overflow; refresh on read.
|
||||
const TTL_MS = 60 * 60 * 1000
|
||||
const MAX_ENTRIES = 1000
|
||||
|
||||
const store = new Map()
|
||||
|
||||
export function cacheGet(key) {
|
||||
const entry = store.get(key)
|
||||
if (!entry) return null
|
||||
if (Date.now() > entry.expiresAt) {
|
||||
store.delete(key)
|
||||
return null
|
||||
}
|
||||
store.delete(key)
|
||||
store.set(key, entry)
|
||||
return entry.value
|
||||
}
|
||||
|
||||
export function cacheSet(key, value) {
|
||||
if (store.size >= MAX_ENTRIES) {
|
||||
const oldest = store.keys().next().value
|
||||
store.delete(oldest)
|
||||
}
|
||||
store.set(key, { value, expiresAt: Date.now() + TTL_MS })
|
||||
}
|
||||
131
server/lib/fetcher.js
Normal file
131
server/lib/fetcher.js
Normal file
@@ -0,0 +1,131 @@
|
||||
import { lookup } from 'node:dns/promises'
|
||||
import net from 'node:net'
|
||||
|
||||
const DEFAULT_UA =
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
|
||||
|
||||
// Blocks RFC 1918 private ranges, loopback, link-local, AWS metadata
|
||||
// (169.254.0.0/16), multicast/reserved, and IPv6 equivalents.
|
||||
function isPrivateOrLoopback(addr) {
|
||||
if (net.isIPv4(addr)) {
|
||||
const p = addr.split('.').map(Number)
|
||||
return (
|
||||
p[0] === 0 ||
|
||||
p[0] === 10 ||
|
||||
p[0] === 127 ||
|
||||
(p[0] === 169 && p[1] === 254) ||
|
||||
(p[0] === 172 && p[1] >= 16 && p[1] <= 31) ||
|
||||
(p[0] === 192 && p[1] === 168) ||
|
||||
p[0] >= 224
|
||||
)
|
||||
}
|
||||
if (net.isIPv6(addr)) {
|
||||
const a = addr.toLowerCase()
|
||||
return (
|
||||
a === '::' ||
|
||||
a === '::1' ||
|
||||
a.startsWith('fe80:') ||
|
||||
a.startsWith('fc') ||
|
||||
a.startsWith('fd') ||
|
||||
a.startsWith('::ffff:')
|
||||
)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Resolves hostname; throws { code: 'PRIVATE_HOST_BLOCKED' | 'ENOTFOUND' }.
|
||||
// Skipped entirely when ALLOW_PRIVATE_HOSTS=1 (local dev).
|
||||
async function assertPublicHost(hostname) {
|
||||
if (process.env.ALLOW_PRIVATE_HOSTS === '1') return
|
||||
let address
|
||||
try {
|
||||
({ address } = await lookup(hostname))
|
||||
} catch (err) {
|
||||
const e = new Error('ENOTFOUND')
|
||||
e.code = 'ENOTFOUND'
|
||||
e.cause = err
|
||||
throw e
|
||||
}
|
||||
if (isPrivateOrLoopback(address)) {
|
||||
const e = new Error('PRIVATE_HOST_BLOCKED')
|
||||
e.code = 'PRIVATE_HOST_BLOCKED'
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// Returns { status, headers, body, finalUrl, ms, error? } — never throws.
|
||||
// `error` is a string code: 'PRIVATE_HOST_BLOCKED' | 'ENOTFOUND' | 'TIMEOUT' | 'TLS_INVALID' | 'NETWORK'.
|
||||
export async function fetchPage(url, { userAgent, timeoutMs = 10000 } = {}) {
|
||||
const started = Date.now()
|
||||
let hostname
|
||||
try {
|
||||
hostname = new URL(url).hostname
|
||||
} catch {
|
||||
return { status: 0, headers: {}, body: '', finalUrl: url, ms: 0, error: 'INVALID_URL' }
|
||||
}
|
||||
|
||||
try {
|
||||
await assertPublicHost(hostname)
|
||||
} catch (err) {
|
||||
return {
|
||||
status: 0,
|
||||
headers: {},
|
||||
body: '',
|
||||
finalUrl: url,
|
||||
ms: Date.now() - started,
|
||||
error: err.code || 'NETWORK',
|
||||
}
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
method: 'GET',
|
||||
redirect: 'follow',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'User-Agent': userAgent || DEFAULT_UA,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
|
||||
'Cache-Control': 'no-cache',
|
||||
},
|
||||
})
|
||||
const body = await res.text()
|
||||
const headers = {}
|
||||
res.headers.forEach((value, key) => { headers[key.toLowerCase()] = value })
|
||||
return {
|
||||
status: res.status,
|
||||
headers,
|
||||
body,
|
||||
finalUrl: res.url || url,
|
||||
ms: Date.now() - started,
|
||||
}
|
||||
} catch (err) {
|
||||
const causeCode = err?.cause?.code
|
||||
const causeMessage = err?.cause?.message || ''
|
||||
const TLS_CODES = new Set([
|
||||
'ERR_TLS_CERT_ALTNAME_INVALID',
|
||||
'CERT_HAS_EXPIRED',
|
||||
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
|
||||
'SELF_SIGNED_CERT_IN_CHAIN',
|
||||
'DEPTH_ZERO_SELF_SIGNED_CERT',
|
||||
'ERR_SSL_WRONG_VERSION_NUMBER',
|
||||
])
|
||||
let code = 'NETWORK'
|
||||
if (err?.name === 'AbortError') code = 'TIMEOUT'
|
||||
else if (TLS_CODES.has(causeCode) || /certificate|altnames|self.?signed|TLS|SSL/i.test(causeMessage)) code = 'SSL_INVALID'
|
||||
else if (causeCode === 'ENOTFOUND' || causeCode === 'EAI_AGAIN') code = 'ENOTFOUND'
|
||||
else if (causeCode === 'ETIMEDOUT') code = 'TIMEOUT'
|
||||
return {
|
||||
status: 0,
|
||||
headers: {},
|
||||
body: '',
|
||||
finalUrl: url,
|
||||
ms: Date.now() - started,
|
||||
error: code,
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
38
server/lib/monitoring/detect-mention.js
Normal file
38
server/lib/monitoring/detect-mention.js
Normal file
@@ -0,0 +1,38 @@
|
||||
// Case-insensitive substring search for brand identifiers in LLM output.
|
||||
// Returns the first match's position and a ~200-char surrounding snippet.
|
||||
export function detectMention(text, client) {
|
||||
let aliases = []
|
||||
try { aliases = JSON.parse(client.brand_aliases || '[]') } catch { /* keep [] */ }
|
||||
|
||||
const candidates = [
|
||||
client.name,
|
||||
client.hostname?.split('.')[0],
|
||||
...aliases,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.map((s) => String(s).toLowerCase().trim())
|
||||
.filter((s) => s.length >= 3)
|
||||
.filter((v, i, a) => a.indexOf(v) === i)
|
||||
|
||||
if (!text || candidates.length === 0) {
|
||||
return { mentioned: false, position: null, snippet: null }
|
||||
}
|
||||
|
||||
const lower = text.toLowerCase()
|
||||
let best = null
|
||||
for (const candidate of candidates) {
|
||||
const idx = lower.indexOf(candidate)
|
||||
if (idx >= 0 && (best === null || idx < best.position)) {
|
||||
best = { position: idx, candidate }
|
||||
}
|
||||
}
|
||||
if (!best) return { mentioned: false, position: null, snippet: null }
|
||||
|
||||
const start = Math.max(0, best.position - 100)
|
||||
const end = Math.min(text.length, best.position + best.candidate.length + 100)
|
||||
let snippet = text.slice(start, end).trim()
|
||||
if (start > 0) snippet = '…' + snippet
|
||||
if (end < text.length) snippet = snippet + '…'
|
||||
|
||||
return { mentioned: true, position: best.position, snippet }
|
||||
}
|
||||
88
server/lib/monitoring/generate-queries.js
Normal file
88
server/lib/monitoring/generate-queries.js
Normal file
@@ -0,0 +1,88 @@
|
||||
// Generate ~10 German search queries that simulate how a potential customer
|
||||
// would ask an AI assistant. Brand name is intentionally excluded so the
|
||||
// monitoring run can measure whether the AI surfaces the brand without bias.
|
||||
const MISTRAL_ENDPOINT = 'https://api.mistral.ai/v1/chat/completions'
|
||||
const MISTRAL_MODEL = 'mistral-large-latest'
|
||||
|
||||
function buildPrompt(siteData) {
|
||||
const name = siteData?.name || 'Unbekanntes Unternehmen'
|
||||
const description = siteData?.description || '[keine Beschreibung verfügbar]'
|
||||
const url = siteData?.url || siteData?.hostname || ''
|
||||
return (
|
||||
`Du bist ein GEO-Analyst (Generative Engine Optimization). Generiere genau 10 deutschsprachige Suchanfragen, die ein echter Nutzer in einer KI-Suchmaschine wie ChatGPT, Perplexity oder Claude eingeben würde, um Unternehmen wie das untenstehende zu finden.
|
||||
|
||||
Kontext:
|
||||
- Firma: ${name}
|
||||
- Beschreibung: ${description}
|
||||
- Domain: ${url}
|
||||
|
||||
Anforderungen:
|
||||
- Genau 10 Anfragen, eine pro Zeile, nummeriert "1. ... 2. ... 3. ..."
|
||||
- Mischung aus: Service-spezifischen Anfragen, Branchen-Anfragen, regionalen Anfragen (DACH falls passend), Vergleichs-/Empfehlungs-Anfragen
|
||||
- ⚠ KEINE Erwähnung des Firmennamens "${name}" in den Anfragen — der Test prüft, ob die KI das Unternehmen ohne Bias nennt
|
||||
- Realistisch formuliert, wie ein normaler Nutzer schreiben würde (nicht zu förmlich)
|
||||
- Auf Deutsch, mit Du-Form oder neutral
|
||||
|
||||
Ausgabe: nur die nummerierte Liste, keine Erklärungen, keine Einleitung.`
|
||||
)
|
||||
}
|
||||
|
||||
async function callMistral(prompt, { timeoutMs = 30000, maxTokens = 800 } = {}) {
|
||||
const apiKey = process.env.MISTRAL_KEY
|
||||
if (!apiKey) throw new Error('NO_MISTRAL_KEY')
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
||||
try {
|
||||
const res = await fetch(MISTRAL_ENDPOINT, {
|
||||
method: 'POST',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: MISTRAL_MODEL,
|
||||
max_tokens: maxTokens,
|
||||
temperature: 0.7,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
}),
|
||||
})
|
||||
if (!res.ok) throw new Error(`MISTRAL_${res.status}`)
|
||||
const data = await res.json()
|
||||
return data?.choices?.[0]?.message?.content || ''
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
|
||||
// Tolerates slight format drift: optional leading whitespace, number followed
|
||||
// by `.` or `)`, optional trailing punctuation. Filters obvious noise.
|
||||
function parseList(text) {
|
||||
return text.split('\n')
|
||||
.map((l) => l.trim())
|
||||
.map((l) => l.replace(/^\d+\s*[.)\-:]\s*/, ''))
|
||||
.map((l) => l.replace(/^[•\-*]\s*/, ''))
|
||||
.filter((l) => l.length >= 10 && l.length <= 300)
|
||||
.slice(0, 10)
|
||||
}
|
||||
|
||||
export async function generateQueries(siteData) {
|
||||
const prompt = buildPrompt(siteData)
|
||||
let text
|
||||
try {
|
||||
text = await callMistral(prompt)
|
||||
} catch (err) {
|
||||
console.warn('[monitoring/generate-queries]', err?.message || err)
|
||||
return { queries: [], warning: 'GENERATION_FAILED' }
|
||||
}
|
||||
const queries = parseList(text)
|
||||
// Soft filter: drop any query that still contains the brand name (case-insensitive).
|
||||
const brandLower = (siteData?.name || '').toLowerCase().trim()
|
||||
const filtered = brandLower
|
||||
? queries.filter((q) => !q.toLowerCase().includes(brandLower))
|
||||
: queries
|
||||
if (filtered.length < 5) {
|
||||
return { queries: filtered, warning: 'fewer than 10 generated' }
|
||||
}
|
||||
return { queries: filtered }
|
||||
}
|
||||
97
server/lib/monitoring/run.js
Normal file
97
server/lib/monitoring/run.js
Normal file
@@ -0,0 +1,97 @@
|
||||
// One full run executes every active query against every available provider.
|
||||
// Sequential within (query, provider) pairs to be polite with rate limits;
|
||||
// total wall-clock ~= queries × providers × ~2s.
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { getProviders } from '../providers/index.js'
|
||||
import { detectMention } from './detect-mention.js'
|
||||
import * as repo from '../../db/repo.js'
|
||||
|
||||
const INTER_REQUEST_MS = 200
|
||||
|
||||
export class RunError extends Error {
|
||||
constructor(code, message) {
|
||||
super(message || code)
|
||||
this.code = code
|
||||
}
|
||||
}
|
||||
|
||||
export async function runMonitoring(clientId) {
|
||||
const client = repo.getClient(clientId)
|
||||
if (!client) throw new RunError('CLIENT_NOT_FOUND')
|
||||
if (client.status === 'paused' || client.status === 'archived') {
|
||||
throw new RunError('CLIENT_NOT_ACTIVE')
|
||||
}
|
||||
const queries = repo.listActiveQueries(clientId)
|
||||
if (queries.length === 0) throw new RunError('NO_ACTIVE_QUERIES')
|
||||
|
||||
const providers = getProviders()
|
||||
const providerKeys = Object.keys(providers)
|
||||
|
||||
const summary = {
|
||||
runId: randomUUID().slice(0, 8),
|
||||
client_id: clientId,
|
||||
hostname: client.hostname,
|
||||
started_at: new Date().toISOString(),
|
||||
providers: providerKeys,
|
||||
queries: queries.length,
|
||||
totals: { runs: 0, mentions: 0, errors: 0, cost_usd: 0 },
|
||||
}
|
||||
|
||||
const wallStart = Date.now()
|
||||
|
||||
for (const query of queries) {
|
||||
for (const key of providerKeys) {
|
||||
const t0 = Date.now()
|
||||
let result
|
||||
try {
|
||||
result = await providers[key].query(query.text, { brandHint: client.name })
|
||||
} catch (e) {
|
||||
result = {
|
||||
provider: key,
|
||||
content: '',
|
||||
ms: Date.now() - t0,
|
||||
costUsd: 0,
|
||||
error: e?.code || e?.message || 'UNKNOWN',
|
||||
}
|
||||
}
|
||||
|
||||
const mention = result.error
|
||||
? { mentioned: false, position: null, snippet: null }
|
||||
: detectMention(result.content, client)
|
||||
|
||||
repo.insertRun({
|
||||
client_id: clientId,
|
||||
query_id: query.id,
|
||||
provider: result.provider || key,
|
||||
mentioned: mention.mentioned,
|
||||
position: mention.position,
|
||||
snippet: mention.snippet,
|
||||
response_full: result.content || null,
|
||||
ms: result.ms || (Date.now() - t0),
|
||||
cost_usd: result.costUsd || 0,
|
||||
error: result.error || null,
|
||||
})
|
||||
|
||||
summary.totals.runs++
|
||||
if (mention.mentioned) summary.totals.mentions++
|
||||
if (result.error) summary.totals.errors++
|
||||
summary.totals.cost_usd += result.costUsd || 0
|
||||
|
||||
await new Promise((r) => setTimeout(r, INTER_REQUEST_MS))
|
||||
}
|
||||
}
|
||||
|
||||
repo.touchClientRun(clientId)
|
||||
summary.finished_at = new Date().toISOString()
|
||||
summary.totals.cost_usd = Number(summary.totals.cost_usd.toFixed(6))
|
||||
|
||||
const ms = Date.now() - wallStart
|
||||
console.log(
|
||||
`[monitoring] runId=${summary.runId} client=${client.hostname} queries=${queries.length} ` +
|
||||
`providers=${providerKeys.length} runs=${summary.totals.runs} ` +
|
||||
`mentions=${summary.totals.mentions} errors=${summary.totals.errors} ` +
|
||||
`cost_usd=${summary.totals.cost_usd} ms=${ms}`
|
||||
)
|
||||
summary.ms = ms
|
||||
return summary
|
||||
}
|
||||
16
server/lib/parser.js
Normal file
16
server/lib/parser.js
Normal file
@@ -0,0 +1,16 @@
|
||||
// Extracts the parts of the HTML response that downstream checks need.
|
||||
export function parseHtml(html) {
|
||||
const safe = html || ''
|
||||
const headMatch = safe.match(/<head[\s\S]*?<\/head>/i)
|
||||
const headHtml = headMatch ? headMatch[0] : safe.slice(0, 4000)
|
||||
|
||||
const jsonLdBlocks = [
|
||||
...safe.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi),
|
||||
]
|
||||
.map((m) => m[1].trim())
|
||||
.filter(Boolean)
|
||||
|
||||
const jsonLdJoined = jsonLdBlocks.join('\n---\n')
|
||||
|
||||
return { headHtml, jsonLdBlocks, jsonLdJoined }
|
||||
}
|
||||
191
server/lib/pipeline.js
Normal file
191
server/lib/pipeline.js
Normal file
@@ -0,0 +1,191 @@
|
||||
import { fetchPage } from './fetcher.js'
|
||||
import { parseHtml } from './parser.js'
|
||||
import { computeScore } from './scoring.js'
|
||||
import { generateSummary } from './summary.js'
|
||||
import { runAllChecks } from '../checks/index.js'
|
||||
import { generateAutofix } from './autofix/index.js'
|
||||
|
||||
const SEVERITY_ORDER = { high: 0, medium: 1, low: 2 }
|
||||
const MAX_ISSUES = 8
|
||||
const MAX_URL_LENGTH = 2048
|
||||
|
||||
// Map internal fetch error codes → { httpStatus, germanMessage }.
|
||||
export const FETCH_ERROR_MAP = {
|
||||
PRIVATE_HOST_BLOCKED: { http: 400, msg: 'Diese Adresse kann nicht analysiert werden.' },
|
||||
ENOTFOUND: { http: 502, msg: 'Die Website ist nicht erreichbar. Bitte URL prüfen.' },
|
||||
TIMEOUT: { http: 504, msg: 'Die Analyse hat zu lange gedauert. Bitte später erneut versuchen.' },
|
||||
SSL_INVALID: { http: 502, msg: 'SSL-Zertifikat der Website ist ungültig. Bitte den Hosting-Anbieter kontaktieren.' },
|
||||
NETWORK: { http: 502, msg: 'Die Website ist nicht erreichbar. Bitte URL prüfen.' },
|
||||
}
|
||||
|
||||
// HTML payload masquerading as text/plain (SPA fallback, custom 404) → treat as empty.
|
||||
function looksLikeText(body) {
|
||||
if (!body) return false
|
||||
return !body.trimStart().startsWith('<')
|
||||
}
|
||||
|
||||
export function validateUrl(rawInput) {
|
||||
if (typeof rawInput !== 'string' || rawInput.trim().length === 0) {
|
||||
return { error: { http: 400, code: 'EMPTY_URL', msg: 'Bitte eine URL eingeben.' } }
|
||||
}
|
||||
let raw = rawInput.trim()
|
||||
if (raw.length > MAX_URL_LENGTH) {
|
||||
return { error: { http: 400, code: 'URL_TOO_LONG', msg: 'URL ist zu lang.' } }
|
||||
}
|
||||
if (/^http:\/\//i.test(raw)) {
|
||||
return { error: { http: 400, code: 'HTTP_NOT_SUPPORTED', msg: 'HTTP wird nicht unterstützt. Bitte eine HTTPS-URL verwenden.' } }
|
||||
}
|
||||
raw = raw.replace(/^https?:\/\//i, '').replace(/\/+$/, '')
|
||||
const targetUrl = 'https://' + raw
|
||||
try {
|
||||
const u = new URL(targetUrl)
|
||||
return { targetUrl, host: u.hostname, originalOrigin: u.origin }
|
||||
} catch {
|
||||
return { error: { http: 400, code: 'INVALID_URL', msg: 'Ungültige URL. Bitte eine gültige Domain eingeben.' } }
|
||||
}
|
||||
}
|
||||
|
||||
// Runs the full analysis pipeline against an already-validated targetUrl.
|
||||
// Returns either `{ error: { http, code, msg } }` for hard fetch failures,
|
||||
// or `{ data: { score, summary, issues, autofix }, debugPayload, mainStatus }`.
|
||||
export async function runAnalysisPipeline(targetUrl, { debugMode = false } = {}) {
|
||||
const main = await fetchPage(targetUrl)
|
||||
|
||||
if (main.status === 0 && main.error && FETCH_ERROR_MAP[main.error]) {
|
||||
const mapping = FETCH_ERROR_MAP[main.error]
|
||||
return { error: { http: mapping.http, code: main.error, msg: mapping.msg } }
|
||||
}
|
||||
|
||||
const finalUrl = main.finalUrl || targetUrl
|
||||
let finalOrigin
|
||||
try {
|
||||
finalOrigin = new URL(finalUrl).origin
|
||||
} catch {
|
||||
finalOrigin = new URL(targetUrl).origin
|
||||
}
|
||||
|
||||
const [robotsRes, llmsRes] = await Promise.all([
|
||||
fetchPage(`${finalOrigin}/robots.txt`),
|
||||
fetchPage(`${finalOrigin}/llms.txt`),
|
||||
])
|
||||
|
||||
const robotsTxt = robotsRes.status === 200 && looksLikeText(robotsRes.body) ? robotsRes.body : ''
|
||||
const llmsTxt = llmsRes.status === 200 && looksLikeText(llmsRes.body) ? llmsRes.body : ''
|
||||
const llmsStatusEffective = llmsTxt ? llmsRes.status : 0
|
||||
|
||||
const { headHtml, jsonLdBlocks, jsonLdJoined } = parseHtml(main.body)
|
||||
|
||||
const probeRecords = {}
|
||||
const recordingFetch = async (url, opts) => {
|
||||
const r = await fetchPage(url, opts)
|
||||
const key = opts?.userAgent
|
||||
? (/Claude/i.test(opts.userAgent) ? 'uaClaudeBot'
|
||||
: /GPT/i.test(opts.userAgent) ? 'uaGptBot'
|
||||
: 'uaCustom')
|
||||
: (url.endsWith('/sitemap.xml') ? 'sitemap'
|
||||
: url.endsWith('/llms-full.txt') ? 'llmsFull'
|
||||
: 'extra')
|
||||
probeRecords[key] = { status: r.status, finalUrl: r.finalUrl, bodyLength: (r.body || '').length, ms: r.ms }
|
||||
return r
|
||||
}
|
||||
|
||||
const context = {
|
||||
baseUrl: finalOrigin,
|
||||
targetUrl: finalUrl,
|
||||
html: main.body || '',
|
||||
headHtml,
|
||||
jsonLdBlocks,
|
||||
jsonLdJoined,
|
||||
robotsTxt,
|
||||
llmsTxt,
|
||||
llmsStatus: llmsStatusEffective,
|
||||
mainStatus: main.status,
|
||||
responseHeaders: main.headers || {},
|
||||
fetchPage: recordingFetch,
|
||||
}
|
||||
|
||||
const results = await runAllChecks(context)
|
||||
const score = computeScore(results)
|
||||
|
||||
const failed = results
|
||||
.filter((r) => r.passed === false)
|
||||
.sort((a, b) => (SEVERITY_ORDER[a.severity] ?? 9) - (SEVERITY_ORDER[b.severity] ?? 9))
|
||||
|
||||
const summary = failed.length === 0
|
||||
? 'Alle GEO- und SEO-Signale sind vorhanden. Die Website ist optimal für KI-Suche konfiguriert.'
|
||||
: await generateSummary(failed.map((r) => r.title), finalUrl)
|
||||
|
||||
const issues = failed.slice(0, MAX_ISSUES).map((r) => ({
|
||||
title: r.title,
|
||||
severity: r.severity,
|
||||
}))
|
||||
|
||||
// Generate autofix from the same context the checks ran on.
|
||||
const autofixFull = generateAutofix(context)
|
||||
const { _siteData, ...autofixPublic } = autofixFull
|
||||
|
||||
const data = { score, summary, issues, autofix: autofixPublic }
|
||||
|
||||
// Surface failed-check IDs for activity logging only. Never returned to client.
|
||||
const failedCheckIds = failed.map((r) => r.id)
|
||||
|
||||
const debugPayload = debugMode
|
||||
? {
|
||||
requestedUrl: targetUrl,
|
||||
finalUrl,
|
||||
finalOrigin,
|
||||
fetches: {
|
||||
main: { status: main.status, finalUrl: main.finalUrl, bodyLength: (main.body || '').length, ms: main.ms, error: main.error },
|
||||
robots: { status: robotsRes.status, finalUrl: robotsRes.finalUrl, bodyLength: (robotsRes.body || '').length, ms: robotsRes.ms, error: robotsRes.error },
|
||||
llms: { status: llmsRes.status, finalUrl: llmsRes.finalUrl, bodyLength: (llmsRes.body || '').length, ms: llmsRes.ms, error: llmsRes.error },
|
||||
...probeRecords,
|
||||
},
|
||||
checks: results.map((r) => ({ id: r.id, passed: r.passed, severity: r.severity })),
|
||||
siteData: _siteData,
|
||||
}
|
||||
: null
|
||||
|
||||
return { data, debugPayload, mainStatus: main.status, _siteData, failedCheckIds }
|
||||
}
|
||||
|
||||
// Strips actionable content from the full pipeline output before sending to
|
||||
// unauthenticated visitors. The full data stays in the cache so admin and
|
||||
// future paid flows can still read it.
|
||||
function lineCount(s) {
|
||||
return (s || '').trim().split('\n').filter(Boolean).length
|
||||
}
|
||||
|
||||
function publicLabelFor(file) {
|
||||
switch (file?.mode) {
|
||||
case 'new': return 'neue Datei'
|
||||
case 'replace': return 'bestehende Datei ersetzen'
|
||||
case 'diff': return `${lineCount(file.content)} Zeilen ergänzen`
|
||||
case 'enhance': return 'erweitern'
|
||||
default: return ''
|
||||
}
|
||||
}
|
||||
|
||||
export function toPublicResponse(data) {
|
||||
const issues = data?.issues || []
|
||||
const issueCounts = issues.reduce(
|
||||
(acc, i) => { acc[i.severity] = (acc[i.severity] || 0) + 1; return acc },
|
||||
{ high: 0, medium: 0, low: 0 }
|
||||
)
|
||||
|
||||
const body = {
|
||||
score: data.score,
|
||||
summary: data.summary,
|
||||
issueCounts,
|
||||
}
|
||||
|
||||
// Score-10 sites get nothing to fix — omit the teaser entirely.
|
||||
const af = data.autofix
|
||||
if (af && issues.length > 0) {
|
||||
body.autofix = {
|
||||
llmsTxt: { mode: af.llmsTxt.mode, label: publicLabelFor(af.llmsTxt) },
|
||||
robotsTxt: { mode: af.robotsTxt.mode, label: publicLabelFor(af.robotsTxt) },
|
||||
jsonLd: { mode: af.jsonLd.mode, label: publicLabelFor(af.jsonLd) },
|
||||
}
|
||||
}
|
||||
return body
|
||||
}
|
||||
56
server/lib/providers/anthropic.js
Normal file
56
server/lib/providers/anthropic.js
Normal file
@@ -0,0 +1,56 @@
|
||||
// Anthropic Messages API. Cheapest current Haiku model.
|
||||
const ENDPOINT = 'https://api.anthropic.com/v1/messages'
|
||||
const MODEL = 'claude-haiku-4-5-20251001'
|
||||
const TIMEOUT_MS = 30_000
|
||||
const SYSTEM_PROMPT =
|
||||
'You are a helpful AI search assistant. Answer the user\'s question concisely, in German if the question is in German. Recommend specific companies, services, or products when relevant.'
|
||||
|
||||
// Approximate per-1M token rates in USD for Haiku 4.5.
|
||||
const COST = { inputPer1M: 1.00, outputPer1M: 5.00 }
|
||||
|
||||
async function once(prompt, signal) {
|
||||
return fetch(ENDPOINT, {
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-api-key': process.env.ANTHROPIC_KEY,
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: MODEL,
|
||||
max_tokens: 600,
|
||||
temperature: 0.3,
|
||||
system: SYSTEM_PROMPT,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
export async function query(prompt) {
|
||||
const t0 = Date.now()
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS)
|
||||
try {
|
||||
let res = await once(prompt, controller.signal)
|
||||
if (res.status >= 500) {
|
||||
res = await once(prompt, controller.signal)
|
||||
}
|
||||
if (!res.ok) {
|
||||
return { provider: 'anthropic', content: '', ms: Date.now() - t0, costUsd: 0, error: `ANTHROPIC_${res.status}` }
|
||||
}
|
||||
const data = await res.json()
|
||||
// Claude returns content as an array of blocks. Concatenate `text` blocks.
|
||||
const blocks = Array.isArray(data?.content) ? data.content : []
|
||||
const content = blocks.map((b) => b?.text || '').join('')
|
||||
const inTok = data?.usage?.input_tokens || 0
|
||||
const outTok = data?.usage?.output_tokens || 0
|
||||
const costUsd = (inTok * COST.inputPer1M + outTok * COST.outputPer1M) / 1_000_000
|
||||
return { provider: 'anthropic', content, ms: Date.now() - t0, costUsd, error: null }
|
||||
} catch (e) {
|
||||
const code = e?.name === 'AbortError' ? 'TIMEOUT' : 'NETWORK'
|
||||
return { provider: 'anthropic', content: '', ms: Date.now() - t0, costUsd: 0, error: code }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
32
server/lib/providers/index.js
Normal file
32
server/lib/providers/index.js
Normal file
@@ -0,0 +1,32 @@
|
||||
// Registry. If a provider's API key is missing, fall back to mock but keep
|
||||
// the original label (so the UI displays "openai (mock)" etc.).
|
||||
import * as openai from './openai.js'
|
||||
import * as perplexity from './perplexity.js'
|
||||
import * as anthropic from './anthropic.js'
|
||||
import * as mock from './mock.js'
|
||||
|
||||
function wrapAsMock(label) {
|
||||
return {
|
||||
query: async (prompt, opts) => {
|
||||
const r = await mock.query(prompt, opts)
|
||||
return { ...r, provider: `${label} (mock)` }
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function getProviders() {
|
||||
return {
|
||||
openai: process.env.OPENAI_KEY ? openai : wrapAsMock('openai'),
|
||||
perplexity: process.env.PERPLEXITY_KEY ? perplexity : wrapAsMock('perplexity'),
|
||||
anthropic: process.env.ANTHROPIC_KEY ? anthropic : wrapAsMock('anthropic'),
|
||||
}
|
||||
}
|
||||
|
||||
// Returns labels for the UI: { openai: 'real' | 'mock', ... }.
|
||||
export function getProviderModes() {
|
||||
return {
|
||||
openai: process.env.OPENAI_KEY ? 'real' : 'mock',
|
||||
perplexity: process.env.PERPLEXITY_KEY ? 'real' : 'mock',
|
||||
anthropic: process.env.ANTHROPIC_KEY ? 'real' : 'mock',
|
||||
}
|
||||
}
|
||||
11
server/lib/providers/mock.js
Normal file
11
server/lib/providers/mock.js
Normal file
@@ -0,0 +1,11 @@
|
||||
// Deterministic fake provider for tests and zero-key dev. The returned content
|
||||
// sometimes includes the brand name (hash-driven) to exercise mention detection.
|
||||
export async function query(prompt, { brandHint } = {}) {
|
||||
await new Promise((r) => setTimeout(r, 80))
|
||||
const hash = String(prompt).split('').reduce((a, c) => a + c.charCodeAt(0), 0)
|
||||
const willMention = brandHint && (hash % 2 === 0)
|
||||
const content = willMention
|
||||
? `Eine Option wäre ${brandHint}. Es gibt auch andere Anbieter wie Beispiel-AG und Muster GmbH.`
|
||||
: `Mögliche Anbieter in diesem Bereich sind unter anderem Beispiel-AG, Muster GmbH und Test Solutions.`
|
||||
return { provider: 'mock', content, ms: 80, costUsd: 0, error: null }
|
||||
}
|
||||
56
server/lib/providers/openai.js
Normal file
56
server/lib/providers/openai.js
Normal file
@@ -0,0 +1,56 @@
|
||||
// OpenAI chat completions provider. Uses the cheapest GPT-4-class model.
|
||||
const ENDPOINT = 'https://api.openai.com/v1/chat/completions'
|
||||
const MODEL = 'gpt-4o-mini'
|
||||
const TIMEOUT_MS = 30_000
|
||||
const SYSTEM_PROMPT =
|
||||
'You are a helpful AI search assistant. Answer the user\'s question concisely, in German if the question is in German. Recommend specific companies, services, or products when relevant.'
|
||||
|
||||
// Approximate per-1M token rates in USD for gpt-4o-mini.
|
||||
const COST = { inputPer1M: 0.15, outputPer1M: 0.60 }
|
||||
|
||||
async function once(prompt, signal) {
|
||||
const res = await fetch(ENDPOINT, {
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${process.env.OPENAI_KEY}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: MODEL,
|
||||
temperature: 0.3,
|
||||
max_tokens: 600,
|
||||
messages: [
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: prompt },
|
||||
],
|
||||
}),
|
||||
})
|
||||
return res
|
||||
}
|
||||
|
||||
export async function query(prompt) {
|
||||
const t0 = Date.now()
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS)
|
||||
try {
|
||||
let res = await once(prompt, controller.signal)
|
||||
if (res.status >= 500) {
|
||||
res = await once(prompt, controller.signal)
|
||||
}
|
||||
if (!res.ok) {
|
||||
return { provider: 'openai', content: '', ms: Date.now() - t0, costUsd: 0, error: `OPENAI_${res.status}` }
|
||||
}
|
||||
const data = await res.json()
|
||||
const content = data?.choices?.[0]?.message?.content || ''
|
||||
const inTok = data?.usage?.prompt_tokens || 0
|
||||
const outTok = data?.usage?.completion_tokens || 0
|
||||
const costUsd = (inTok * COST.inputPer1M + outTok * COST.outputPer1M) / 1_000_000
|
||||
return { provider: 'openai', content, ms: Date.now() - t0, costUsd, error: null }
|
||||
} catch (e) {
|
||||
const code = e?.name === 'AbortError' ? 'TIMEOUT' : 'NETWORK'
|
||||
return { provider: 'openai', content: '', ms: Date.now() - t0, costUsd: 0, error: code }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
64
server/lib/providers/perplexity.js
Normal file
64
server/lib/providers/perplexity.js
Normal file
@@ -0,0 +1,64 @@
|
||||
// Perplexity is the highest-priority provider — it is an actual AI search
|
||||
// engine, where real users go for recommendations. OpenAI-compatible schema.
|
||||
const ENDPOINT = 'https://api.perplexity.ai/chat/completions'
|
||||
const MODEL = 'sonar'
|
||||
const TIMEOUT_MS = 30_000
|
||||
const SYSTEM_PROMPT =
|
||||
'You are a helpful AI search assistant. Answer the user\'s question concisely, in German if the question is in German. Recommend specific companies, services, or products when relevant.'
|
||||
|
||||
// Approximate per-1M token rates in USD for `sonar`.
|
||||
const COST = { inputPer1M: 1.00, outputPer1M: 1.00 }
|
||||
|
||||
async function once(prompt, signal) {
|
||||
return fetch(ENDPOINT, {
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${process.env.PERPLEXITY_KEY}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: MODEL,
|
||||
temperature: 0.3,
|
||||
max_tokens: 600,
|
||||
messages: [
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: prompt },
|
||||
],
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
export async function query(prompt) {
|
||||
const t0 = Date.now()
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS)
|
||||
try {
|
||||
let res = await once(prompt, controller.signal)
|
||||
if (res.status >= 500) {
|
||||
res = await once(prompt, controller.signal)
|
||||
}
|
||||
if (!res.ok) {
|
||||
return { provider: 'perplexity', content: '', ms: Date.now() - t0, costUsd: 0, error: `PERPLEXITY_${res.status}` }
|
||||
}
|
||||
const data = await res.json()
|
||||
const content = data?.choices?.[0]?.message?.content || ''
|
||||
const inTok = data?.usage?.prompt_tokens || 0
|
||||
const outTok = data?.usage?.completion_tokens || 0
|
||||
// Fall back to a character-based rough estimate if usage is absent.
|
||||
let costUsd
|
||||
if (inTok || outTok) {
|
||||
costUsd = (inTok * COST.inputPer1M + outTok * COST.outputPer1M) / 1_000_000
|
||||
} else {
|
||||
const approxIn = Math.ceil((prompt.length + SYSTEM_PROMPT.length) / 4)
|
||||
const approxOut = Math.ceil(content.length / 4)
|
||||
costUsd = (approxIn * COST.inputPer1M + approxOut * COST.outputPer1M) / 1_000_000
|
||||
}
|
||||
return { provider: 'perplexity', content, ms: Date.now() - t0, costUsd, error: null }
|
||||
} catch (e) {
|
||||
const code = e?.name === 'AbortError' ? 'TIMEOUT' : 'NETWORK'
|
||||
return { provider: 'perplexity', content: '', ms: Date.now() - t0, costUsd: 0, error: code }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
15
server/lib/scoring.js
Normal file
15
server/lib/scoring.js
Normal file
@@ -0,0 +1,15 @@
|
||||
const WEIGHTS = { high: 3, medium: 2, low: 1 }
|
||||
|
||||
export function computeScore(results) {
|
||||
if (!Array.isArray(results) || results.length === 0) return 1
|
||||
let earned = 0
|
||||
let max = 0
|
||||
for (const r of results) {
|
||||
const w = WEIGHTS[r.severity] || 1
|
||||
max += w
|
||||
if (r.passed) earned += w
|
||||
}
|
||||
if (max === 0) return 1
|
||||
const raw = 1 + 9 * (earned / max)
|
||||
return Math.max(1, Math.min(10, Math.round(raw)))
|
||||
}
|
||||
42
server/lib/summary.js
Normal file
42
server/lib/summary.js
Normal file
@@ -0,0 +1,42 @@
|
||||
const SUMMARY_PROMPT = `You are a GEO/SEO auditor. The following checks were run on a website. Some FAILED.
|
||||
Write a summary in German: 1-2 sentences describing what is missing and why it matters for AI visibility.
|
||||
Be specific about which signals are missing. Do not invent checks — only describe the ones listed.
|
||||
Return ONLY the summary text, no JSON, no markdown.`
|
||||
|
||||
function fallback(failedTitles) {
|
||||
return `${failedTitles.length} GEO/SEO-Signale fehlen. Behebe die aufgelisteten Probleme für bessere KI-Sichtbarkeit.`
|
||||
}
|
||||
|
||||
export async function generateSummary(failedTitles, targetUrl) {
|
||||
const apiKey = process.env.MISTRAL_KEY
|
||||
if (!apiKey || failedTitles.length === 0) return fallback(failedTitles)
|
||||
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), 10000)
|
||||
try {
|
||||
const res = await fetch('https://api.mistral.ai/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'mistral-large-latest',
|
||||
max_tokens: 150,
|
||||
messages: [
|
||||
{ role: 'system', content: SUMMARY_PROMPT },
|
||||
{ role: 'user', content: `Website: ${targetUrl}\nFehlende Signale: ${failedTitles.join(', ')}` },
|
||||
],
|
||||
}),
|
||||
})
|
||||
if (!res.ok) return fallback(failedTitles)
|
||||
const data = await res.json()
|
||||
const text = (data?.choices?.[0]?.message?.content || '').trim()
|
||||
return text || fallback(failedTitles)
|
||||
} catch {
|
||||
return fallback(failedTitles)
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
1409
server/package-lock.json
generated
Normal file
1409
server/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
18
server/package.json
Normal file
18
server/package.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "visigine-server",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "node --watch index.js",
|
||||
"start": "node index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^12.10.0",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.21.0",
|
||||
"express-rate-limit": "^7.4.0",
|
||||
"jszip": "^3.10.1"
|
||||
}
|
||||
}
|
||||
232
server/routes/admin-monitoring.js
Normal file
232
server/routes/admin-monitoring.js
Normal file
@@ -0,0 +1,232 @@
|
||||
import { Router } from 'express'
|
||||
import * as repo from '../db/repo.js'
|
||||
import { runMonitoring, RunError } from '../lib/monitoring/run.js'
|
||||
import { generateQueries } from '../lib/monitoring/generate-queries.js'
|
||||
import { runAnalysisPipeline, validateUrl } from '../lib/pipeline.js'
|
||||
import { getProviderModes } from '../lib/providers/index.js'
|
||||
|
||||
const router = Router()
|
||||
|
||||
function serializeClient(c) {
|
||||
if (!c) return null
|
||||
let aliases = []
|
||||
try { aliases = JSON.parse(c.brand_aliases || '[]') } catch { /* keep [] */ }
|
||||
return { ...c, brand_aliases: aliases }
|
||||
}
|
||||
|
||||
function mentionRate(mentions, runs) {
|
||||
if (!runs || runs <= 0) return null
|
||||
return Number((mentions / runs).toFixed(4))
|
||||
}
|
||||
|
||||
// ─── clients ──────────────────────────────────────────────────────
|
||||
|
||||
router.get('/clients', (_req, res) => {
|
||||
const rows = repo.listClients().map((c) => ({
|
||||
id: c.id,
|
||||
hostname: c.hostname,
|
||||
name: c.name,
|
||||
status: c.status,
|
||||
queries_count: c.queries_count,
|
||||
last_run_at: c.last_run_at,
|
||||
mention_rate_30d: mentionRate(c.mentions_30d, c.runs_30d),
|
||||
runs_30d: c.runs_30d,
|
||||
mentions_30d: c.mentions_30d,
|
||||
}))
|
||||
res.json({ clients: rows, providerModes: getProviderModes() })
|
||||
})
|
||||
|
||||
router.post('/clients', async (req, res) => {
|
||||
const body = req.body || {}
|
||||
if (!body.url) return res.status(400).json({ error: 'URL erforderlich.' })
|
||||
|
||||
const v = validateUrl(body.url)
|
||||
if (v.error) return res.status(v.error.http).json({ error: v.error.msg })
|
||||
|
||||
// If hostname already exists, return existing instead of duplicating.
|
||||
const existing = repo.getClientByHost(v.host)
|
||||
if (existing) return res.json(serializeClient(existing))
|
||||
|
||||
// Pull siteData from the analyze pipeline if name/description not provided.
|
||||
let { hostname, name, description, language, brand_aliases } = body
|
||||
hostname = hostname || v.host
|
||||
|
||||
if (!name || !description) {
|
||||
try {
|
||||
const out = await runAnalysisPipeline(v.targetUrl, { debugMode: false })
|
||||
if (!out.error) {
|
||||
const sd = out._siteData || {}
|
||||
name = name || sd.name || v.host
|
||||
description = description || sd.description || null
|
||||
language = language || sd.language || 'de'
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[admin-monitoring] prefill failed:', e?.message || e)
|
||||
}
|
||||
}
|
||||
|
||||
const aliasesJson = JSON.stringify(Array.isArray(brand_aliases) ? brand_aliases : [])
|
||||
|
||||
try {
|
||||
const created = repo.insertClient({
|
||||
hostname,
|
||||
url: v.targetUrl,
|
||||
name: name || v.host,
|
||||
description,
|
||||
brand_aliases: aliasesJson,
|
||||
language: language || 'de',
|
||||
})
|
||||
return res.status(201).json(serializeClient(created))
|
||||
} catch (err) {
|
||||
if (String(err?.message || '').includes('UNIQUE')) {
|
||||
return res.status(409).json({ error: 'Marke existiert bereits.' })
|
||||
}
|
||||
console.error('[admin-monitoring] insert error', err)
|
||||
return res.status(500).json({ error: 'Ein interner Fehler ist aufgetreten.' })
|
||||
}
|
||||
})
|
||||
|
||||
router.get('/clients/:id', (req, res) => {
|
||||
const c = repo.getClient(Number(req.params.id))
|
||||
if (!c) return res.status(404).json({ error: 'Marke nicht gefunden.' })
|
||||
res.json(serializeClient(c))
|
||||
})
|
||||
|
||||
router.patch('/clients/:id', (req, res) => {
|
||||
const id = Number(req.params.id)
|
||||
const existing = repo.getClient(id)
|
||||
if (!existing) return res.status(404).json({ error: 'Marke nicht gefunden.' })
|
||||
const patch = req.body || {}
|
||||
if (patch.status && !['active', 'paused', 'archived'].includes(patch.status)) {
|
||||
return res.status(400).json({ error: 'Ungültiger Status.' })
|
||||
}
|
||||
if (patch.brand_aliases !== undefined && Array.isArray(patch.brand_aliases)) {
|
||||
patch.brand_aliases = JSON.stringify(patch.brand_aliases)
|
||||
}
|
||||
const updated = repo.updateClient(id, patch)
|
||||
res.json(serializeClient(updated))
|
||||
})
|
||||
|
||||
router.delete('/clients/:id', (req, res) => {
|
||||
const ok = repo.deleteClient(Number(req.params.id))
|
||||
if (!ok) return res.status(404).json({ error: 'Marke nicht gefunden.' })
|
||||
res.json({ deleted: true })
|
||||
})
|
||||
|
||||
// ─── queries ──────────────────────────────────────────────────────
|
||||
|
||||
router.post('/clients/:id/generate-queries', async (req, res) => {
|
||||
const id = Number(req.params.id)
|
||||
const c = repo.getClient(id)
|
||||
if (!c) return res.status(404).json({ error: 'Marke nicht gefunden.' })
|
||||
|
||||
const previousCount = repo.listAllQueries(id).length
|
||||
const result = await generateQueries({
|
||||
name: c.name,
|
||||
description: c.description,
|
||||
url: c.url,
|
||||
hostname: c.hostname,
|
||||
})
|
||||
|
||||
if (!result.queries || result.queries.length === 0) {
|
||||
return res.status(502).json({ error: 'Query-Generierung fehlgeschlagen. Bitte später erneut versuchen.' })
|
||||
}
|
||||
|
||||
repo.replaceQueries(id, result.queries)
|
||||
return res.json({
|
||||
generated: result.queries.length,
|
||||
replaced: previousCount,
|
||||
warning: result.warning || null,
|
||||
})
|
||||
})
|
||||
|
||||
router.get('/clients/:id/queries', (req, res) => {
|
||||
const id = Number(req.params.id)
|
||||
if (!repo.getClient(id)) return res.status(404).json({ error: 'Marke nicht gefunden.' })
|
||||
res.json({ queries: repo.listAllQueries(id) })
|
||||
})
|
||||
|
||||
router.post('/clients/:id/queries', (req, res) => {
|
||||
const id = Number(req.params.id)
|
||||
if (!repo.getClient(id)) return res.status(404).json({ error: 'Marke nicht gefunden.' })
|
||||
const text = String(req.body?.text || '').trim()
|
||||
if (text.length < 5) return res.status(400).json({ error: 'Query zu kurz.' })
|
||||
const q = repo.insertQuery(id, text)
|
||||
res.status(201).json(q)
|
||||
})
|
||||
|
||||
router.patch('/queries/:id', (req, res) => {
|
||||
const id = Number(req.params.id)
|
||||
if (!repo.getQuery(id)) return res.status(404).json({ error: 'Query nicht gefunden.' })
|
||||
const patch = req.body || {}
|
||||
if (patch.text !== undefined) patch.text = String(patch.text).trim()
|
||||
if (patch.text !== undefined && patch.text.length < 5) {
|
||||
return res.status(400).json({ error: 'Query zu kurz.' })
|
||||
}
|
||||
const q = repo.updateQuery(id, patch)
|
||||
res.json(q)
|
||||
})
|
||||
|
||||
router.delete('/queries/:id', (req, res) => {
|
||||
const ok = repo.deleteQuery(Number(req.params.id))
|
||||
if (!ok) return res.status(404).json({ error: 'Query nicht gefunden.' })
|
||||
res.json({ deleted: true })
|
||||
})
|
||||
|
||||
// ─── runs ─────────────────────────────────────────────────────────
|
||||
|
||||
router.post('/clients/:id/run', async (req, res) => {
|
||||
const id = Number(req.params.id)
|
||||
try {
|
||||
const summary = await runMonitoring(id)
|
||||
res.json(summary)
|
||||
} catch (err) {
|
||||
if (err instanceof RunError) {
|
||||
const status = err.code === 'CLIENT_NOT_FOUND' ? 404 : 400
|
||||
const message = err.code === 'CLIENT_NOT_FOUND' ? 'Marke nicht gefunden.'
|
||||
: err.code === 'NO_ACTIVE_QUERIES' ? 'Keine aktiven Queries — bitte zuerst generieren oder anlegen.'
|
||||
: err.code === 'CLIENT_NOT_ACTIVE' ? 'Marke ist pausiert oder archiviert.'
|
||||
: err.code
|
||||
return res.status(status).json({ error: message, code: err.code })
|
||||
}
|
||||
console.error('[admin-monitoring] run error', err)
|
||||
return res.status(500).json({ error: 'Ein interner Fehler ist aufgetreten.' })
|
||||
}
|
||||
})
|
||||
|
||||
router.get('/clients/:id/runs', (req, res) => {
|
||||
const id = Number(req.params.id)
|
||||
if (!repo.getClient(id)) return res.status(404).json({ error: 'Marke nicht gefunden.' })
|
||||
const page = Math.max(1, Number(req.query.page || 1))
|
||||
const limit = Math.min(200, Math.max(1, Number(req.query.limit || 50)))
|
||||
const offset = (page - 1) * limit
|
||||
res.json({
|
||||
page,
|
||||
limit,
|
||||
total: repo.countRuns(id),
|
||||
runs: repo.recentRuns(id, limit, offset),
|
||||
})
|
||||
})
|
||||
|
||||
router.get('/clients/:id/stats', (req, res) => {
|
||||
const id = Number(req.params.id)
|
||||
if (!repo.getClient(id)) return res.status(404).json({ error: 'Marke nicht gefunden.' })
|
||||
const byProvider = repo.statsByProvider(id)
|
||||
const totals = repo.totalsLast30d(id)
|
||||
res.json({
|
||||
byProvider: byProvider.map((r) => ({
|
||||
provider: r.provider,
|
||||
total: r.total,
|
||||
mentions: r.mentions,
|
||||
cost: Number((r.cost || 0).toFixed(6)),
|
||||
mention_rate: mentionRate(r.mentions, r.total),
|
||||
last_run: r.last_run,
|
||||
})),
|
||||
totalRuns30d: totals?.total || 0,
|
||||
totalMentions30d: totals?.mentions || 0,
|
||||
mentionRate30d: mentionRate(totals?.mentions, totals?.total),
|
||||
totalCost30d: Number(((totals?.cost) || 0).toFixed(6)),
|
||||
})
|
||||
})
|
||||
|
||||
export default router
|
||||
159
server/routes/admin.js
Normal file
159
server/routes/admin.js
Normal file
@@ -0,0 +1,159 @@
|
||||
import { Router } from 'express'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import JSZip from 'jszip'
|
||||
import { runAnalysisPipeline, validateUrl } from '../lib/pipeline.js'
|
||||
import { buildReadme } from '../lib/autofix/index.js'
|
||||
import { recordAnalysis, recentAnalyses, computeStats } from '../lib/activity.js'
|
||||
import { cacheGet, cacheSet } from '../lib/cache.js'
|
||||
|
||||
function normalizeForCache(url) {
|
||||
try {
|
||||
const u = new URL(url)
|
||||
return `${u.protocol}//${u.hostname.toLowerCase()}${u.pathname || '/'}`
|
||||
} catch {
|
||||
return url.toLowerCase()
|
||||
}
|
||||
}
|
||||
|
||||
function safeFilename(host) {
|
||||
if (!host) return 'visigine-autofix.zip'
|
||||
const cleaned = host.toLowerCase().replace(/[^a-z0-9.\-]/g, '').slice(0, 50)
|
||||
return cleaned ? `visigine-autofix-${cleaned}.zip` : 'visigine-autofix.zip'
|
||||
}
|
||||
|
||||
const router = Router()
|
||||
|
||||
// Admin analyze: always debug mode. Cache bypass defaults to true but can be
|
||||
// toggled off via { bypassCache: false } in the request body — useful for
|
||||
// inspecting what a regular cached response would look like.
|
||||
// Critical: `_debug` is NEVER stripped here, even in NODE_ENV=production.
|
||||
router.post('/analyze', async (req, res) => {
|
||||
const started = Date.now()
|
||||
const requestId = randomUUID().slice(0, 8)
|
||||
res.setHeader('X-Request-Id', requestId)
|
||||
|
||||
const v = validateUrl(req?.body?.url)
|
||||
if (v.error) {
|
||||
const ms = Date.now() - started
|
||||
recordAnalysis({ requestId, host: null, ms, status: 'err', code: v.error.code, admin: true })
|
||||
return res.status(v.error.http).json({ error: v.error.msg })
|
||||
}
|
||||
const { targetUrl, host } = v
|
||||
const bypassCache = req.body?.bypassCache !== false
|
||||
const cacheKey = normalizeForCache(targetUrl)
|
||||
|
||||
// When the admin opts in to the cache, serve from it just like /api/analyze
|
||||
// would. We still attach a _debug shell so the UI can render its sections.
|
||||
if (!bypassCache) {
|
||||
const cached = cacheGet(cacheKey)
|
||||
if (cached) {
|
||||
const ms = Date.now() - started
|
||||
res.setHeader('X-Cache', 'HIT')
|
||||
recordAnalysis({
|
||||
requestId, host,
|
||||
score: cached.data.score, issuesCount: cached.data.issues.length,
|
||||
failedCheckIds: cached.failedCheckIds || [],
|
||||
cacheHit: true, ms, status: 'ok', admin: true,
|
||||
})
|
||||
const response = {
|
||||
...cached.data,
|
||||
_debug: cached.debugPayload
|
||||
? { ...cached.debugPayload, totalMs: ms, cacheHit: true }
|
||||
: { totalMs: ms, cacheHit: true, checks: [], fetches: {}, siteData: null, note: 'Aus Cache geladen — keine frischen Debug-Daten verfügbar.' },
|
||||
}
|
||||
return res.json(response)
|
||||
}
|
||||
}
|
||||
res.setHeader('X-Cache', 'MISS')
|
||||
|
||||
try {
|
||||
const out = await runAnalysisPipeline(targetUrl, { debugMode: true })
|
||||
if (out.error) {
|
||||
const ms = Date.now() - started
|
||||
recordAnalysis({ requestId, host, cacheHit: false, ms, status: 'err', code: out.error.code, admin: true })
|
||||
return res.status(out.error.http).json({ error: out.error.msg })
|
||||
}
|
||||
|
||||
const ms = Date.now() - started
|
||||
const response = { ...out.data, _debug: { ...out.debugPayload, totalMs: ms, cacheHit: false } }
|
||||
|
||||
// Populate the cache so non-bypass admin requests (and the public route)
|
||||
// can later read it.
|
||||
if (!bypassCache && out.mainStatus >= 200 && out.mainStatus < 400) {
|
||||
cacheSet(cacheKey, { data: out.data, failedCheckIds: out.failedCheckIds, debugPayload: out.debugPayload })
|
||||
}
|
||||
|
||||
recordAnalysis({
|
||||
requestId, host,
|
||||
score: out.data.score, issuesCount: out.data.issues.length,
|
||||
failedCheckIds: out.failedCheckIds,
|
||||
cacheHit: false, ms, status: 'ok', admin: true,
|
||||
})
|
||||
return res.json(response)
|
||||
} catch (err) {
|
||||
console.error('[admin-analyze-error]', requestId, err?.message || err)
|
||||
const ms = Date.now() - started
|
||||
recordAnalysis({ requestId, host, cacheHit: false, ms, status: 'err', code: 'INTERNAL', admin: true })
|
||||
return res.status(500).json({ error: 'Ein interner Fehler ist aufgetreten.' })
|
||||
}
|
||||
})
|
||||
|
||||
router.get('/recent', (_req, res) => {
|
||||
res.json({ analyses: recentAnalyses() })
|
||||
})
|
||||
|
||||
router.get('/stats', (_req, res) => {
|
||||
res.json(computeStats())
|
||||
})
|
||||
|
||||
// Admin ZIP: always bypass cache, no rate limit.
|
||||
router.post('/autofix/zip', async (req, res) => {
|
||||
const started = Date.now()
|
||||
const requestId = randomUUID().slice(0, 8)
|
||||
res.setHeader('X-Request-Id', requestId)
|
||||
|
||||
const v = validateUrl(req?.body?.url)
|
||||
if (v.error) {
|
||||
const ms = Date.now() - started
|
||||
recordAnalysis({ requestId, host: null, ms, status: 'err', code: v.error.code, admin: true })
|
||||
return res.status(v.error.http).json({ error: v.error.msg })
|
||||
}
|
||||
const { targetUrl, host } = v
|
||||
|
||||
try {
|
||||
const out = await runAnalysisPipeline(targetUrl, { debugMode: false })
|
||||
if (out.error) {
|
||||
const ms = Date.now() - started
|
||||
recordAnalysis({ requestId, host, cacheHit: false, ms, status: 'err', code: out.error.code, admin: true })
|
||||
return res.status(out.error.http).json({ error: out.error.msg })
|
||||
}
|
||||
|
||||
const zip = new JSZip()
|
||||
zip.file('llms.txt', out.data.autofix.llmsTxt.content)
|
||||
zip.file('robots.txt', out.data.autofix.robotsTxt.content)
|
||||
zip.file('jsonld.html', out.data.autofix.jsonLd.content)
|
||||
zip.file('README.txt', buildReadme(out.data.autofix))
|
||||
const buffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' })
|
||||
|
||||
const filename = safeFilename(host)
|
||||
res.setHeader('Content-Type', 'application/zip')
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`)
|
||||
res.setHeader('Content-Length', buffer.length)
|
||||
|
||||
const ms = Date.now() - started
|
||||
recordAnalysis({
|
||||
requestId, host,
|
||||
score: out.data.score, issuesCount: out.data.issues.length,
|
||||
failedCheckIds: out.failedCheckIds,
|
||||
cacheHit: false, ms, status: 'ok', admin: true,
|
||||
})
|
||||
return res.send(buffer)
|
||||
} catch (err) {
|
||||
console.error('[admin-zip-error]', requestId, err?.message || err)
|
||||
const ms = Date.now() - started
|
||||
recordAnalysis({ requestId, host, cacheHit: false, ms, status: 'err', code: 'INTERNAL', admin: true })
|
||||
return res.status(500).json({ error: 'Ein interner Fehler ist aufgetreten.' })
|
||||
}
|
||||
})
|
||||
|
||||
export default router
|
||||
118
server/routes/analyze.js
Normal file
118
server/routes/analyze.js
Normal file
@@ -0,0 +1,118 @@
|
||||
import { Router } from 'express'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { cacheGet, cacheSet } from '../lib/cache.js'
|
||||
import { runAnalysisPipeline, validateUrl, toPublicResponse } from '../lib/pipeline.js'
|
||||
import { recordAnalysis } from '../lib/activity.js'
|
||||
|
||||
function normalizeForCache(url) {
|
||||
try {
|
||||
const u = new URL(url)
|
||||
return `${u.protocol}//${u.hostname.toLowerCase()}${u.pathname || '/'}`
|
||||
} catch {
|
||||
return url.toLowerCase()
|
||||
}
|
||||
}
|
||||
|
||||
function nowIso() {
|
||||
return new Date().toISOString().replace(/\.\d{3}Z$/, 'Z')
|
||||
}
|
||||
|
||||
function logLine({ requestId, host, score, issues, cache, ms, status, code }) {
|
||||
const parts = [
|
||||
`[analyze]`,
|
||||
`id=${requestId}`,
|
||||
`ts=${nowIso()}`,
|
||||
`host=${host || '-'}`,
|
||||
]
|
||||
if (score !== undefined) parts.push(`score=${score}`)
|
||||
if (issues !== undefined) parts.push(`issues=${issues}`)
|
||||
parts.push(`cache=${cache}`)
|
||||
parts.push(`ms=${ms}`)
|
||||
parts.push(`status=${status}`)
|
||||
if (code) parts.push(`code=${code}`)
|
||||
console.log(parts.join(' '))
|
||||
}
|
||||
|
||||
const router = Router()
|
||||
|
||||
router.post('/', async (req, res) => {
|
||||
const started = Date.now()
|
||||
const requestId = randomUUID().slice(0, 8)
|
||||
res.setHeader('X-Request-Id', requestId)
|
||||
|
||||
const debugRequested = req.query?.debug === '1'
|
||||
const isProd = process.env.NODE_ENV === 'production'
|
||||
const debugMode = debugRequested && !isProd
|
||||
|
||||
const v = validateUrl(req?.body?.url)
|
||||
if (v.error) {
|
||||
const ms = Date.now() - started
|
||||
logLine({ requestId, cache: 'miss', ms, status: 'err', code: v.error.code })
|
||||
recordAnalysis({ requestId, host: null, ms, status: 'err', code: v.error.code })
|
||||
return res.status(v.error.http).json({ error: v.error.msg })
|
||||
}
|
||||
const { targetUrl, host: hostForLog } = v
|
||||
const cacheKey = normalizeForCache(targetUrl)
|
||||
|
||||
if (!debugMode) {
|
||||
const cached = cacheGet(cacheKey)
|
||||
if (cached) {
|
||||
const ms = Date.now() - started
|
||||
res.setHeader('X-Cache', 'HIT')
|
||||
logLine({
|
||||
requestId, host: hostForLog,
|
||||
score: cached.data.score, issues: cached.data.issues.length,
|
||||
cache: 'hit', ms, status: 'ok',
|
||||
})
|
||||
recordAnalysis({
|
||||
requestId, host: hostForLog,
|
||||
score: cached.data.score, issuesCount: cached.data.issues.length,
|
||||
failedCheckIds: cached.failedCheckIds || [],
|
||||
cacheHit: true, ms, status: 'ok',
|
||||
})
|
||||
return res.json(toPublicResponse(cached.data))
|
||||
}
|
||||
}
|
||||
res.setHeader('X-Cache', 'MISS')
|
||||
|
||||
try {
|
||||
const out = await runAnalysisPipeline(targetUrl, { debugMode })
|
||||
if (out.error) {
|
||||
const ms = Date.now() - started
|
||||
logLine({ requestId, host: hostForLog, cache: 'miss', ms, status: 'err', code: out.error.code })
|
||||
recordAnalysis({ requestId, host: hostForLog, cacheHit: false, ms, status: 'err', code: out.error.code })
|
||||
return res.status(out.error.http).json({ error: out.error.msg })
|
||||
}
|
||||
|
||||
const response = toPublicResponse(out.data)
|
||||
if (debugMode && out.debugPayload) response._debug = out.debugPayload
|
||||
// Belt-and-suspenders: in production, never serve _debug.
|
||||
if (isProd && response._debug) delete response._debug
|
||||
|
||||
if (!debugMode && out.mainStatus >= 200 && out.mainStatus < 400) {
|
||||
cacheSet(cacheKey, { data: out.data, failedCheckIds: out.failedCheckIds })
|
||||
}
|
||||
|
||||
const ms = Date.now() - started
|
||||
logLine({
|
||||
requestId, host: hostForLog,
|
||||
score: out.data.score, issues: out.data.issues.length,
|
||||
cache: 'miss', ms, status: 'ok',
|
||||
})
|
||||
recordAnalysis({
|
||||
requestId, host: hostForLog,
|
||||
score: out.data.score, issuesCount: out.data.issues.length,
|
||||
failedCheckIds: out.failedCheckIds,
|
||||
cacheHit: false, ms, status: 'ok',
|
||||
})
|
||||
return res.json(response)
|
||||
} catch (err) {
|
||||
console.error('[analyze-error]', requestId, err?.message || err)
|
||||
const ms = Date.now() - started
|
||||
logLine({ requestId, host: hostForLog, cache: 'miss', ms, status: 'err', code: 'INTERNAL' })
|
||||
recordAnalysis({ requestId, host: hostForLog, cacheHit: false, ms, status: 'err', code: 'INTERNAL' })
|
||||
return res.status(500).json({ error: 'Ein interner Fehler ist aufgetreten.' })
|
||||
}
|
||||
})
|
||||
|
||||
export default router
|
||||
64
server/routes/demo-monitoring.js
Normal file
64
server/routes/demo-monitoring.js
Normal file
@@ -0,0 +1,64 @@
|
||||
// Reserved for iteration 4b. Backend-only for now — no UI surfaces this yet.
|
||||
// No auth, 5/hour/IP rate limit. Runs a single (query × provider) against one
|
||||
// real-or-mock provider and returns the result. Nothing is persisted.
|
||||
import { Router } from 'express'
|
||||
import { validateUrl, runAnalysisPipeline } from '../lib/pipeline.js'
|
||||
import { getProviders } from '../lib/providers/index.js'
|
||||
import { detectMention } from '../lib/monitoring/detect-mention.js'
|
||||
import { generateQueries } from '../lib/monitoring/generate-queries.js'
|
||||
|
||||
const router = Router()
|
||||
|
||||
router.post('/', async (req, res) => {
|
||||
const url = req.body?.url
|
||||
let queryText = (req.body?.query || '').trim()
|
||||
|
||||
const v = validateUrl(url)
|
||||
if (v.error) return res.status(v.error.http).json({ error: v.error.msg })
|
||||
|
||||
// If no query supplied, generate one quickly. We pull siteData from a fresh
|
||||
// analyze pass so the generated query is on-brand.
|
||||
let clientLike
|
||||
try {
|
||||
const out = await runAnalysisPipeline(v.targetUrl, { debugMode: false })
|
||||
if (out.error) return res.status(out.error.http).json({ error: out.error.msg })
|
||||
const sd = out._siteData || {}
|
||||
clientLike = {
|
||||
name: sd.name || v.host,
|
||||
hostname: v.host,
|
||||
brand_aliases: '[]',
|
||||
}
|
||||
if (!queryText) {
|
||||
const { queries } = await generateQueries({
|
||||
name: clientLike.name,
|
||||
description: sd.description,
|
||||
url: v.targetUrl,
|
||||
hostname: v.host,
|
||||
})
|
||||
queryText = queries?.[0] || `Welche Anbieter gibt es für ${clientLike.name}?`
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[demo-monitoring]', err?.message || err)
|
||||
return res.status(500).json({ error: 'Ein interner Fehler ist aufgetreten.' })
|
||||
}
|
||||
|
||||
const providers = getProviders()
|
||||
const provider = providers.openai
|
||||
const result = await provider.query(queryText, { brandHint: clientLike.name })
|
||||
const mention = result.error
|
||||
? { mentioned: false, position: null, snippet: null }
|
||||
: detectMention(result.content, clientLike)
|
||||
|
||||
const truncated = (result.content || '').slice(0, 500)
|
||||
res.json({
|
||||
provider: result.provider,
|
||||
query: queryText,
|
||||
response: truncated,
|
||||
truncated: (result.content || '').length > 500,
|
||||
mentioned: mention.mentioned,
|
||||
snippet: mention.snippet,
|
||||
error: result.error || null,
|
||||
})
|
||||
})
|
||||
|
||||
export default router
|
||||
Reference in New Issue
Block a user