Initial commit: Visigine (Vite client + Express/SQLite backend)
Container-ready via docker/ compose (frontend nginx + backend Node). Compose adjusted for Coolify on the prod server: frontend uses expose:80 (no host binding — host 8080 is taken by the Coolify proxy; Traefik routes visigine.de), backend ALLOWED_ORIGINS=https://visigine.de. Secrets stay in server/.env (git-ignored); see server/.env.example. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
131
server/lib/fetcher.js
Normal file
131
server/lib/fetcher.js
Normal file
@@ -0,0 +1,131 @@
|
||||
import { lookup } from 'node:dns/promises'
|
||||
import net from 'node:net'
|
||||
|
||||
const DEFAULT_UA =
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
|
||||
|
||||
// Blocks RFC 1918 private ranges, loopback, link-local, AWS metadata
|
||||
// (169.254.0.0/16), multicast/reserved, and IPv6 equivalents.
|
||||
function isPrivateOrLoopback(addr) {
|
||||
if (net.isIPv4(addr)) {
|
||||
const p = addr.split('.').map(Number)
|
||||
return (
|
||||
p[0] === 0 ||
|
||||
p[0] === 10 ||
|
||||
p[0] === 127 ||
|
||||
(p[0] === 169 && p[1] === 254) ||
|
||||
(p[0] === 172 && p[1] >= 16 && p[1] <= 31) ||
|
||||
(p[0] === 192 && p[1] === 168) ||
|
||||
p[0] >= 224
|
||||
)
|
||||
}
|
||||
if (net.isIPv6(addr)) {
|
||||
const a = addr.toLowerCase()
|
||||
return (
|
||||
a === '::' ||
|
||||
a === '::1' ||
|
||||
a.startsWith('fe80:') ||
|
||||
a.startsWith('fc') ||
|
||||
a.startsWith('fd') ||
|
||||
a.startsWith('::ffff:')
|
||||
)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Resolves hostname; throws { code: 'PRIVATE_HOST_BLOCKED' | 'ENOTFOUND' }.
|
||||
// Skipped entirely when ALLOW_PRIVATE_HOSTS=1 (local dev).
|
||||
async function assertPublicHost(hostname) {
|
||||
if (process.env.ALLOW_PRIVATE_HOSTS === '1') return
|
||||
let address
|
||||
try {
|
||||
({ address } = await lookup(hostname))
|
||||
} catch (err) {
|
||||
const e = new Error('ENOTFOUND')
|
||||
e.code = 'ENOTFOUND'
|
||||
e.cause = err
|
||||
throw e
|
||||
}
|
||||
if (isPrivateOrLoopback(address)) {
|
||||
const e = new Error('PRIVATE_HOST_BLOCKED')
|
||||
e.code = 'PRIVATE_HOST_BLOCKED'
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// Returns { status, headers, body, finalUrl, ms, error? } — never throws.
|
||||
// `error` is a string code: 'PRIVATE_HOST_BLOCKED' | 'ENOTFOUND' | 'TIMEOUT' | 'TLS_INVALID' | 'NETWORK'.
|
||||
export async function fetchPage(url, { userAgent, timeoutMs = 10000 } = {}) {
|
||||
const started = Date.now()
|
||||
let hostname
|
||||
try {
|
||||
hostname = new URL(url).hostname
|
||||
} catch {
|
||||
return { status: 0, headers: {}, body: '', finalUrl: url, ms: 0, error: 'INVALID_URL' }
|
||||
}
|
||||
|
||||
try {
|
||||
await assertPublicHost(hostname)
|
||||
} catch (err) {
|
||||
return {
|
||||
status: 0,
|
||||
headers: {},
|
||||
body: '',
|
||||
finalUrl: url,
|
||||
ms: Date.now() - started,
|
||||
error: err.code || 'NETWORK',
|
||||
}
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
method: 'GET',
|
||||
redirect: 'follow',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'User-Agent': userAgent || DEFAULT_UA,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
|
||||
'Cache-Control': 'no-cache',
|
||||
},
|
||||
})
|
||||
const body = await res.text()
|
||||
const headers = {}
|
||||
res.headers.forEach((value, key) => { headers[key.toLowerCase()] = value })
|
||||
return {
|
||||
status: res.status,
|
||||
headers,
|
||||
body,
|
||||
finalUrl: res.url || url,
|
||||
ms: Date.now() - started,
|
||||
}
|
||||
} catch (err) {
|
||||
const causeCode = err?.cause?.code
|
||||
const causeMessage = err?.cause?.message || ''
|
||||
const TLS_CODES = new Set([
|
||||
'ERR_TLS_CERT_ALTNAME_INVALID',
|
||||
'CERT_HAS_EXPIRED',
|
||||
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
|
||||
'SELF_SIGNED_CERT_IN_CHAIN',
|
||||
'DEPTH_ZERO_SELF_SIGNED_CERT',
|
||||
'ERR_SSL_WRONG_VERSION_NUMBER',
|
||||
])
|
||||
let code = 'NETWORK'
|
||||
if (err?.name === 'AbortError') code = 'TIMEOUT'
|
||||
else if (TLS_CODES.has(causeCode) || /certificate|altnames|self.?signed|TLS|SSL/i.test(causeMessage)) code = 'SSL_INVALID'
|
||||
else if (causeCode === 'ENOTFOUND' || causeCode === 'EAI_AGAIN') code = 'ENOTFOUND'
|
||||
else if (causeCode === 'ETIMEDOUT') code = 'TIMEOUT'
|
||||
return {
|
||||
status: 0,
|
||||
headers: {},
|
||||
body: '',
|
||||
finalUrl: url,
|
||||
ms: Date.now() - started,
|
||||
error: code,
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user