From 2ef4b1ebac511cae65d80dd2688d433703b74af0 Mon Sep 17 00:00:00 2001
From: Ihor_Zhekov <ihor@profice.ai>
Date: Wed, 24 Jun 2026 12:00:43 +0200
Subject: [PATCH] fix(nginx): serve .txt as UTF-8 and make AI-discovery files
 reachable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

llms.txt and robots.txt were served without a charset, so browsers
decoded the valid UTF-8 as Windows-1252 and showed mojibake (â€", für).

- Add `charset utf-8;` (+ charset_types) so text responses carry
  `; charset=utf-8`.
- Add an explicit `location ~* \.txt$` that serves the file as plain
  text, returns 404 instead of falling back to index.html, and sets
  `Access-Control-Allow-Origin: *` so any AI crawler/tool can fetch
  llms.txt cross-origin.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 nginx.conf | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/nginx.conf b/nginx.conf
index 73deeb1..f889dc8 100644
--- a/nginx.conf
+++ b/nginx.conf
@@ -4,6 +4,21 @@ server {
     root /usr/share/nginx/html;
     index index.html;
 
+    # Serve text responses as UTF-8. Without this nginx omits the charset and
+    # browsers fall back to a locale guess (Windows-1252 on DE systems), which
+    # turns valid UTF-8 in llms.txt / robots.txt into mojibake (â€", für).
+    charset utf-8;
+    charset_types text/plain text/css text/xml application/javascript application/json image/svg+xml;
+
+    # AI-discovery files (llms.txt, robots.txt): always served as UTF-8 plain
+    # text, never swallowed by the clean-URL fallback, and readable cross-origin
+    # so any AI crawler or tool can fetch them.
+    location ~* \.txt$ {
+        try_files $uri =404;
+        default_type text/plain;
+        add_header Access-Control-Allow-Origin "*" always;
+        add_header Cache-Control "public, max-age=3600" always;
+    }
 
     # Serve clean URLs by trying $uri, then $uri.html
     location / {