From 376aef3fce0847ec038eb619d8ce8596d5438fe2 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sat, 25 Jul 2026 11:27:29 +0200
Subject: [PATCH 01/53] stop creating doubles of same doctype, correspondent or
 tag.

---
 routes/setup.js                 |  169 +---
 server.js                       |  879 +++++------------
 services/dataProcessingUtils.js |  123 +++
 services/mistralOcrService.js   |   84 +-
 services/ollamaService.js       | 1548 ++++++++++++++----------------
 services/paperlessService.js    | 1564 ++++++++++---------------------
 views/settings.ejs              |    3 +-
 7 files changed, 1698 insertions(+), 2672 deletions(-)
 create mode 100644 services/dataProcessingUtils.js

diff --git a/routes/setup.js b/routes/setup.js
index 90def938..1730c34d 100644
--- a/routes/setup.js
+++ b/routes/setup.js
@@ -17,7 +17,6 @@ const path = require('path');
 const crypto = require('crypto');
 const {
   validateApiUrl,
-  validateCustomFieldValue,
   shouldQueueForOcrOnAiError,
   classifyOcrQueueReasonFromAiError,
   stripTrailingSlashes,
@@ -29,6 +28,11 @@ const QRCode = require('qrcode');
 const { isAuthenticated } = require('./auth.js');
 const customService = require('../services/customService.js');
 const mistralOcrService = require('../services/mistralOcrService');
+const {
+  updateCustomFieldsData,
+  updateDocumentTypeData,
+  updateCorrespondentData,
+} = require('../services/dataProcessingUtils');
 const quickstartService = require('../services/quickstartService');
 const reconciliationService = require('../services/reconciliationService');
 const scanHealthService = require('../services/scanHealthService');
@@ -3263,116 +3267,9 @@ async function buildUpdateData(analysis, doc) {
   // Add created date regardless of settings as it's a core field
   updateData.created = analysis.document.document_date || doc.created;
 
-  // Only process document type if document type classification is activated
-  if (
-    config.limitFunctions?.activateDocumentType !== 'no' &&
-    analysis.document.document_type
-  ) {
-    try {
-      const documentType = await paperlessService.getOrCreateDocumentType(
-        analysis.document.document_type,
-        options
-      );
-      if (documentType) {
-        updateData.document_type = documentType.id;
-      }
-    } catch (error) {
-      console.error(`[ERROR] Error processing document type:`, error);
-    }
-  }
-
-  // Only process custom fields if custom fields detection is activated
-  if (
-    config.limitFunctions?.activateCustomFields !== 'no' &&
-    analysis.document.custom_fields
-  ) {
-    const customFields = analysis.document.custom_fields;
-    const processedFields = [];
-    const customFieldsForHistory = [];
-
-    // Get existing custom fields
-    const existingFields = await paperlessService.getExistingCustomFields(
-      doc.id
-    );
-    console.log(`[DEBUG] Found existing fields:`, existingFields);
-
-    // Keep track of which fields we've processed to avoid duplicates
-    const processedFieldIds = new Set();
-
-    // First, add any new/updated fields
-    for (const customField of Object.values(customFields)) {
-      if (!customField || typeof customField !== 'object') {
-        console.log('[DEBUG] Skipping null/invalid custom field entry');
-        continue;
-      }
-
-      if (
-        !customField.field_name ||
-        customField.value === null ||
-        customField.value === undefined ||
-        String(customField.value).trim() === ''
-      ) {
-        console.log(`[DEBUG] Skipping empty/invalid custom field`);
-        continue;
-      }
-
-      const fieldDetails = await paperlessService.findExistingCustomField(
-        customField.field_name
-      );
-      if (fieldDetails?.id) {
-        const validation = validateCustomFieldValue(
-          customField.field_name,
-          customField.value,
-          fieldDetails.data_type
-        );
-        if (validation.skip) {
-          if (validation.warn) console.warn(validation.warn);
-          continue;
-        }
-        processedFields.push({
-          field: fieldDetails.id,
-          value: validation.value,
-        });
-        customFieldsForHistory.push({
-          field_name: customField.field_name,
-          value: validation.value,
-        });
-        processedFieldIds.add(fieldDetails.id);
-      }
-    }
-
-    // Then add any existing fields that weren't updated
-    for (const existingField of existingFields) {
-      if (!processedFieldIds.has(existingField.field)) {
-        processedFields.push(existingField);
-      }
-    }
-
-    if (processedFields.length > 0) {
-      updateData.custom_fields = processedFields;
-    }
-    if (customFieldsForHistory.length > 0) {
-      updateData._customFieldsForHistory = customFieldsForHistory;
-    }
-  }
-
-  // Only process correspondent if correspondent detection is activated
-  if (
-    config.limitFunctions?.activateCorrespondents !== 'no' &&
-    analysis.document.correspondent
-  ) {
-    try {
-      const correspondent = await paperlessService.getOrCreateCorrespondent(
-        analysis.document.correspondent,
-        options
-      );
-      if (correspondent) {
-        updateData.correspondent = correspondent.id;
-      }
-    } catch (error) {
-      console.error(`[ERROR] Error processing correspondent:`, error);
-    }
-  }
+  await updateDocumentTypeData(analysis, updateData, config, paperlessService, options);
+  await updateCustomFieldsData(analysis, doc, updateData, config, paperlessService);
+  await updateCorrespondentData(analysis, updateData, config, paperlessService, options);
 
   // Always include language if provided as it's a core field
   if (analysis.document.language) {
@@ -3397,16 +3294,30 @@ async function saveDocumentChanges(docId, updateData, analysis, originalData) {
   const origDocType = originalData.document_type ?? null;
   const origLanguage = originalData.language ?? null;
 
+  await documentModel.saveOriginalData(
+    docId,
+    originalTags,
+    originalCorrespondent,
+    originalTitle,
+    origDocType,
+    origLanguage
+  );
+
+  const updatedDocument = await paperlessService.updateDocument(docId, updateData);
+  if (!updatedDocument) {
+    throw new Error(`Paperless update failed for document ${docId}`);
+  }
+
+  const currentCorrespondentId = updatedDocument.correspondent ?? updateData.correspondent;
+  const currentCorrespondent = currentCorrespondentId
+    ? await paperlessService.getCorrespondentNameById(currentCorrespondentId)
+    : null;
+  const currentDocumentTypeId = updatedDocument.document_type ?? updateData.document_type;
+  const currentDocumentType = currentDocumentTypeId
+    ? await paperlessService.getDocumentTypeNameById(currentDocumentTypeId)
+    : null;
+
   await Promise.all([
-    documentModel.saveOriginalData(
-      docId,
-      originalTags,
-      originalCorrespondent,
-      originalTitle,
-      origDocType,
-      origLanguage
-    ),
-    paperlessService.updateDocument(docId, updateData),
     documentModel.addProcessedDocument(docId, updateData.title),
     documentModel.addOpenAIMetrics(
       docId,
@@ -3418,9 +3329,9 @@ async function saveDocumentChanges(docId, updateData, analysis, originalData) {
       docId,
       updateData.tags,
       updateData.title,
-      analysis.document.correspondent,
+      currentCorrespondent?.name ?? null,
       historyCustomFields,
-      historyDocTypeName,
+      currentDocumentType?.name ?? null,
       historyLanguage
     ),
   ]);
@@ -8644,10 +8555,6 @@ router.post('/settings', express.json(), async (req, res) => {
       ...updatedConfig,
     };
 
-    // The route has already validated the submitted values (and Paperless
-    // reachability when its settings changed). Skip saveConfig's built-in
-    // validateConfig so saving does not live-test Paperless/AI on every save.
-    await setupService.saveConfig(mergedConfig, { skipValidation: true });
     try {
       for (const field of processedCustomFields) {
         await paperlessService.createCustomFieldSafely(
@@ -8657,9 +8564,17 @@ router.post('/settings', express.json(), async (req, res) => {
         );
       }
     } catch (error) {
-      console.log('[ERROR] Error creating custom fields:', error);
+      if (error.code === 'CUSTOM_FIELD_TYPE_MISMATCH') {
+        return res.status(400).json({ error: error.message });
+      }
+      throw error;
     }
 
+    // The route has already validated the submitted values (and Paperless
+    // reachability when its settings changed). Skip saveConfig's built-in
+    // validateConfig so saving does not live-test Paperless/AI on every save.
+    await setupService.saveConfig(mergedConfig, { skipValidation: true });
+
     res.json({
       success: true,
       message: 'Configuration saved successfully.',
diff --git a/server.js b/server.js
index 93ae6d4a..10ea9412 100644
--- a/server.js
+++ b/server.js
@@ -13,10 +13,7 @@ const { runStartupMigrations } = require('./services/startupMigrations');
 const setupRoutes = require('./routes/setup');
 const { isAuthenticated } = require('./routes/auth');
 const mistralOcrService = require('./services/mistralOcrService');
-const ocrAutoProcessService = require('./services/ocrAutoProcessService');
 const reconciliationService = require('./services/reconciliationService');
-const scanHealthService = require('./services/scanHealthService');
-const { RUN_STATUS } = scanHealthService;
 const cors = require('cors');
 const cookieParser = require('cookie-parser');
 const { doubleCsrf } = require('csrf-csrf');
@@ -25,11 +22,15 @@ const { ipKeyGenerator } = require('express-rate-limit');
 const jwt = require('jsonwebtoken');
 const Logger = require('./services/loggerService');
 const {
-  validateCustomFieldValue,
+  updateCustomFieldsData,
+  updateDocumentTypeData,
+  updateCorrespondentData,
+} = require('./services/dataProcessingUtils');
+const {
   shouldQueueForOcrOnAiError,
   classifyOcrQueueReasonFromAiError,
   isTimeoutError,
-  buildTimeoutErrorMessage,
+  buildTimeoutErrorMessage
 } = require('./services/serviceUtils');
 const dataDir = path.join(process.cwd(), 'data');
 const openApiDir = path.join(dataDir, 'OPENAPI');
@@ -41,7 +42,7 @@ const htmlLogger = new Logger({
   logDir: dataLogsDir,
   format: 'html',
   timestamp: true,
-  maxFileSize: 1024 * 1024 * 10,
+  maxFileSize: 1024 * 1024 * 10
 });
 
 const txtLogger = new Logger({
@@ -49,7 +50,7 @@ const txtLogger = new Logger({
   logDir: dataLogsDir,
   format: 'txt',
   timestamp: true,
-  maxFileSize: 1024 * 1024 * 10,
+  maxFileSize: 1024 * 1024 * 10
 });
 
 const app = express();
@@ -63,7 +64,7 @@ const scanControl = global.__paperlessAiScanControl || {
   stopRequested: false,
   source: null,
   startedAt: null,
-  stopRequestedAt: null,
+  stopRequestedAt: null
 };
 global.__paperlessAiScanControl = scanControl;
 
@@ -83,14 +84,12 @@ async function triggerScanNow(source = 'manual') {
       started: false,
       running: true,
       stopRequested: scanControl.stopRequested,
-      message: 'Scan is already running.',
+      message: 'Scan is already running.'
     };
   }
 
   scanDocuments(source).catch((error) => {
-    console.error(
-      `[ERROR] scanDocuments() failed in triggerScanNow: ${error.message}`
-    );
+    console.error(`[ERROR] scanDocuments() failed in triggerScanNow: ${error.message}`);
     console.debug(error);
   });
 
@@ -98,7 +97,7 @@ async function triggerScanNow(source = 'manual') {
     started: true,
     running: true,
     stopRequested: false,
-    message: 'Scan started.',
+    message: 'Scan started.'
   };
 }
 
@@ -108,10 +107,7 @@ global.__paperlessAiRequestScanStop = requestScanStop;
 function persistJwtSecret(secret) {
   const runtimeDataDir = path.join(process.cwd(), 'data');
   const envFilePath = path.join(runtimeDataDir, '.env');
-  const runtimeOverridesPath = path.join(
-    runtimeDataDir,
-    'runtime-overrides.json'
-  );
+  const runtimeOverridesPath = path.join(runtimeDataDir, 'runtime-overrides.json');
 
   try {
     fsSync.mkdirSync(runtimeDataDir, { recursive: true });
@@ -125,23 +121,15 @@ function persistJwtSecret(secret) {
     let updatedEnvContent = envContent;
 
     if (hasJwtSecretLine) {
-      updatedEnvContent = envContent.replace(
-        /^\s*JWT_SECRET\s*=.*$/m,
-        `JWT_SECRET=${secret}`
-      );
+      updatedEnvContent = envContent.replace(/^\s*JWT_SECRET\s*=.*$/m, `JWT_SECRET=${secret}`);
     } else {
       const trimmed = envContent.trimEnd();
-      updatedEnvContent = trimmed
-        ? `${trimmed}\nJWT_SECRET=${secret}\n`
-        : `JWT_SECRET=${secret}\n`;
+      updatedEnvContent = trimmed ? `${trimmed}\nJWT_SECRET=${secret}\n` : `JWT_SECRET=${secret}\n`;
     }
 
     fsSync.writeFileSync(envFilePath, updatedEnvContent, 'utf8');
   } catch (error) {
-    console.warn(
-      '[WARN] Could not persist generated JWT_SECRET to data/.env:',
-      error.message
-    );
+    console.warn('[WARN] Could not persist generated JWT_SECRET to data/.env:', error.message);
   }
 
   try {
@@ -154,17 +142,10 @@ function persistJwtSecret(secret) {
 
     if (!parsed.JWT_SECRET || String(parsed.JWT_SECRET).trim() === '') {
       parsed.JWT_SECRET = secret;
-      fsSync.writeFileSync(
-        runtimeOverridesPath,
-        JSON.stringify(parsed, null, 2),
-        'utf8'
-      );
+      fsSync.writeFileSync(runtimeOverridesPath, JSON.stringify(parsed, null, 2), 'utf8');
     }
   } catch (error) {
-    console.warn(
-      '[WARN] Could not update JWT_SECRET in runtime-overrides.json:',
-      error.message
-    );
+    console.warn('[WARN] Could not update JWT_SECRET in runtime-overrides.json:', error.message);
   }
 }
 
@@ -178,18 +159,14 @@ function ensureJwtSecret() {
   process.env.JWT_SECRET = generatedSecret;
   persistJwtSecret(generatedSecret);
 
-  console.warn(
-    '[WARN] JWT_SECRET was missing. Generated and persisted a new secret. Existing sessions may require re-login.'
-  );
+  console.warn('[WARN] JWT_SECRET was missing. Generated and persisted a new secret. Existing sessions may require re-login.');
   return generatedSecret;
 }
 
 const JWT_SECRET = ensureJwtSecret();
 
 if (!JWT_SECRET) {
-  console.error(
-    'JWT_SECRET environment variable is not set. Refusing to start without a secure JWT secret.'
-  );
+  console.error('JWT_SECRET environment variable is not set. Refusing to start without a secure JWT secret.');
   process.exit(1);
 }
 
@@ -201,9 +178,7 @@ if (trustProxy !== false) {
 function getCookieSecureMode() {
   return typeof config.getCookieSecureMode === 'function'
     ? config.getCookieSecureMode()
-    : String(process.env.COOKIE_SECURE_MODE || 'auto')
-        .trim()
-        .toLowerCase();
+    : String(process.env.COOKIE_SECURE_MODE || 'auto').trim().toLowerCase();
 }
 
 function shouldUseSecureCookies(req) {
@@ -244,15 +219,16 @@ const retryTracker = new Map();
 // Configurable minimum content length (default: 10 characters)
 const MIN_CONTENT_LENGTH = config.minContentLength;
 
+
 const corsOptions = {
   origin: true,
   methods: ['GET', 'POST', 'OPTIONS'],
   allowedHeaders: [
-    'Content-Type',
+    'Content-Type', 
     'x-api-key',
-    'Access-Control-Allow-Private-Network',
+    'Access-Control-Allow-Private-Network'
   ],
-  credentials: false,
+  credentials: false
 };
 
 const apiGlobalLimiter = rateLimit({
@@ -260,7 +236,7 @@ const apiGlobalLimiter = rateLimit({
   max: config.globalRateLimitMax,
   message: {
     success: false,
-    error: 'Too many requests. Please try again later.',
+    error: 'Too many requests. Please try again later.'
   },
   standardHeaders: true,
   legacyHeaders: false,
@@ -275,18 +251,17 @@ const apiGlobalLimiter = rateLimit({
     if (token) {
       try {
         const decoded = jwt.verify(token, JWT_SECRET);
-        const userIdentifier =
-          decoded?.id || decoded?.userId || decoded?.username || decoded?.sub;
+        const userIdentifier = decoded?.id || decoded?.userId || decoded?.username || decoded?.sub;
         if (userIdentifier) {
           return `user:${userIdentifier}`;
         }
-      } catch {
+      } catch (error) {
         // Ignore invalid token and fallback to IP
       }
     }
 
     return ipKeyGenerator(req.ip);
-  },
+  }
 });
 
 app.use(cors(corsOptions));
@@ -308,18 +283,15 @@ app.use((req, res, next) => {
   res.locals.theme = resolvedTheme;
   res.locals.appVersion = config.PAPERLESS_AI_VERSION || 'unknown';
   res.locals.appCommitSha = process.env.PAPERLESS_AI_COMMIT_SHA || 'unknown';
-  res.locals.appPaperlessNgxVersion =
-    process.env.PAPERLESS_NGX_VERSION || 'unknown';
-  res.locals.appAiProvider =
-    config.aiProvider || process.env.AI_PROVIDER || 'openai';
+  res.locals.appPaperlessNgxVersion = process.env.PAPERLESS_NGX_VERSION || 'unknown';
+  res.locals.appAiProvider = config.aiProvider || process.env.AI_PROVIDER || 'openai';
   res.locals.appOcrEnabled = config.mistralOcr?.enabled === 'yes';
   res.locals.appOcrProvider = config.mistralOcr?.provider || 'mistral';
   res.locals.appNodeEnv = process.env.NODE_ENV || 'production';
   res.locals.appNodeVersion = process.version;
   res.locals.appPlatform = `${process.platform} (${process.arch})`;
   res.locals.appServerTimeUtc = new Date().toISOString();
-  res.locals.appServerTimezone =
-    Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
+  res.locals.appServerTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
   res.locals.appPaperlessApiUrl = config.paperless?.apiUrl || 'unknown';
   res.locals.appOllamaApiUrl = config.ollama?.apiUrl || 'unknown';
   res.locals.appOllamaModel = config.ollama?.model || 'unknown';
@@ -335,10 +307,8 @@ app.use((req, res, next) => {
   res.locals.appTrustProxy = String(config.trustProxy);
   res.locals.appUseExistingData = config.useExistingData || 'no';
   res.locals.appRestrictTags = config.restrictToExistingTags || 'no';
-  res.locals.appRestrictCorrespondents =
-    config.restrictToExistingCorrespondents || 'no';
-  res.locals.appRestrictDocumentTypes =
-    config.restrictToExistingDocumentTypes || 'no';
+  res.locals.appRestrictCorrespondents = config.restrictToExistingCorrespondents || 'no';
+  res.locals.appRestrictDocumentTypes = config.restrictToExistingDocumentTypes || 'no';
   res.locals.appPaperlessTokenSet = Boolean(config.paperless?.apiToken);
   res.locals.appOpenAiKeySet = Boolean(config.openai?.apiKey);
   res.locals.appCustomKeySet = Boolean(config.custom?.apiKey);
@@ -348,49 +318,49 @@ app.use((req, res, next) => {
   res.locals.loginCookieSecurityWarning = null;
 
   if (req.path === '/login' && csrfCookieSecure && !isHttpsRequest(req)) {
-    res.locals.loginCookieSecurityWarning =
-      'You are accessing the login page over HTTP while the system is configured to use HTTPS by default. To resolve this, either switch to HTTPS or set COOKIE_SECURE_MODE=never in your .env or docker-compose.yml file and restart the container.';
+    res.locals.loginCookieSecurityWarning = 'You are accessing the login page over HTTP while the system is configured to use HTTPS by default. To resolve this, either switch to HTTPS or set COOKIE_SECURE_MODE=never in your .env or docker-compose.yml file and restart the container.';
   }
 
   next();
 });
 
 // CSRF Protection configuration
-const { invalidCsrfTokenError, generateCsrfToken, doubleCsrfProtection } =
-  doubleCsrf({
-    getSecret: () => JWT_SECRET,
-    getSessionIdentifier: (req) => {
-      const token =
-        req.cookies?.jwt || req.headers.authorization?.split(' ')[1];
-      if (token) {
-        return `jwt:${token}`;
-      }
+const {
+  invalidCsrfTokenError,
+  generateCsrfToken,
+  doubleCsrfProtection,
+} = doubleCsrf({
+  getSecret: () => JWT_SECRET,
+  getSessionIdentifier: (req) => {
+    const token = req.cookies?.jwt || req.headers.authorization?.split(' ')[1];
+    if (token) {
+      return `jwt:${token}`;
+    }
 
-      const apiKey = req.headers['x-api-key'];
-      const currentApiKey = config.getApiKey();
-      if (currentApiKey && apiKey && apiKey === currentApiKey) {
-        return `api-key:${apiKey}`;
-      }
+    const apiKey = req.headers['x-api-key'];
+    const currentApiKey = config.getApiKey();
+    if (currentApiKey && apiKey && apiKey === currentApiKey) {
+      return `api-key:${apiKey}`;
+    }
 
-      return `ip:${req.ip || 'unknown'}`;
-    },
-    cookieName: 'psai.x-csrf-token',
-    cookieOptions: {
-      sameSite: 'lax',
-      path: '/',
-      secure: csrfCookieSecure,
-    },
-    size: 64,
-    ignoredMethods: ['GET', 'HEAD', 'OPTIONS'],
-    getCsrfTokenFromRequest: (req) =>
-      req.headers['x-csrf-token'] || req.body._csrf,
-  });
+    return `ip:${req.ip || 'unknown'}`;
+  },
+  cookieName: "psai.x-csrf-token",
+  cookieOptions: {
+    sameSite: "lax",
+    path: "/",
+    secure: csrfCookieSecure,
+  },
+  size: 64,
+  ignoredMethods: ["GET", "HEAD", "OPTIONS"],
+  getCsrfTokenFromRequest: (req) => req.headers["x-csrf-token"] || req.body._csrf,
+});
 
 // Middleware to skip CSRF for API Key authenticated requests and provide token to EJS
 app.use((req, res, next) => {
   const apiKey = req.headers['x-api-key'];
   const currentApiKey = config.getApiKey();
-
+  
   // If API Key is valid, skip CSRF
   if (currentApiKey && apiKey && apiKey === currentApiKey) {
     return next();
@@ -401,8 +371,7 @@ app.use((req, res, next) => {
     if (err) {
       if (err === invalidCsrfTokenError) {
         if (req.method === 'POST' && req.path === '/login') {
-          const baseError =
-            'Invalid CSRF token. The login page may have expired or your browser did not send the CSRF cookie.';
+          const baseError = 'Invalid CSRF token. The login page may have expired or your browser did not send the CSRF cookie.';
           const guidance = res.locals.loginCookieSecurityWarning
             ? ' This is commonly caused by HTTP access with secure cookies enabled. Set COOKIE_SECURE_MODE=never for local HTTP and restart, or switch to HTTPS. See: https://zettelrob.be/getting-started/configuration/#cookie-and-proxy-flags-all-supported-values'
             : ' Refresh the login page and try again.';
@@ -410,15 +379,15 @@ app.use((req, res, next) => {
           return res.status(403).render('login', {
             error: `${baseError}${guidance}`,
             mfaRequired: false,
-            username: String(req.body?.username || ''),
+            username: String(req.body?.username || '')
           });
         }
 
-        return res.status(403).json({ error: 'Invalid CSRF token' });
+        return res.status(403).json({ error: "Invalid CSRF token" });
       }
       return next(err);
     }
-
+    
     // Make CSRF token available to EJS templates
     res.locals.csrfToken = generateCsrfToken(req, res);
     next();
@@ -435,16 +404,11 @@ if (isApiDocsEnabled) {
   swaggerSpec = require('./swagger');
 
   // Swagger documentation route (protected)
-  app.use(
-    '/api-docs',
-    isAuthenticated,
-    swaggerUi.serve,
-    swaggerUi.setup(swaggerSpec, {
-      swaggerOptions: {
-        url: '/api-docs/openapi.json',
-      },
-    })
-  );
+  app.use('/api-docs', isAuthenticated, swaggerUi.serve, swaggerUi.setup(swaggerSpec, {
+    swaggerOptions: {
+      url: '/api-docs/openapi.json'
+    }
+  }));
 
   /**
    * @swagger
@@ -455,7 +419,7 @@ if (isApiDocsEnabled) {
    *       Returns the complete OpenAPI specification for the Zettelrobbe API.
    *       This endpoint attempts to serve a static OpenAPI JSON file first, falling back
    *       to dynamically generating the specification if the file cannot be read.
-   *
+   *       
    *       The OpenAPI specification document contains all API endpoints, parameters,
    *       request bodies, responses, and schemas for the entire application.
    *     tags: [API, System]
@@ -489,17 +453,14 @@ if (isApiDocsEnabled) {
    */
   app.get('/api-docs/openapi.json', isAuthenticated, (req, res) => {
     res.setHeader('Content-Type', 'application/json');
-
+    
     // Try to serve the static file first
     fs.readFile(openApiPath)
-      .then((data) => {
+      .then(data => {
         res.send(JSON.parse(data));
       })
-      .catch((err) => {
-        console.warn(
-          'Error reading OpenAPI file, generating dynamically:',
-          err.message
-        );
+      .catch(err => {
+        console.warn('Error reading OpenAPI file, generating dynamically:', err.message);
         // Fallback to generating the spec if file can't be read
         res.send(swaggerSpec);
       });
@@ -543,6 +504,7 @@ app.set('views', path.join(__dirname, 'views'));
 //   next();
 // });
 
+
 // Initialize data directory
 async function initializeDataDirectory() {
   try {
@@ -567,7 +529,7 @@ async function saveOpenApiSpec() {
       console.log('Creating OPENAPI directory...');
       await fs.mkdir(openApiDir, { recursive: true });
     }
-
+    
     // Write the specification to file
     await fs.writeFile(openApiPath, JSON.stringify(swaggerSpec, null, 2));
     console.log(`OpenAPI specification saved to ${openApiPath}`);
@@ -580,41 +542,28 @@ async function saveOpenApiSpec() {
 }
 
 // Document processing functions
-async function processDocument(
-  doc,
-  existingTags,
-  existingCorrespondentList,
-  existingDocumentTypesList
-) {
+async function processDocument(doc, existingTags, existingCorrespondentList, existingDocumentTypesList, ownUserId) {
   const isProcessed = await documentModel.isDocumentProcessed(doc.id);
   if (isProcessed) return null;
 
   const isIgnored = await documentModel.isDocumentIgnored(doc.id);
   if (isIgnored) {
-    console.debug(
-      `Document ${doc.id} is marked as ignored, skipping permanently`
-    );
+    console.debug(`Document ${doc.id} is marked as ignored, skipping permanently`);
     return null;
   }
 
   const isFailed = await documentModel.isDocumentFailed(doc.id);
   if (isFailed) {
-    console.debug(
-      `Document ${doc.id} is marked as permanently failed, skipping until reset`
-    );
+    console.debug(`Document ${doc.id} is marked as permanently failed, skipping until reset`);
     return null;
   }
 
   await documentModel.setProcessingStatus(doc.id, doc.title, 'processing');
 
   // Check if the document can be edited.
-  const documentEditable = await paperlessService.getPermissionOfDocument(
-    doc.id
-  );
+  const documentEditable = await paperlessService.getPermissionOfDocument(doc.id);
   if (!documentEditable) {
-    console.debug(
-      `Document ${doc.id} is not editable by the Zettelrobbe user, skipping analysis`
-    );
+    console.debug(`Document ${doc.id} is not editable by the Zettelrobbe user, skipping analysis`);
     return null;
   }
 
@@ -622,33 +571,20 @@ async function processDocument(
 
   let [content, originalData] = await Promise.all([
     paperlessService.getDocumentContent(doc.id),
-    paperlessService.getDocument(doc.id),
+    paperlessService.getDocument(doc.id)
   ]);
 
   if (!content || content.length < MIN_CONTENT_LENGTH) {
-    console.debug(
-      `Document ${doc.id} has insufficient content (${content?.length || 0} chars, minimum: ${MIN_CONTENT_LENGTH}), skipping analysis`
-    );
+    console.debug(`Document ${doc.id} has insufficient content (${content?.length || 0} chars, minimum: ${MIN_CONTENT_LENGTH}), skipping analysis`);
     // Queue for Mistral OCR if enabled.
     if (mistralOcrService.isEnabled()) {
-      const added = await documentModel.addToOcrQueue(
-        doc.id,
-        doc.title,
-        `short_content_lt_${MIN_CONTENT_LENGTH}`
-      );
+      const added = await documentModel.addToOcrQueue(doc.id, doc.title, `short_content_lt_${MIN_CONTENT_LENGTH}`);
       if (added) {
-        console.info(
-          `Document ${doc.id} queued for Mistral OCR (short_content)`
-        );
+        console.info(`Document ${doc.id} queued for Mistral OCR (short_content)`);
       }
     } else {
       await documentModel.setProcessingStatus(doc.id, doc.title, 'failed');
-      await documentModel.addFailedDocument(
-        doc.id,
-        doc.title,
-        `insufficient_content_lt_${MIN_CONTENT_LENGTH}`,
-        'ai'
-      );
+      await documentModel.addFailedDocument(doc.id, doc.title, `insufficient_content_lt_${MIN_CONTENT_LENGTH}`, 'ai');
       retryTracker.delete(doc.id);
     }
     return null;
@@ -657,9 +593,7 @@ async function processDocument(
   // Check retry limit to prevent infinite retry loops
   const docRetries = retryTracker.get(doc.id) || 0;
   if (docRetries >= 3) {
-    console.warn(
-      `Document ${doc.id} has failed ${docRetries} times, skipping to prevent infinite retry loop`
-    );
+    console.warn(`Document ${doc.id} has failed ${docRetries} times, skipping to prevent infinite retry loop`);
     await documentModel.setProcessingStatus(doc.id, doc.title, 'failed');
     retryTracker.delete(doc.id);
     return null;
@@ -670,13 +604,7 @@ async function processDocument(
   }
 
   const aiService = AIServiceFactory.getService();
-  const analysis = await aiService.analyzeDocument(
-    content,
-    existingTags,
-    existingCorrespondentList,
-    existingDocumentTypesList,
-    doc.id
-  );
+  const analysis = await aiService.analyzeDocument(content, existingTags, existingCorrespondentList, existingDocumentTypesList, doc.id);
   console.debug('Response from AI service:', analysis);
   if (analysis.error) {
     const aiErrorMessage = isTimeoutError(analysis.error)
@@ -690,42 +618,23 @@ async function processDocument(
     let queuedForOcr = false;
     let markedTerminalFailed = false;
     // Queue for Mistral OCR on OCR-relevant AI errors (e.g. low content, invalid response structure)
-    if (
-      mistralOcrService.isEnabled() &&
-      shouldQueueForOcrOnAiError(aiErrorMessage)
-    ) {
+    if (mistralOcrService.isEnabled() && shouldQueueForOcrOnAiError(aiErrorMessage)) {
       const queueReason = classifyOcrQueueReasonFromAiError(aiErrorMessage);
-      const added = await documentModel.addToOcrQueue(
-        doc.id,
-        doc.title,
-        queueReason
-      );
+      const added = await documentModel.addToOcrQueue(doc.id, doc.title, queueReason);
       if (added) {
-        console.log(
-          `[OCR] Document ${doc.id} queued for Mistral OCR (ai_failed: ${aiErrorMessage})`
-        );
+        console.log(`[OCR] Document ${doc.id} queued for Mistral OCR (ai_failed: ${aiErrorMessage})`);
       }
       queuedForOcr = true;
     }
 
     if (!mistralOcrService.isEnabled()) {
       await documentModel.setProcessingStatus(doc.id, doc.title, 'failed');
-      await documentModel.addFailedDocument(
-        doc.id,
-        doc.title,
-        'ai_failed_ocr_disabled',
-        'ai'
-      );
+      await documentModel.addFailedDocument(doc.id, doc.title, 'ai_failed_ocr_disabled', 'ai');
       retryTracker.delete(doc.id);
       markedTerminalFailed = true;
     } else if (!queuedForOcr) {
       await documentModel.setProcessingStatus(doc.id, doc.title, 'failed');
-      await documentModel.addFailedDocument(
-        doc.id,
-        doc.title,
-        'ai_failed_without_ocr_fallback',
-        'ai'
-      );
+      await documentModel.addFailedDocument(doc.id, doc.title, 'ai_failed_without_ocr_fallback', 'ai');
       retryTracker.delete(doc.id);
       markedTerminalFailed = true;
     }
@@ -746,36 +655,23 @@ async function buildUpdateData(analysis, doc) {
   const updateData = {};
   const options = {
     restrictToExistingTags: config.restrictToExistingTags === 'yes',
-    restrictToExistingCorrespondents:
-      config.restrictToExistingCorrespondents === 'yes',
-    restrictToExistingDocumentTypes:
-      config.restrictToExistingDocumentTypes === 'yes',
+    restrictToExistingCorrespondents: config.restrictToExistingCorrespondents === 'yes',
+    restrictToExistingDocumentTypes: config.restrictToExistingDocumentTypes === 'yes'
   };
 
   // Only process tags if tagging is activated
   if (config.limitFunctions?.activateTagging !== 'no') {
-    const { tagIds, errors } = await paperlessService.processTags(
-      analysis.document.tags,
-      options
-    );
+    const { tagIds, errors } = await paperlessService.processTags(analysis.document.tags, options);
     if (errors.length > 0) {
       console.warn('[ERROR] Some tags could not be processed:', errors);
     }
     updateData.tags = tagIds;
-  } else if (
-    config.limitFunctions?.activateTagging === 'no' &&
-    config.addAIProcessedTag === 'yes'
-  ) {
+  } else if (config.limitFunctions?.activateTagging === 'no' && config.addAIProcessedTag === 'yes') {
     // Add AI processed tags to the document (processTags function awaits a tags array)
     // get tags from .env file and split them by comma and make an array
-    console.debug(
-      'Tagging is deactivated but the AI processed tag will still be added'
-    );
+    console.debug('Tagging is deactivated but the AI processed tag will still be added');
     const tags = config.addAIProcessedTags.split(',');
-    const { tagIds, errors } = await paperlessService.processTags(
-      tags,
-      options
-    );
+    const { tagIds, errors } = await paperlessService.processTags(tags, options);
     if (errors.length > 0) {
       console.warn('[ERROR] Some tags could not be processed:', errors);
     }
@@ -791,116 +687,9 @@ async function buildUpdateData(analysis, doc) {
   // Add created date regardless of settings as it's a core field
   updateData.created = analysis.document.document_date || doc.created;
 
-  // Only process document type if document type classification is activated
-  if (
-    config.limitFunctions?.activateDocumentType !== 'no' &&
-    analysis.document.document_type
-  ) {
-    try {
-      const documentType = await paperlessService.getOrCreateDocumentType(
-        analysis.document.document_type,
-        options
-      );
-      if (documentType) {
-        updateData.document_type = documentType.id;
-      }
-    } catch (error) {
-      console.error(`[ERROR] Error processing document type: ${error.message}`);
-      console.debug(error);
-    }
-  }
-
-  // Only process custom fields if custom fields detection is activated
-  if (
-    config.limitFunctions?.activateCustomFields !== 'no' &&
-    analysis.document.custom_fields
-  ) {
-    const customFields = analysis.document.custom_fields;
-    const processedFields = [];
-    const customFieldsForHistory = [];
-
-    // Get existing custom fields
-    const existingFields = await paperlessService.getExistingCustomFields(
-      doc.id
-    );
-    console.debug('Found existing fields:', existingFields);
-
-    // Keep track of which fields we've processed to avoid duplicates
-    const processedFieldIds = new Set();
-
-    // First, add any new/updated fields
-    for (const key in customFields) {
-      const customField = customFields[key];
-
-      if (
-        !customField.field_name ||
-        customField.value === null ||
-        customField.value === undefined ||
-        String(customField.value).trim() === ''
-      ) {
-        console.debug('Skipping empty or invalid custom field');
-        continue;
-      }
-
-      const fieldDetails = await paperlessService.findExistingCustomField(
-        customField.field_name
-      );
-      if (fieldDetails?.id) {
-        const validation = validateCustomFieldValue(
-          customField.field_name,
-          customField.value,
-          fieldDetails.data_type
-        );
-        if (validation.skip) {
-          if (validation.warn) console.warn(validation.warn);
-          continue;
-        }
-        processedFields.push({
-          field: fieldDetails.id,
-          value: validation.value,
-        });
-        // Capture name + validated value for history at the point where we have both
-        customFieldsForHistory.push({
-          field_name: customField.field_name,
-          value: validation.value,
-        });
-        processedFieldIds.add(fieldDetails.id);
-      }
-    }
-
-    // Then add any existing fields that weren't updated
-    for (const existingField of existingFields) {
-      if (!processedFieldIds.has(existingField.field)) {
-        processedFields.push(existingField);
-      }
-    }
-
-    if (processedFields.length > 0) {
-      updateData.custom_fields = processedFields;
-    }
-    if (customFieldsForHistory.length > 0) {
-      updateData._customFieldsForHistory = customFieldsForHistory;
-    }
-  }
-
-  // Only process correspondent if correspondent detection is activated
-  if (
-    config.limitFunctions?.activateCorrespondents !== 'no' &&
-    analysis.document.correspondent
-  ) {
-    try {
-      const correspondent = await paperlessService.getOrCreateCorrespondent(
-        analysis.document.correspondent,
-        options
-      );
-      if (correspondent) {
-        updateData.correspondent = correspondent.id;
-      }
-    } catch (error) {
-      console.error(`[ERROR] Error processing correspondent: ${error.message}`);
-      console.debug(error);
-    }
-  }
+  await updateDocumentTypeData(analysis, updateData, config, paperlessService, options);
+  await updateCustomFieldsData(analysis, doc, updateData, config, paperlessService);
+  await updateCorrespondentData(analysis, updateData, config, paperlessService, options);
 
   // Always include language if provided as it's a core field
   if (analysis.document.language) {
@@ -911,34 +700,20 @@ async function buildUpdateData(analysis, doc) {
 }
 
 async function saveDocumentChanges(docId, updateData, analysis, originalData) {
-  const {
-    tags: originalTags,
-    correspondent: originalCorrespondent,
-    title: originalTitle,
-  } = originalData;
+  const { tags: originalTags, correspondent: originalCorrespondent, title: originalTitle } = originalData;
 
   // Pull out history-only data and remove it before sending updateData to Paperless
   const historyCustomFields = updateData._customFieldsForHistory || null;
   delete updateData._customFieldsForHistory;
 
   const historyDocTypeName = analysis.document.document_type ?? null;
-  const historyLanguage = analysis.document.language ?? null;
-  const origDocType = originalData.document_type ?? null;
-  const origLanguage = originalData.language ?? null;
-
-  await documentModel.saveOriginalData(
-    docId,
-    originalTags,
-    originalCorrespondent,
-    originalTitle,
-    origDocType,
-    origLanguage
-  );
-
-  const updatedDocument = await paperlessService.updateDocument(
-    docId,
-    updateData
-  );
+  const historyLanguage    = analysis.document.language ?? null;
+  const origDocType        = originalData.document_type ?? null;
+  const origLanguage       = originalData.language ?? null;
+
+  await documentModel.saveOriginalData(docId, originalTags, originalCorrespondent, originalTitle, origDocType, origLanguage);
+
+  const updatedDocument = await paperlessService.updateDocument(docId, updateData);
   if (!updatedDocument) {
     throw new Error(`Paperless update failed for document ${docId}`);
   }
@@ -953,7 +728,7 @@ async function saveDocumentChanges(docId, updateData, analysis, originalData) {
       historyCustomFields,
       historyDocTypeName,
       historyLanguage
-    ),
+    )
   ];
 
   if (analysis.metrics) {
@@ -970,9 +745,50 @@ async function saveDocumentChanges(docId, updateData, analysis, originalData) {
   await Promise.all(persistenceTasks);
 }
 
-// Main scanning function
-// The initial scan runs through here as well (source='initial') so it shares the
-// concurrency guard, the stop support and the health reporting below.
+// Main scanning functions
+async function scanInitial() {
+  try {
+    const isConfigured = await setupService.isConfigured();
+    if (!isConfigured) {
+      console.log('[ERROR] Setup not completed. Skipping document scan.');
+      return;
+    }
+
+    let [existingTags, documents, ownUserId, existingCorrespondentList, existingDocumentTypes] = await Promise.all([
+      paperlessService.getTags(),
+      paperlessService.getAllDocuments(),
+      paperlessService.getOwnUserID(),
+      paperlessService.listCorrespondentsNames(),
+      paperlessService.listDocumentTypesNames()
+    ]);
+    //get existing correspondent list
+    existingCorrespondentList = existingCorrespondentList.map(correspondent => correspondent.name);
+    let existingDocumentTypesList = existingDocumentTypes.map(docType => docType.name);
+    
+    // Extract tag names from tag objects
+    const existingTagNames = existingTags.map(tag => tag.name);
+
+    for (const doc of documents) {
+      try {
+        const result = await processDocument(doc, existingTagNames, existingCorrespondentList, existingDocumentTypesList, ownUserId);
+        if (!result) continue;
+
+        const { analysis, originalData } = result;
+        const updateData = await buildUpdateData(analysis, doc);
+        await saveDocumentChanges(doc.id, updateData, analysis, originalData);
+        await documentModel.setProcessingStatus(doc.id, doc.title, 'complete');
+      } catch (error) {
+        await documentModel.setProcessingStatus(doc.id, doc.title, 'failed');
+        console.error(`[ERROR] processing document ${doc.id}: ${error.message}`);
+        console.debug(error);
+      }
+    }
+  } catch (error) {
+    console.error(`[ERROR] during initial document scan: ${error.message}`);
+    console.debug(error);
+  }
+}
+
 async function scanDocuments(source = 'scheduler') {
   if (scanControl.running) {
     console.info('Scan request ignored because a task is already running');
@@ -986,8 +802,7 @@ async function scanDocuments(source = 'scheduler') {
     processed: 0,
     skipped: 0,
     failed: 0,
-    stopRequested: false,
-    abortReason: null,
+    stopRequested: false
   };
 
   scanControl.running = true;
@@ -995,55 +810,25 @@ async function scanDocuments(source = 'scheduler') {
   scanControl.source = source;
   scanControl.startedAt = new Date().toISOString();
   scanControl.stopRequestedAt = null;
-  scanHealthService.recordRunStart(source);
 
   console.info(`Scan started (source=${source})`);
 
   try {
-    // Probe first: the read helpers below swallow transport errors and return
-    // empty lists, which would make an unreachable Paperless-ngx look like a
-    // successful scan with nothing to do.
-    const connection = await paperlessService.checkConnection();
-    scanHealthService.recordConnectivity(connection);
-
-    if (!connection.reachable || !connection.authorized) {
-      scanStats.abortReason = connection.reachable
-        ? 'paperless_unauthorized'
-        : 'paperless_unreachable';
-      scanHealthService.recordRunResult({
-        status: RUN_STATUS.PAPERLESS_UNREACHABLE,
-        error: connection.error,
-      });
-      console.error(
-        `[ERROR] Scan aborted: Paperless-ngx is not usable (${connection.error}). ` +
-          `The scheduler stays armed and retries at the next interval (${config.scanInterval}).`
-      );
-      return;
-    }
-
-    let [
-      existingTags,
-      documents,
-      existingCorrespondentList,
-      existingDocumentTypes,
-    ] = await Promise.all([
+    let [existingTags, documents, ownUserId, existingCorrespondentList, existingDocumentTypes] = await Promise.all([
       paperlessService.getTags(),
       paperlessService.getAllDocuments(),
+      paperlessService.getOwnUserID(),
       paperlessService.listCorrespondentsNames(),
-      paperlessService.listDocumentTypesNames(),
+      paperlessService.listDocumentTypesNames()
     ]);
 
     scanStats.total = documents.length;
 
     // get existing correspondent list
-    existingCorrespondentList = existingCorrespondentList.map(
-      (correspondent) => correspondent.name
-    );
+    existingCorrespondentList = existingCorrespondentList.map((correspondent) => correspondent.name);
 
     // get existing document types list
-    const existingDocumentTypesList = existingDocumentTypes.map(
-      (docType) => docType.name
-    );
+    const existingDocumentTypesList = existingDocumentTypes.map((docType) => docType.name);
 
     // Extract tag names from tag objects
     const existingTagNames = existingTags.map((tag) => tag.name);
@@ -1051,19 +836,12 @@ async function scanDocuments(source = 'scheduler') {
     for (const doc of documents) {
       if (scanControl.stopRequested) {
         scanStats.stopRequested = true;
-        console.info(
-          `Graceful stop requested. Halting scan before next document (source=${scanControl.source || 'unknown'})`
-        );
+        console.info(`Graceful stop requested. Halting scan before next document (source=${scanControl.source || 'unknown'})`);
         break;
       }
 
       try {
-        const result = await processDocument(
-          doc,
-          existingTagNames,
-          existingCorrespondentList,
-          existingDocumentTypesList
-        );
+        const result = await processDocument(doc, existingTagNames, existingCorrespondentList, existingDocumentTypesList, ownUserId);
         if (!result) {
           scanStats.skipped += 1;
           continue;
@@ -1077,31 +855,17 @@ async function scanDocuments(source = 'scheduler') {
       } catch (error) {
         await documentModel.setProcessingStatus(doc.id, doc.title, 'failed');
         scanStats.failed += 1;
-        console.error(
-          `[ERROR] processing document ${doc.id}: ${error.message}`
-        );
+        console.error(`[ERROR] processing document ${doc.id}: ${error.message}`);
         console.debug(error);
       }
     }
-
-    // Documents that fail individually are not an infrastructure problem, so
-    // the run itself still counts as successful for health reporting.
-    scanHealthService.recordRunResult({ status: RUN_STATUS.OK });
   } catch (error) {
-    scanHealthService.recordRunResult({
-      status: RUN_STATUS.ERROR,
-      error: error.message,
-    });
-    scanStats.abortReason = 'error';
     console.error(`[ERROR] during document scan: ${error.message}`);
     console.debug(error);
   } finally {
     const durationMs = Date.now() - scanStartedAtMs;
-    const abortSuffix = scanStats.abortReason
-      ? `, aborted=${scanStats.abortReason}`
-      : '';
     console.info(
-      `Scan completed (source=${scanStats.source}, total=${scanStats.total}, processed=${scanStats.processed}, skipped=${scanStats.skipped}, failed=${scanStats.failed}, stopRequested=${scanStats.stopRequested}${abortSuffix}, durationMs=${durationMs})`
+      `Scan completed (source=${scanStats.source}, total=${scanStats.total}, processed=${scanStats.processed}, skipped=${scanStats.skipped}, failed=${scanStats.failed}, stopRequested=${scanStats.stopRequested}, durationMs=${durationMs})`
     );
 
     scanControl.running = false;
@@ -1150,252 +914,110 @@ app.get('/', async (req, res) => {
   }
 });
 
-// /health is served by routes/setup.js, which is mounted above this file's
-// routes. A second handler here was never reachable and has been removed so the
-// OpenAPI spec documents the handler that actually answers.
-
-// Error handler
-// Express detects error middleware by its four-argument signature, so `next`
-// must stay even though it is unused.
-// eslint-disable-next-line no-unused-vars
-app.use((err, req, res, next) => {
-  console.error(err.stack);
-  res.status(500).send('Something broke!');
-});
-
-// Backoff schedule for the startup connectivity retry, in milliseconds.
-// The last value repeats until the retry window configured via
-// STARTUP_PAPERLESS_RETRY_MINUTES is exhausted.
-const INITIAL_CONNECT_BACKOFF_MS = [5000, 15000, 30000, 60000, 120000, 300000];
-const DEFAULT_STARTUP_RETRY_MINUTES = 30;
-
-function initialConnectDelayMs(attempt) {
-  const index = Math.min(attempt - 1, INITIAL_CONNECT_BACKOFF_MS.length - 1);
-  return INITIAL_CONNECT_BACKOFF_MS[Math.max(index, 0)];
-}
-
-function startupRetryWindowMs() {
-  const configured = Number(config.startup?.paperlessRetryMinutes);
-  const minutes =
-    Number.isFinite(configured) && configured >= 0
-      ? configured
-      : DEFAULT_STARTUP_RETRY_MINUTES;
-  return minutes * 60 * 1000;
-}
-
 /**
- * Waits for Paperless-ngx to become usable and then runs the initial scan.
- *
- * Runs detached from startup: the scan scheduler is already armed at this
- * point, so giving up here only means the first scan waits for the next cron
- * tick instead of never happening (issue #272).
+ * @swagger
+ * /health:
+ *   get:
+ *     summary: System health check endpoint
+ *     description: |
+ *       Checks if the application is properly configured and the database is reachable.
+ *       This endpoint can be used by monitoring systems to verify service health.
+ *       
+ *       The endpoint returns a 200 status code with a "healthy" status if everything is 
+ *       working correctly, or a 503 status code with error details if there are issues.
+ *     tags: [System]
+ *     responses:
+ *       200:
+ *         description: System is healthy and operational
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               properties:
+ *                 status:
+ *                   type: string
+ *                   example: "healthy"
+ *                   description: Health status indication
+ *       503:
+ *         description: System is not fully configured or database is unreachable
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               properties:
+ *                 status:
+ *                   type: string
+ *                   enum: [not_configured, error]
+ *                   example: "not_configured"
+ *                   description: Error status type
+ *                 message:
+ *                   type: string
+ *                   example: "Application setup not completed"
+ *                   description: Detailed error message
  */
-async function runInitialScanWhenReachable() {
-  const deadlineMs = Date.now() + startupRetryWindowMs();
-  let attempt = 0;
-
-  for (;;) {
+app.get('/health', async (req, res) => {
+  try {
     const isConfigured = await setupService.isConfigured();
     if (!isConfigured) {
-      console.warn(
-        'Initial scan skipped: setup is not completed yet. The scheduled scan stays armed.'
-      );
-      return;
-    }
-
-    const connection = await paperlessService.checkConnection();
-    scanHealthService.recordConnectivity(connection);
-
-    if (connection.reachable && connection.authorized) {
-      console.log(`Starting initial scan at ${new Date().toISOString()}`);
-      await scanDocuments('initial');
-      return;
-    }
-
-    attempt += 1;
-    const delayMs = initialConnectDelayMs(attempt);
-
-    if (Date.now() + delayMs > deadlineMs) {
-      // Count the abandoned initial scan as exactly one failed run. Counting
-      // every retry would trip the degraded threshold within minutes and make
-      // /health answer 503 during a perfectly normal slow start.
-      scanHealthService.recordRunResult({
-        status: RUN_STATUS.PAPERLESS_UNREACHABLE,
-        error: connection.error,
+      return res.status(503).json({ 
+        status: 'not_configured',
+        message: 'Application setup not completed'
       });
-      console.error(
-        `[STARTUP] Paperless-ngx still not usable after ${attempt} attempt(s): ${connection.error}. ` +
-          `Skipping the initial scan — the scheduled scan (${config.scanInterval}) stays armed and keeps retrying.`
-      );
-      return;
     }
 
-    console.warn(
-      `[STARTUP] Paperless-ngx not usable yet: ${connection.error}. ` +
-        `Retrying in ${Math.round(delayMs / 1000)}s (attempt ${attempt}). ` +
-        `The scheduled scan (${config.scanInterval}) is armed regardless.`
-    );
-    await new Promise((resolve) => setTimeout(resolve, delayMs));
-  }
-}
-
-const DEFAULT_PAPERLESS_PROBE_INTERVAL_SECONDS = 60;
-const MIN_PAPERLESS_PROBE_INTERVAL_SECONDS = 10;
-
-function paperlessProbeIntervalMs() {
-  const configured = Number(config.health?.paperlessProbeIntervalSeconds);
-  if (!Number.isFinite(configured)) {
-    return DEFAULT_PAPERLESS_PROBE_INTERVAL_SECONDS * 1000;
-  }
-  if (configured <= 0) {
-    return 0;
-  }
-  return Math.max(configured, MIN_PAPERLESS_PROBE_INTERVAL_SECONDS) * 1000;
-}
-
-/**
- * Probes Paperless-ngx on a fixed interval, independently of the scan loop.
- *
- * The scan loop only probes when it runs, so between two ticks an outage was
- * invisible to the dashboard — and entirely invisible with
- * DISABLE_AUTOMATIC_PROCESSING=yes. This keeps `paperless.lastCheckedAt` fresh
- * so the UI can warn immediately.
- *
- * Only connectivity is recorded here; the consecutive-failure counter belongs
- * to actual scan runs and must not be moved by a passive probe.
- */
-function startConnectivityMonitor() {
-  const intervalMs = paperlessProbeIntervalMs();
-  if (intervalMs === 0) {
-    console.info(
-      '[HEALTH] Paperless-ngx connectivity probe is disabled (PAPERLESS_PROBE_INTERVAL_SECONDS=0).'
-    );
-    return;
+    await documentModel.isDocumentProcessed(1);
+    res.json({ status: 'healthy' });
+  } catch (error) {
+    console.error(`Health check failed: ${error.message}`);
+    console.debug(error);
+    res.status(503).json({ 
+      status: 'error', 
+      message: error.message 
+    });
   }
+});
 
-  let probeRunning = false;
-
-  const probe = async () => {
-    // A scan probes on its own and would only duplicate the request here.
-    if (probeRunning || scanControl.running) {
-      return;
-    }
-    probeRunning = true;
-
-    try {
-      if (!(await setupService.isConfigured())) {
-        // Nothing configured yet — reporting "not reachable" would warn about
-        // a connection the user has not set up.
-        scanHealthService.clearConnectivity();
-        return;
-      }
-
-      scanHealthService.recordConnectivity(
-        await paperlessService.checkConnection()
-      );
-    } catch (error) {
-      console.debug(
-        `[HEALTH] Connectivity probe failed unexpectedly: ${error.message}`
-      );
-    } finally {
-      probeRunning = false;
-    }
-  };
-
-  const timer = setInterval(probe, intervalMs);
-  // Never keep the event loop alive just for the probe.
-  timer.unref?.();
-  console.info(
-    `[HEALTH] Paperless-ngx connectivity probe armed (every ${intervalMs / 1000}s).`
-  );
-
-  probe();
-}
+// Error handler
+app.use((err, req, res, next) => {
+  console.error(err.stack);
+  res.status(500).send('Something broke!');
+});
 
 // Start scanning
 async function startScanning() {
   try {
     const isConfigured = await setupService.isConfigured();
     if (!isConfigured) {
-      console.log(
-        `Setup not completed. Visit http://your-machine-ip:${process.env.PAPERLESS_AI_PORT || 3000}/setup to complete setup.`
-      );
+      console.log(`Setup not completed. Visit http://your-machine-ip:${process.env.PAPERLESS_AI_PORT || 3000}/setup to complete setup.`);
     }
 
-    // Reconciliation: remove stale documents deleted in Paperless-ngx.
-    // Armed independently of Paperless-ngx reachability so a temporary outage
-    // cannot leave the app without any scheduled work.
-    if (config.reconciliationEnabled === 'yes') {
-      console.log(
-        'Configured reconciliation interval:',
-        config.reconciliationInterval
-      );
-      cron.schedule(config.reconciliationInterval, async () => {
-        console.debug(
-          `[RECONCILIATION] Scheduled run triggered at ${new Date().toISOString()}`
-        );
-        await reconciliationService.reconcileAllDocuments();
-      });
-    } else {
-      console.info(
-        '[RECONCILIATION] Automatic reconciliation is disabled (RECONCILIATION_ENABLED=no).'
-      );
+    const userId = await paperlessService.getOwnUserID();
+    if (!userId) {
+      console.error('Failed to get own user ID. Abort scanning.');
+      return;
     }
 
-    if (config.disableAutomaticProcessing === 'yes') {
-      scanHealthService.markAutomaticProcessingDisabled();
-      console.info(
-        'Automatic document processing is disabled (DISABLE_AUTOMATIC_PROCESSING=yes). No scan is scheduled.'
-      );
-      return;
+    console.log('Configured scan interval:', config.scanInterval);
+    console.log(`Starting initial scan at ${new Date().toISOString()}`);
+    if(config.disableAutomaticProcessing != 'yes') {
+      await scanInitial();
+  
+      cron.schedule(config.scanInterval, async () => {
+        console.log(`Starting scheduled scan at ${new Date().toISOString()}`);
+        await scanDocuments();
+      });
     }
 
-    // OCR auto-processing: drain the pending OCR queue without anyone having
-    // to press "Process All Pending". Armed after the kill-switch above
-    // because OCR + AI writes results back to Paperless-ngx, which is exactly
-    // what DISABLE_AUTOMATIC_PROCESSING is meant to stop.
-    if (ocrAutoProcessService.isEnabled()) {
-      const ocrAutoProcessInterval = ocrAutoProcessService.interval;
-      console.log(
-        'Configured OCR auto-processing interval:',
-        ocrAutoProcessInterval
-      );
-      cron.schedule(ocrAutoProcessInterval, async () => {
-        // Never compete with a running scan for the same AI backend.
-        if (scanControl.running) {
-          console.debug(
-            '[OCR] Auto-processing skipped: a document scan is currently running.'
-          );
-          return;
-        }
-        await ocrAutoProcessService.drainQueue();
+    // Reconciliation: remove stale documents deleted in Paperless-ngx
+    if (config.reconciliationEnabled) {
+      console.log('Configured reconciliation interval:', config.reconciliationInterval);
+      cron.schedule(config.reconciliationInterval, async () => {
+        console.debug(`[RECONCILIATION] Scheduled run triggered at ${new Date().toISOString()}`);
+        await reconciliationService.reconcileAllDocuments();
       });
     } else {
-      console.info(
-        '[OCR] Automatic OCR queue processing is disabled (OCR_AUTO_PROCESS_ENABLED=no).'
-      );
+      console.info('[RECONCILIATION] Automatic reconciliation is disabled (RECONCILIATION_ENABLED=no).');
     }
-
-    // Arm the scheduler before talking to Paperless-ngx. A connection failure
-    // at startup must never leave the app running without a scan loop —
-    // every scheduled run re-checks connectivity on its own.
-    console.log('Configured scan interval:', config.scanInterval);
-    cron.schedule(config.scanInterval, async () => {
-      console.log(`Starting scheduled scan at ${new Date().toISOString()}`);
-      if (!(await setupService.isConfigured())) {
-        console.warn('Scheduled scan skipped: setup is not completed.');
-        return;
-      }
-      await scanDocuments();
-    });
-    scanHealthService.markArmed(config.scanInterval);
-
-    // Detached on purpose: startup must not block while Paperless-ngx is still
-    // coming up. unhandledRejection terminates the process, so catch here.
-    runInitialScanWhenReachable().catch((error) => {
-      console.error(`[ERROR] during initial scan: ${error.message}`);
-      console.debug(error);
-    });
   } catch (error) {
     console.error(`[ERROR] in startScanning: ${error.message}`);
     console.debug(error);
@@ -1456,9 +1078,6 @@ async function startServer() {
       console.log(`Server running on port ${port}`);
       warnIfRemoteSetupExposed();
       startScanning();
-      // Armed separately from the scan scheduler so the dashboard also warns
-      // when automatic processing is switched off.
-      startConnectivityMonitor();
     });
   } catch (error) {
     console.error(`Failed to start server: ${error.message}`);
diff --git a/services/dataProcessingUtils.js b/services/dataProcessingUtils.js
new file mode 100644
index 00000000..0b9f02d2
--- /dev/null
+++ b/services/dataProcessingUtils.js
@@ -0,0 +1,123 @@
+const { validateCustomFieldValue } = require('./serviceUtils');
+
+async function updateCustomFieldsData(analysis, doc, updateData, config, paperlessService) {
+  // Only process custom fields if custom fields detection is activated
+  if (
+    config.limitFunctions?.activateCustomFields === 'no' ||
+    !analysis.document.custom_fields
+  ) {
+    console.log('[DEBUG] no processing of customFields.');
+    return;
+  }
+
+  const customFields = analysis.document.custom_fields;
+  const processedFields = [];
+  const customFieldsForHistory = [];
+
+  console.log('[DEBUG] updateCustomFieldsData:', customFields);
+
+  // Get existing custom fields
+  const existingFields = await paperlessService.getExistingCustomFields(doc.id);
+  console.debug('Found existing fields:', existingFields);
+
+  // Keep track of which fields we've processed to avoid duplicates
+  const processedFieldIds = new Set();
+
+  // First, add any new/updated fields
+  for (const key in customFields) {
+    const customField = customFields[key];
+
+    if (
+      !customField.field_name ||
+      customField.value === null ||
+      customField.value === undefined ||
+      String(customField.value).trim() === ''
+    ) {
+      console.debug('Skipping empty or invalid custom field');
+      continue;
+    }
+
+    const fieldDetails = await paperlessService.findExistingCustomField(
+      customField.field_name
+    );
+    if (fieldDetails?.id) {
+      const validation = validateCustomFieldValue(
+        customField.field_name,
+        customField.value,
+        fieldDetails.data_type
+      );
+      if (validation.skip) {
+        if (validation.warn) console.warn(validation.warn);
+        continue;
+      }
+      processedFields.push({
+        field: fieldDetails.id,
+        value: validation.value,
+      });
+      // Capture name + validated value for history at the point where we have both
+      customFieldsForHistory.push({
+        field_name: customField.field_name,
+        value: validation.value,
+      });
+      processedFieldIds.add(fieldDetails.id);
+    }
+  }
+
+  // Then add any existing fields that weren't updated
+  for (const existingField of existingFields) {
+    if (!processedFieldIds.has(existingField.field)) {
+      processedFields.push(existingField);
+    }
+  }
+
+  if (processedFields.length > 0) {
+    updateData.custom_fields = processedFields;
+  }
+  if (customFieldsForHistory.length > 0) {
+    updateData._customFieldsForHistory = customFieldsForHistory;
+  }
+}
+
+async function updateDocumentTypeData(analysis, updateData, config, paperlessService, options = {}) {
+  if (config.limitFunctions?.activateDocumentType === 'no' || !analysis.document.document_type) {
+    return;
+  }
+
+  try {
+    const documentType = await paperlessService.getOrCreateDocumentType(
+      analysis.document.document_type,
+      options
+    );
+    if (documentType) {
+      updateData.document_type = documentType.id;
+    }
+  } catch (error) {
+    console.error(`[ERROR] Error processing document type: ${error.message}`);
+    console.debug(error);
+  }
+}
+
+async function updateCorrespondentData(analysis, updateData, config, paperlessService, options = {}) {
+  if (config.limitFunctions?.activateCorrespondents === 'no' || !analysis.document.correspondent) {
+    return;
+  }
+
+  try {
+    const correspondent = await paperlessService.getOrCreateCorrespondent(
+      analysis.document.correspondent,
+      options
+    );
+    if (correspondent) {
+      updateData.correspondent = correspondent.id;
+    }
+  } catch (error) {
+    console.error(`[ERROR] Error processing correspondent: ${error.message}`);
+    console.debug(error);
+  }
+}
+
+module.exports = {
+  updateCustomFieldsData,
+  updateDocumentTypeData,
+  updateCorrespondentData
+};
diff --git a/services/mistralOcrService.js b/services/mistralOcrService.js
index 792c2cb2..28b3d3d4 100644
--- a/services/mistralOcrService.js
+++ b/services/mistralOcrService.js
@@ -13,6 +13,11 @@ const popplerService = require('./popplerService');
 const documentModel = require('../models/document');
 const AIServiceFactory = require('./aiServiceFactory');
 const { isTimeoutError, buildTimeoutErrorMessage } = require('./serviceUtils');
+const {
+  updateCustomFieldsData,
+  updateDocumentTypeData,
+  updateCorrespondentData
+} = require('./dataProcessingUtils');
 
 class MistralOcrService {
   constructor() {
@@ -956,8 +961,8 @@ class MistralOcrService {
       throw new Error(analysis.error);
     }
 
-    // Build update data (simplified – reuse paperlessService helpers)
     const updateData = {};
+
     const config = require('../config/config');
     const options = {
       restrictToExistingTags: config.restrictToExistingTags === 'yes',
@@ -967,41 +972,7 @@ class MistralOcrService {
         config.restrictToExistingDocumentTypes === 'yes',
     };
 
-    if (config.limitFunctions?.activateTagging !== 'no') {
-      const { tagIds } = await PaperlessService.processTags(
-        analysis.document.tags,
-        options
-      );
-      updateData.tags = tagIds;
-    }
-    if (config.limitFunctions?.activateTitle !== 'no') {
-      updateData.title = analysis.document.title || originalData.title;
-    }
-    updateData.created =
-      analysis.document.document_date || originalData.created;
-    if (
-      config.limitFunctions?.activateDocumentType !== 'no' &&
-      analysis.document.document_type
-    ) {
-      const dt = await PaperlessService.getOrCreateDocumentType(
-        analysis.document.document_type,
-        options
-      );
-      if (dt) updateData.document_type = dt.id;
-    }
-    if (
-      config.limitFunctions?.activateCorrespondents !== 'no' &&
-      analysis.document.correspondent
-    ) {
-      const corr = await PaperlessService.getOrCreateCorrespondent(
-        analysis.document.correspondent,
-        options
-      );
-      if (corr) updateData.correspondent = corr.id;
-    }
-    if (analysis.document.language) {
-      updateData.language = analysis.document.language;
-    }
+    await this.calculateNewDocumentMetadata(config, analysis, options, updateData, originalData);
 
     // Apply updates to Paperless
     const updatedDocument = await PaperlessService.updateDocument(
@@ -1012,6 +983,15 @@ class MistralOcrService {
       throw new Error(`Paperless update failed for document ${documentId}`);
     }
 
+    const currentCorrespondentId = updatedDocument.correspondent ?? updateData.correspondent;
+    const currentCorrespondent = currentCorrespondentId
+      ? await PaperlessService.getCorrespondentNameById(currentCorrespondentId)
+      : null;
+    const currentDocumentTypeId = updatedDocument.document_type ?? updateData.document_type;
+    const currentDocumentType = currentDocumentTypeId
+      ? await PaperlessService.getDocumentTypeNameById(currentDocumentTypeId)
+      : null;
+
     // Persist metrics & history
     if (analysis.metrics) {
       await documentModel.addOpenAIMetrics(
@@ -1029,14 +1009,42 @@ class MistralOcrService {
       documentId,
       updateData.tags || [],
       updateData.title || originalData.title,
-      analysis.document.correspondent,
+      currentCorrespondent?.name ?? null,
       null,
-      analysis.document.document_type || null,
+      currentDocumentType?.name ?? null,
       analysis.document.language || null
     );
 
     return analysis;
   }
+
+
+  async calculateNewDocumentMetadata(config, analysis, options,
+                                     updateData, originalData) {
+    if (config.limitFunctions?.activateTagging !== 'no') {
+      const { tagIds } = await PaperlessService.processTags(
+        analysis.document.tags,
+        options
+      );
+      updateData.tags = tagIds;
+    }
+    if (config.limitFunctions?.activateTitle !== 'no') {
+      updateData.title = analysis.document.title || originalData.title;
+    }
+
+    await updateDocumentTypeData(analysis,
+      updateData, config, PaperlessService, options);
+    await updateCorrespondentData(analysis,
+      updateData, config, PaperlessService, options);
+    await updateCustomFieldsData(analysis,
+      originalData, updateData, config, PaperlessService);
+
+    updateData.created =
+      analysis.document.document_date || originalData.created;
+
+    updateData.language =
+      analysis.document.language || updateData.language;
+  }
 }
 
 module.exports = new MistralOcrService();
diff --git a/services/ollamaService.js b/services/ollamaService.js
index 13291f26..2aef95ae 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -1,534 +1,453 @@
-const { writePromptToFile, toNameList } = require('./serviceUtils');
+const {
+    calculateTokens,
+    calculateTotalPromptTokens,
+    truncateToTokenLimit,
+    writePromptToFile
+} = require('./serviceUtils');
 const axios = require('axios');
 const config = require('../config/config');
 const fs = require('fs').promises;
+const path = require('path');
 const paperlessService = require('./paperlessService');
 const os = require('os');
-const {
-  THUMBNAIL_CACHE_DIR,
-  getThumbnailCachePath,
-} = require('./thumbnailCachePaths');
+const OpenAI = require('openai');
+const { THUMBNAIL_CACHE_DIR, getThumbnailCachePath } = require('./thumbnailCachePaths');
 const RestrictionPromptService = require('./restrictionPromptService');
 
 /**
  * Service for document analysis using Ollama
  */
 class OllamaService {
-  /**
-   * Initialize the Ollama service
-   */
-  constructor() {
-    this.apiUrl = config.ollama.apiUrl;
-    this.model = config.ollama.model;
-    this.client = axios.create({
-      timeout: 1800000, // 30 minutes timeout
-    });
-
-    // JSON schema for document analysis output
-    this.documentAnalysisSchema = {
-      type: 'object',
-      properties: {
-        title: { type: 'string' },
-        correspondent: { type: 'string' },
-        tags: {
-          type: 'array',
-          items: { type: 'string' },
-        },
-        document_type: { type: 'string' },
-        document_date: { type: 'string' },
-        language: { type: 'string' },
-        custom_fields: {
-          type: 'object',
-          additionalProperties: true,
-        },
-      },
-      required: [
-        'title',
-        'correspondent',
-        'tags',
-        'document_type',
-        'document_date',
-        'language',
-      ],
-    };
-
-    // Schema for playground analysis (simpler version)
-    this.playgroundSchema = {
-      type: 'object',
-      properties: {
-        title: { type: 'string' },
-        correspondent: { type: 'string' },
-        tags: {
-          type: 'array',
-          items: { type: 'string' },
-        },
-        document_type: { type: 'string' },
-        document_date: { type: 'string' },
-        language: { type: 'string' },
-      },
-      required: [
-        'title',
-        'correspondent',
-        'tags',
-        'document_type',
-        'document_date',
-        'language',
-      ],
-    };
-  }
-
-  /**
-   * Analyze a document and extract metadata
-   * @param {string} content - Document content
-   * @param {Array} existingTags - List of existing tags
-   * @param {Array} existingCorrespondentList - List of existing correspondents
-   * @param {string} id - Document ID
-   * @param {string} customPrompt - Custom prompt (optional)
-   * @returns {Object} Analysis results
-   */
-  async analyzeDocument(
-    content,
-    existingTags = [],
-    existingCorrespondentList = [],
-    existingDocumentTypesList = [],
-    id,
-    customPrompt = null,
-    options = {}
-  ) {
-    try {
-      // Truncate content if needed
-      content = this._truncateContent(content);
-
-      // Cache thumbnail
-      await this._handleThumbnailCaching(id);
-
-      // Get external API data if available and validate it
-      let externalApiData = options.externalApiData || null;
-      let validatedExternalApiData = null;
-
-      if (externalApiData) {
+    /**
+     * Initialize the Ollama service
+     */
+    constructor() {
+        this.apiUrl = config.ollama.apiUrl;
+        this.model = config.ollama.model;
+        this.client = axios.create({
+            timeout: 1800000 // 30 minutes timeout
+        });
+
+        // JSON schema for document analysis output
+        this.documentAnalysisSchema = {
+            type: "object",
+            properties: {
+                title: { type: "string" },
+                correspondent: { type: "string" },
+                tags: {
+                    type: "array",
+                    items: { type: "string" }
+                },
+                document_type: { type: "string" },
+                document_date: { type: "string" },
+                language: { type: "string" },
+                custom_fields: {
+                    type: "object",
+                    additionalProperties: true
+                }
+            },
+            required: ["title", "correspondent", "tags", "document_type", "document_date", "language"]
+        };
+
+        // Schema for playground analysis (simpler version)
+        this.playgroundSchema = {
+            type: "object",
+            properties: {
+                title: { type: "string" },
+                correspondent: { type: "string" },
+                tags: {
+                    type: "array",
+                    items: { type: "string" }
+                },
+                document_type: { type: "string" },
+                document_date: { type: "string" },
+                language: { type: "string" }
+            },
+            required: ["title", "correspondent", "tags", "document_type", "document_date", "language"]
+        };
+    }
+
+    /**
+     * Analyze a document and extract metadata
+     * @param {string} content - Document content
+     * @param {Array} existingTags - List of existing tags
+     * @param {Array} existingCorrespondentList - List of existing correspondents
+     * @param {string} id - Document ID
+     * @param {string} customPrompt - Custom prompt (optional)
+     * @returns {Object} Analysis results
+     */
+    async analyzeDocument(content, existingTags = [], existingCorrespondentList = [], existingDocumentTypesList = [], id, customPrompt = null, options = {}) {
+        try {
+            // Truncate content if needed
+            content = this._truncateContent(content);
+
+            // Cache thumbnail
+            await this._handleThumbnailCaching(id);
+
+            // Get external API data if available and validate it
+            let externalApiData = options.externalApiData || null;
+            let validatedExternalApiData = null;
+
+            if (externalApiData) {
+                try {
+                    validatedExternalApiData = await this._validateAndTruncateExternalApiData(externalApiData);
+                    console.log('[DEBUG] External API data validated and included');
+                } catch (error) {
+                    console.warn('[WARNING] External API data validation failed:', error.message);
+                    validatedExternalApiData = null;
+                }
+            }
+
+            // Build prompt
+            let prompt;
+            if (!customPrompt) {
+                prompt = this._buildPrompt(content, existingTags, existingCorrespondentList, existingDocumentTypesList, options);
+            } else {
+                // Parse CUSTOM_FIELDS for custom prompt
+                let customFieldsObj;
+                try {
+                    customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
+                } catch (error) {
+                    console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); console.debug(error);
+                    customFieldsObj = { custom_fields: [] };
+                }
+
+                const customFieldsTemplate = {};
+                customFieldsObj.custom_fields.forEach((field, index) => {
+                    let valueHint;
+                    if (field.data_type === 'date') {
+                        valueHint = "Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis";
+                    } else if (field.data_type === 'boolean') {
+                        valueHint = "Fill in 'true' or 'false' based on your analysis";
+                    } else {
+                        valueHint = "Fill in the value based on your analysis";
+                    }
+                    customFieldsTemplate[index] = {
+                        field_name: field.value,
+                        value: valueHint
+                    };
+                });
+
+                const customFieldsStr = '"custom_fields": ' + JSON.stringify(customFieldsTemplate, null, 2)
+                    .split('\n')
+                    .map(line => '    ' + line)
+                    .join('\n');
+
+                prompt = customPrompt + '\n\n' + config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr) + "\n\n" + JSON.stringify(content);
+                console.log('[DEBUG] Ollama Service started with custom prompt');
+            }
+
+            // Generate custom fields for the prompt
+            const customFieldsStr = this._generateCustomFieldsTemplate();
+
+            // Generate system prompt
+            const systemPrompt = this._generateSystemPrompt(customFieldsStr);
+
+            // Calculate context window size
+            const promptTokenCount = this._calculatePromptTokenCount(prompt);
+            const numCtx = this._calculateNumCtx(promptTokenCount, Number(config.responseTokens));
+
+            console.log(`[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`);
+            console.log(`[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`);
+
+            // Call Ollama API
+            const response = await this._callOllamaAPI(prompt, systemPrompt, numCtx, this.documentAnalysisSchema);
+
+            // Process response
+            const parsedResponse = this._processOllamaResponse(response);
+            const metrics = this._extractOllamaMetrics(response);
+
+            // Check for missing data
+            if (parsedResponse.tags.length === 0 && parsedResponse.correspondent === null) {
+                console.warn('No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.');
+            }
+
+            // Log the prompt and response
+            await this._logPromptAndResponse(prompt, parsedResponse);
+
+            // Return results in consistent format
+            return {
+                document: parsedResponse,
+                metrics,
+                truncated: false
+            };
+        } catch (error) {
+            console.error(`Error analyzing document with Ollama: ${error.message}`);
+            console.debug(error);
+            return {
+                document: { tags: [], correspondent: null },
+                metrics: null,
+                error: error.message
+            };
+        }
+    }
+
+    /**
+     * Analyze a document in playground mode
+     * @param {string} content - Document content
+     * @param {string} prompt - User-provided prompt
+     * @returns {Object} Analysis results
+     */
+    async analyzePlayground(content, prompt) {
         try {
-          validatedExternalApiData =
-            await this._validateAndTruncateExternalApiData(externalApiData);
-          console.log('[DEBUG] External API data validated and included');
+            // Calculate context window size — include both prompt and content
+            const fullPrompt = prompt + "\n\n" + JSON.stringify(content);
+            const promptTokenCount = this._calculatePromptTokenCount(fullPrompt);
+            const numCtx = this._calculateNumCtx(promptTokenCount, Number(config.responseTokens));
+
+            // Generate playground system prompt (simpler than full analysis)
+            const systemPrompt = this._generatePlaygroundSystemPrompt();
+
+            // Call Ollama API
+            const response = await this._callOllamaAPI(
+                fullPrompt,
+                systemPrompt,
+                numCtx,
+                this.playgroundSchema
+            );
+
+            // Process response
+            const parsedResponse = this._processOllamaResponse(response);
+            const metrics = this._extractOllamaMetrics(response);
+
+            // Check for missing data
+            if (parsedResponse.tags.length === 0 && parsedResponse.correspondent === null) {
+                console.warn('No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.');
+            }
+
+            // Return results in consistent format
+            return {
+                document: parsedResponse,
+                metrics,
+                truncated: false
+            };
         } catch (error) {
-          console.warn(
-            '[WARNING] External API data validation failed:',
-            error.message
-          );
-          validatedExternalApiData = null;
+            console.error(`Error analyzing document with Ollama: ${error.message}`);
+            console.debug(error);
+            return {
+                document: { tags: [], correspondent: null },
+                metrics: null,
+                error: error.message
+            };
         }
-      }
-
-      // Build prompt
-      let prompt;
-      if (!customPrompt) {
-        prompt = this._buildPrompt(
-          content,
-          existingTags,
-          existingCorrespondentList,
-          existingDocumentTypesList,
-          options
-        );
-      } else {
-        // Parse CUSTOM_FIELDS for custom prompt
+    }
+
+    /**
+     * Truncate content to maximum length if specified
+     * @param {string} content - Content to truncate
+     * @returns {string} Truncated content
+     */
+    _truncateContent(content) {
+        try {
+            if (process.env.CONTENT_MAX_LENGTH) {
+                console.log('Truncating content to max length:', process.env.CONTENT_MAX_LENGTH);
+                return content.substring(0, process.env.CONTENT_MAX_LENGTH);
+            }
+        } catch (error) {
+            console.error(`Error truncating content: ${error.message}`); console.debug(error);
+        }
+        return content;
+    }
+
+    /**
+     * Build prompt from content and existing data
+     * @param {string} content - Document content
+     * @param {Array} existingTags - List of existing tags
+     * @param {Array} existingCorrespondent - List of existing correspondents
+     * @param {Array} existingDocumentTypes - List of existing document types
+     * @returns {string} Formatted prompt
+     */
+    _buildPrompt(content, existingTags = [], existingCorrespondent = [], existingDocumentTypes = [], options = {}) {
+        let systemPrompt;
+        let promptTags = '';
+
+        // Validate that existingCorrespondent is an array and handle if it's not
+        const correspondentList = Array.isArray(existingCorrespondent)
+            ? existingCorrespondent
+            : [];
+
+        // Parse CUSTOM_FIELDS from environment variable
         let customFieldsObj;
         try {
-          customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
+            customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
         } catch (error) {
-          console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
-          console.debug(error);
-          customFieldsObj = { custom_fields: [] };
+             console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); console.debug(error);
+            customFieldsObj = { custom_fields: [] };
         }
 
+        // Generate custom fields template for the prompt
         const customFieldsTemplate = {};
+
         customFieldsObj.custom_fields.forEach((field, index) => {
-          let valueHint;
-          if (field.data_type === 'date') {
-            valueHint =
-              'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
-          } else if (field.data_type === 'boolean') {
-            valueHint = "Fill in 'true' or 'false' based on your analysis";
-          } else {
-            valueHint = 'Fill in the value based on your analysis';
-          }
-          customFieldsTemplate[index] = {
-            field_name: field.value,
-            value: valueHint,
-          };
+            let valueHint;
+            if (field.data_type === 'date') {
+                valueHint = "Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis";
+            } else if (field.data_type === 'boolean') {
+                valueHint = "Fill in 'true' or 'false' based on your analysis";
+            } else {
+                valueHint = "Fill in the value based on your analysis";
+            }
+            customFieldsTemplate[index] = {
+                field_name: field.value,
+                value: valueHint
+            };
         });
 
-        const customFieldsStr =
-          '"custom_fields": ' +
-          JSON.stringify(customFieldsTemplate, null, 2)
+        // Convert template to string for replacement and wrap in custom_fields
+        const customFieldsStr = '"custom_fields": ' + JSON.stringify(customFieldsTemplate, null, 2)
             .split('\n')
-            .map((line) => '    ' + line)
+            .map(line => '    ' + line)  // Add proper indentation
             .join('\n');
 
-        prompt =
-          customPrompt +
-          '\n\n' +
-          config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr) +
-          '\n\n' +
-          JSON.stringify(content);
-        console.log('[DEBUG] Ollama Service started with custom prompt');
-      }
-
-      // Generate custom fields for the prompt
-      const customFieldsStr = this._generateCustomFieldsTemplate();
-
-      // Generate system prompt
-      const systemPrompt = this._generateSystemPrompt(customFieldsStr);
-
-      // Calculate context window size
-      const promptTokenCount = this._calculatePromptTokenCount(prompt);
-      const numCtx = this._calculateNumCtx(
-        promptTokenCount,
-        Number(config.responseTokens)
-      );
-
-      console.log(
-        `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
-      );
-      console.log(
-        `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`
-      );
-
-      // Call Ollama API
-      const response = await this._callOllamaAPI(
-        prompt,
-        systemPrompt,
-        numCtx,
-        this.documentAnalysisSchema
-      );
-
-      // Process response
-      const parsedResponse = this._processOllamaResponse(response);
-      const metrics = this._extractOllamaMetrics(response);
-
-      // Check for missing data
-      if (
-        parsedResponse.tags.length === 0 &&
-        parsedResponse.correspondent === null
-      ) {
-        console.warn(
-          'No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.'
-        );
-      }
-
-      // Log the prompt and response
-      await this._logPromptAndResponse(prompt, parsedResponse);
-
-      // Return results in consistent format
-      return {
-        document: parsedResponse,
-        metrics,
-        truncated: false,
-      };
-    } catch (error) {
-      console.error(`Error analyzing document with Ollama: ${error.message}`);
-      console.debug(error);
-      return {
-        document: { tags: [], correspondent: null },
-        metrics: null,
-        error: error.message,
-      };
-    }
-  }
-
-  /**
-   * Analyze a document in playground mode
-   * @param {string} content - Document content
-   * @param {string} prompt - User-provided prompt
-   * @returns {Object} Analysis results
-   */
-  async analyzePlayground(content, prompt) {
-    try {
-      // Calculate context window size — include both prompt and content
-      const fullPrompt = prompt + '\n\n' + JSON.stringify(content);
-      const promptTokenCount = this._calculatePromptTokenCount(fullPrompt);
-      const numCtx = this._calculateNumCtx(
-        promptTokenCount,
-        Number(config.responseTokens)
-      );
-
-      // Generate playground system prompt (simpler than full analysis)
-      const systemPrompt = this._generatePlaygroundSystemPrompt();
-
-      // Call Ollama API
-      const response = await this._callOllamaAPI(
-        fullPrompt,
-        systemPrompt,
-        numCtx,
-        this.playgroundSchema
-      );
-
-      // Process response
-      const parsedResponse = this._processOllamaResponse(response);
-      const metrics = this._extractOllamaMetrics(response);
-
-      // Check for missing data
-      if (
-        parsedResponse.tags.length === 0 &&
-        parsedResponse.correspondent === null
-      ) {
-        console.warn(
-          'No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.'
-        );
-      }
-
-      // Return results in consistent format
-      return {
-        document: parsedResponse,
-        metrics,
-        truncated: false,
-      };
-    } catch (error) {
-      console.error(`Error analyzing document with Ollama: ${error.message}`);
-      console.debug(error);
-      return {
-        document: { tags: [], correspondent: null },
-        metrics: null,
-        error: error.message,
-      };
-    }
-  }
-
-  /**
-   * Truncate content to maximum length if specified
-   * @param {string} content - Content to truncate
-   * @returns {string} Truncated content
-   */
-  _truncateContent(content) {
-    try {
-      if (process.env.CONTENT_MAX_LENGTH) {
-        console.log(
-          'Truncating content to max length:',
-          process.env.CONTENT_MAX_LENGTH
-        );
-        return content.substring(0, process.env.CONTENT_MAX_LENGTH);
-      }
-    } catch (error) {
-      console.error(`Error truncating content: ${error.message}`);
-      console.debug(error);
-    }
-    return content;
-  }
-
-  /**
-   * Build prompt from content and existing data
-   * @param {string} content - Document content
-   * @param {Array} existingTags - List of existing tags
-   * @param {Array} existingCorrespondent - List of existing correspondents
-   * @param {Array} existingDocumentTypes - List of existing document types
-   * @returns {string} Formatted prompt
-   */
-  _buildPrompt(
-    content,
-    existingTags = [],
-    existingCorrespondent = [],
-    existingDocumentTypes = [],
-    options = {}
-  ) {
-    let systemPrompt;
-
-    // Validate that existingCorrespondent is an array and handle if it's not
-    const correspondentList = Array.isArray(existingCorrespondent)
-      ? existingCorrespondent
-      : [];
-
-    // Parse CUSTOM_FIELDS from environment variable
-    let customFieldsObj;
-    try {
-      customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-    } catch (error) {
-      console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
-      console.debug(error);
-      customFieldsObj = { custom_fields: [] };
-    }
-
-    // Generate custom fields template for the prompt
-    const customFieldsTemplate = {};
-
-    customFieldsObj.custom_fields.forEach((field, index) => {
-      let valueHint;
-      if (field.data_type === 'date') {
-        valueHint =
-          'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
-      } else if (field.data_type === 'boolean') {
-        valueHint = "Fill in 'true' or 'false' based on your analysis";
-      } else {
-        valueHint = 'Fill in the value based on your analysis';
-      }
-      customFieldsTemplate[index] = {
-        field_name: field.value,
-        value: valueHint,
-      };
-    });
-
-    // Convert template to string for replacement and wrap in custom_fields
-    const customFieldsStr =
-      '"custom_fields": ' +
-      JSON.stringify(customFieldsTemplate, null, 2)
-        .split('\n')
-        .map((line) => '    ' + line) // Add proper indentation
-        .join('\n');
-
-    // Get system prompt based on configuration
-    if (
-      config.useExistingData === 'yes' &&
-      config.restrictToExistingTags === 'no' &&
-      config.restrictToExistingCorrespondents === 'no'
-    ) {
-      // Format existing data - callers may hand over entity objects or plain names
-      const existingTagsList = toNameList(existingTags).join(', ');
-      const existingCorrespondentList =
-        toNameList(correspondentList).join(', ');
-      const existingDocumentTypesList = toNameList(existingDocumentTypes).join(
-        ', '
-      );
-
-      systemPrompt =
-        `
+        // Get system prompt based on configuration
+        if (config.useExistingData === 'yes' && config.restrictToExistingTags === 'no' && config.restrictToExistingCorrespondents === 'no') {
+            // Format existing tags
+            const existingTagsList = existingTags.join(', ');
+
+            // Format existing correspondents - handle both array of objects and array of strings
+            const existingCorrespondentList = correspondentList
+                .filter(Boolean)  // Remove any null/undefined entries
+                .map(correspondent => {
+                    if (typeof correspondent === 'string') return correspondent;
+                    return correspondent?.name || '';
+                })
+                .filter(name => name.length > 0)  // Remove empty strings
+                .join(', ');
+
+            // Format existing document types - handle both array of objects and array of strings
+            const existingDocumentTypesList = existingDocumentTypes
+                .filter(Boolean)  // Remove any null/undefined entries
+                .map(docType => {
+                    if (typeof docType === 'string') return docType;
+                    return docType?.name || '';
+                })
+                .filter(name => name.length > 0)  // Remove empty strings
+                .join(', ');
+
+            systemPrompt = `
             Pre-existing tags: ${existingTagsList}\n\n
             Pre-existing correspondents: ${existingCorrespondentList}\n\n
             Pre-existing document types: ${existingDocumentTypesList}\n\n
-            ` +
-        process.env.SYSTEM_PROMPT +
-        '\n\n' +
-        config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
-    } else {
-      const mustHavePrompt = config.mustHavePrompt.replace(
-        '%CUSTOMFIELDS%',
-        customFieldsStr
-      );
-      systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt;
-    }
+            ` + process.env.SYSTEM_PROMPT + '\n\n' + config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
+            promptTags = '';
+        } else {
+            const mustHavePrompt = config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
+            systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt;
+            promptTags = '';
+        }
 
-    // Get validated external API data if available
-    let validatedExternalApiData = null;
-    if (options.externalApiData) {
-      try {
-        validatedExternalApiData = this._validateAndTruncateExternalApiData(
-          options.externalApiData
-        );
-        console.log('[DEBUG] External API data validated and included');
-      } catch (error) {
-        console.warn(
-          '[WARNING] External API data validation failed:',
-          error.message
+        // Get validated external API data if available
+        let validatedExternalApiData = null;
+        if (options.externalApiData) {
+            try {
+                validatedExternalApiData = this._validateAndTruncateExternalApiData(options.externalApiData);
+                console.log('[DEBUG] External API data validated and included');
+            } catch (error) {
+                console.warn('[WARNING] External API data validation failed:', error.message);
+                validatedExternalApiData = null;
+            }
+        }
+
+        // Process placeholder replacements in system prompt
+        systemPrompt = RestrictionPromptService.processRestrictionsInPrompt(
+            systemPrompt,
+            existingTags,
+            correspondentList,
+            existingDocumentTypes,
+            config
         );
-        validatedExternalApiData = null;
-      }
-    }
 
-    // Process placeholder replacements in system prompt
-    systemPrompt = RestrictionPromptService.processRestrictionsInPrompt(
-      systemPrompt,
-      existingTags,
-      correspondentList,
-      existingDocumentTypes
-    );
-
-    // Include validated external API data if available
-    if (validatedExternalApiData) {
-      systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`;
-    }
+        // Include validated external API data if available
+        if (validatedExternalApiData) {
+            systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`;
+        }
 
-    if (process.env.USE_PROMPT_TAGS === 'yes') {
-      systemPrompt =
-        `
+        if (process.env.USE_PROMPT_TAGS === 'yes') {
+            promptTags = process.env.PROMPT_TAGS;
+            systemPrompt = `
             Take these tags and try to match one or more to the document content.\n\n
             ` + config.specialPromptPreDefinedTags;
-    }
+        }
 
-    return `${systemPrompt}
+        return `${systemPrompt}
         ${JSON.stringify(content)}
         `;
-  }
-
-  /**
-   * Validate and truncate external API data to prevent token overflow
-   * @param {any} apiData - The external API data to validate
-   * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500)
-   * @returns {string} - Validated and potentially truncated data string
-   */
-  async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
-    if (!apiData) {
-      return null;
     }
 
-    const dataString =
-      typeof apiData === 'object'
-        ? JSON.stringify(apiData, null, 2)
-        : String(apiData);
-
-    // Calculate tokens for the data (conservative 2 chars/token for non-English)
-    const dataTokens = Math.ceil(dataString.length / 2);
-
-    if (dataTokens > maxTokens) {
-      console.warn(
-        `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating`
-      );
-      // Simple truncation based on character count
-      const maxChars = maxTokens * 2;
-      return dataString.substring(0, maxChars);
+    /**
+     * Validate and truncate external API data to prevent token overflow
+     * @param {any} apiData - The external API data to validate
+     * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500)
+     * @returns {string} - Validated and potentially truncated data string
+     */
+    async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
+        if (!apiData) {
+            return null;
+        }
+
+        const dataString = typeof apiData === 'object'
+            ? JSON.stringify(apiData, null, 2)
+            : String(apiData);
+
+        // Calculate tokens for the data (conservative 2 chars/token for non-English)
+        const dataTokens = Math.ceil(dataString.length / 2);
+
+        if (dataTokens > maxTokens) {
+            console.warn(`[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating`);
+            // Simple truncation based on character count
+            const maxChars = maxTokens * 2;
+            return dataString.substring(0, maxChars);
+        }
+
+        console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`);
+        return dataString;
     }
 
-    console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`);
-    return dataString;
-  }
-
-  /**
-   * Generate custom fields template for prompts
-   * @returns {string} Custom fields template as a string
-   */
-  _generateCustomFieldsTemplate() {
-    let customFieldsObj;
-    try {
-      customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-    } catch (error) {
-      console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
-      console.debug(error);
-      customFieldsObj = { custom_fields: [] };
+    /**
+     * Generate custom fields template for prompts
+     * @returns {string} Custom fields template as a string
+     */
+    _generateCustomFieldsTemplate() {
+        let customFieldsObj;
+        try {
+            customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
+        } catch (error) {
+            console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); console.debug(error);
+            customFieldsObj = { custom_fields: [] };
+        }
+
+        // Generate custom fields template for the prompt
+        const customFieldsTemplate = {};
+
+        customFieldsObj.custom_fields.forEach((field, index) => {
+            let valueHint;
+            if (field.data_type === 'date') {
+                valueHint = "Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis";
+            } else if (field.data_type === 'boolean') {
+                valueHint = "Fill in 'true' or 'false' based on your analysis";
+            } else {
+                valueHint = "Fill in the value based on your analysis";
+            }
+            customFieldsTemplate[index] = {
+                field_name: field.value,
+                value: valueHint
+            };
+        });
+
+        // Convert template to string for replacement and wrap in custom_fields
+        return '"custom_fields": ' + JSON.stringify(customFieldsTemplate, null, 2)
+            .split('\n')
+            .map(line => '    ' + line)  // Add proper indentation
+            .join('\n');
     }
 
-    // Generate custom fields template for the prompt
-    const customFieldsTemplate = {};
-
-    customFieldsObj.custom_fields.forEach((field, index) => {
-      let valueHint;
-      if (field.data_type === 'date') {
-        valueHint =
-          'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
-      } else if (field.data_type === 'boolean') {
-        valueHint = "Fill in 'true' or 'false' based on your analysis";
-      } else {
-        valueHint = 'Fill in the value based on your analysis';
-      }
-      customFieldsTemplate[index] = {
-        field_name: field.value,
-        value: valueHint,
-      };
-    });
-
-    // Convert template to string for replacement and wrap in custom_fields
-    return (
-      '"custom_fields": ' +
-      JSON.stringify(customFieldsTemplate, null, 2)
-        .split('\n')
-        .map((line) => '    ' + line) // Add proper indentation
-        .join('\n')
-    );
-  }
-
-  /**
-   * Generate system prompt for document analysis
-   * @param {string} customFieldsStr - Custom fields as a string
-   * @returns {string} System prompt
-   */
-  _generateSystemPrompt(customFieldsStr) {
-    let systemPromptTemplate = `
+    /**
+     * Generate system prompt for document analysis
+     * @param {string} customFieldsStr - Custom fields as a string
+     * @returns {string} System prompt
+     */
+    _generateSystemPrompt(customFieldsStr) {
+        let systemPromptTemplate = `
             You are a document analyzer. Your task is to analyze documents and extract relevant information. You do not ask back questions. 
             YOU MUSTNOT: Ask for additional information or clarification, or ask questions about the document, or ask for additional context.
             YOU MUSTNOT: Return a response without the desired JSON format.
@@ -547,15 +466,15 @@ class OllamaService {
             ALWAYS USE THE INFORMATION TO FILL OUT THE JSON OBJECT. DO NOT ASK BACK QUESTIONS.
         `;
 
-    return systemPromptTemplate.replace('%CUSTOMFIELDS%', customFieldsStr);
-  }
+        return systemPromptTemplate.replace('%CUSTOMFIELDS%', customFieldsStr);
+    }
 
-  /**
-   * Generate system prompt for playground analysis
-   * @returns {string} System prompt
-   */
-  _generatePlaygroundSystemPrompt() {
-    return `
+    /**
+     * Generate system prompt for playground analysis
+     * @returns {string} System prompt
+     */
+    _generatePlaygroundSystemPrompt() {
+        return `
             You are a document analyzer. Your task is to analyze documents and extract relevant information. You do not ask back questions. 
             YOU MUSTNOT: Ask for additional information or clarification, or ask questions about the document, or ask for additional context.
             YOU MUSTNOT: Return a response without the desired JSON format.
@@ -569,354 +488,335 @@ class OllamaService {
             }
             ALWAYS USE THE INFORMATION TO FILL OUT THE JSON OBJECT. DO NOT ASK BACK QUESTIONS.
         `;
-  }
-
-  /**
-   * Calculate prompt token count
-   * @param {string} prompt - Prompt text
-   * @returns {number} Estimated token count
-   */
-  _calculatePromptTokenCount(prompt) {
-    // Use conservative 2 chars/token estimate to avoid truncation
-    // with non-English models (CJK, German, etc.) where tokenization
-    // produces roughly 2 chars per token instead of ~4 for English
-    return Math.ceil(prompt.length / 2);
-  }
-
-  /**
-   * Calculate context window size for Ollama
-   * @param {number} promptTokenCount - Token count for prompt
-   * @param {number} expectedResponseTokens - Expected response token count
-   * @returns {number} Context window size
-   */
-  _calculateNumCtx(promptTokenCount, expectedResponseTokens) {
-    const totalTokenUsage = promptTokenCount + expectedResponseTokens;
-    const maxCtxLimit = Number(config.tokenLimit);
-
-    const numCtx = Math.min(totalTokenUsage, maxCtxLimit);
-
-    console.log('Prompt Token Count:', promptTokenCount);
-    console.log('Expected Response Tokens:', expectedResponseTokens);
-    console.log('Dynamic calculated num_ctx:', numCtx);
-
-    return numCtx;
-  }
-
-  /**
-   * Get available system memory
-   * @returns {Object} Object with totalMemoryMB and freeMemoryMB
-   */
-  async _getAvailableMemory() {
-    const totalMemory = os.totalmem();
-    const freeMemory = os.freemem();
-    const totalMemoryMB = (totalMemory / (1024 * 1024)).toFixed(0);
-    const freeMemoryMB = (freeMemory / (1024 * 1024)).toFixed(0);
-    return { totalMemoryMB, freeMemoryMB };
-  }
-
-  /**
-   * Handle thumbnail caching for documents
-   * @param {string} id - Document ID
-   */
-  async _handleThumbnailCaching(id) {
-    if (!id) return;
-
-    const cachePath = getThumbnailCachePath(id);
-    try {
-      await fs.access(cachePath);
-      console.log('[DEBUG] Thumbnail already cached');
-    } catch {
-      console.log('Thumbnail not cached, fetching from Paperless');
-      const thumbnailData = await paperlessService.getThumbnailImage(id);
-      if (!thumbnailData) {
-        console.warn('Thumbnail not found');
-        return;
-      }
-      await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
-      await fs.writeFile(cachePath, thumbnailData);
     }
-  }
-
-  /**
-   * Build request headers, adding bearer auth when an API key is set.
-   * The key is read at request time so runtime config changes apply
-   * without re-instantiating the singleton.
-   * @returns {Object} Headers object
-   */
-  _buildRequestHeaders() {
-    const headers = { 'Content-Type': 'application/json' };
-    const apiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || '';
-    if (apiKey) {
-      headers.Authorization = `Bearer ${apiKey}`;
+
+    /**
+     * Calculate prompt token count
+     * @param {string} prompt - Prompt text
+     * @returns {number} Estimated token count
+     */
+    _calculatePromptTokenCount(prompt) {
+        // Use conservative 2 chars/token estimate to avoid truncation
+        // with non-English models (CJK, German, etc.) where tokenization
+        // produces roughly 2 chars per token instead of ~4 for English
+        return Math.ceil(prompt.length / 2);
+    }
+
+    /**
+     * Calculate context window size for Ollama
+     * @param {number} promptTokenCount - Token count for prompt
+     * @param {number} expectedResponseTokens - Expected response token count
+     * @returns {number} Context window size
+     */
+    _calculateNumCtx(promptTokenCount, expectedResponseTokens) {
+        const totalTokenUsage = promptTokenCount + expectedResponseTokens;
+        const maxCtxLimit = Number(config.tokenLimit);
+
+        const numCtx = Math.min(totalTokenUsage, maxCtxLimit);
+
+        console.log('Prompt Token Count:', promptTokenCount);
+        console.log('Expected Response Tokens:', expectedResponseTokens);
+        console.log('Dynamic calculated num_ctx:', numCtx);
+
+        return numCtx;
     }
-    return headers;
-  }
-
-  /**
-   * Call Ollama API
-   * @param {string} prompt - Prompt text
-   * @param {string} systemPrompt - System prompt
-   * @param {number} numCtx - Context window size
-   * @param {Object} schema - Response schema
-   * @returns {Object} Ollama API response
-   */
-  async _callOllamaAPI(prompt, systemPrompt, numCtx, schema) {
-    const requestBody = {
-      model: this.model,
-      prompt: prompt,
-      system: systemPrompt,
-      stream: false,
-      format: schema,
-      options: {
-        temperature: config.aiTemperatureAnalysis,
-        top_p: 0.9,
-        repeat_penalty: 1.1,
-        top_k: 7,
-        num_predict: 256,
-        num_ctx: numCtx,
-      },
-    };
-
-    // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
-    // unless explicitly enabled via OLLAMA_THINK=true
-    if (!config.ollama.think) {
-      requestBody.think = false;
+
+    /**
+     * Get available system memory
+     * @returns {Object} Object with totalMemoryMB and freeMemoryMB
+     */
+    async _getAvailableMemory() {
+        const totalMemory = os.totalmem();
+        const freeMemory = os.freemem();
+        const totalMemoryMB = (totalMemory / (1024 * 1024)).toFixed(0);
+        const freeMemoryMB = (freeMemory / (1024 * 1024)).toFixed(0);
+        return { totalMemoryMB, freeMemoryMB };
     }
 
-    const response = await this.client.post(
-      `${this.apiUrl}/api/generate`,
-      requestBody,
-      {
-        headers: this._buildRequestHeaders(),
-      }
-    );
+    /**
+     * Handle thumbnail caching for documents
+     * @param {string} id - Document ID
+     */
+    async _handleThumbnailCaching(id) {
+        if (!id) return;
 
-    if (!response.data) {
-      throw new Error('Invalid response from Ollama API');
+        const cachePath = getThumbnailCachePath(id);
+        try {
+            await fs.access(cachePath);
+            console.log('[DEBUG] Thumbnail already cached');
+        } catch (err) {
+            console.log('Thumbnail not cached, fetching from Paperless');
+            const thumbnailData = await paperlessService.getThumbnailImage(id);
+            if (!thumbnailData) {
+                console.warn('Thumbnail not found');
+                return;
+            }
+            await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+            await fs.writeFile(cachePath, thumbnailData);
+        }
     }
 
-    return response.data;
-  }
-
-  /**
-   * Process Ollama API response
-   * @param {Object} responseData - Ollama API response data
-   * @returns {Object} Parsed response
-   */
-  _processOllamaResponse(responseData) {
-    // Check if we got a structured response or need to parse from text
-    if (responseData.response && typeof responseData.response === 'object') {
-      // We got a structured response directly
-      console.log('Using structured output response');
-      return {
-        tags: Array.isArray(responseData.response.tags)
-          ? responseData.response.tags
-          : [],
-        correspondent: responseData.response.correspondent || null,
-        title: responseData.response.title || null,
-        document_date: responseData.response.document_date || null,
-        document_type: responseData.response.document_type || null,
-        language: responseData.response.language || null,
-        custom_fields: responseData.response.custom_fields || null,
-      };
-    } else if (responseData.response) {
-      // Fall back to parsing from text response
-      console.log('Falling back to text response parsing');
-      return this._parseResponse(responseData.response);
-    } else {
-      throw new Error('No response data from Ollama API');
+    /**
+     * Build request headers, adding bearer auth when an API key is set.
+     * The key is read at request time so runtime config changes apply
+     * without re-instantiating the singleton.
+     * @returns {Object} Headers object
+     */
+    _buildRequestHeaders() {
+        const headers = { 'Content-Type': 'application/json' };
+        const apiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || '';
+        if (apiKey) {
+            headers.Authorization = `Bearer ${apiKey}`;
+        }
+        return headers;
     }
-  }
-
-  /**
-   * Extract token usage metrics from an Ollama /api/generate response.
-   * @param {Object} responseData - Ollama API response data
-   * @returns {{promptTokens:number, completionTokens:number, totalTokens:number}}
-   */
-  _extractOllamaMetrics(responseData) {
-    const promptTokens = Number(responseData?.prompt_eval_count) || 0;
-    const completionTokens = Number(responseData?.eval_count) || 0;
-
-    return {
-      promptTokens,
-      completionTokens,
-      totalTokens: promptTokens + completionTokens,
-    };
-  }
-
-  /**
-   * Parse text response to extract JSON
-   * @param {string} response - Response text
-   * @returns {Object} Parsed object
-   */
-  _parseResponse(response) {
-    try {
-      // Find JSON in response using regex
-      const jsonMatch = response.match(/\{[\s\S]*\}/);
-      if (!jsonMatch) {
-        return { tags: [], correspondent: null };
-      }
-
-      let jsonStr = jsonMatch[0];
-      console.log('Extracted JSON String:', jsonStr);
-
-      try {
-        // Attempt to parse the JSON
-        const result = JSON.parse(jsonStr);
-
-        // Validate and return the result
-        return {
-          tags: Array.isArray(result.tags) ? result.tags : [],
-          correspondent: result.correspondent || null,
-          title: result.title || null,
-          document_date: result.document_date || null,
-          document_type: result.document_type || null,
-          language: result.language || null,
-          custom_fields: result.custom_fields || null,
+
+    /**
+     * Call Ollama API
+     * @param {string} prompt - Prompt text
+     * @param {string} systemPrompt - System prompt
+     * @param {number} numCtx - Context window size
+     * @param {Object} schema - Response schema
+     * @returns {Object} Ollama API response
+     */
+    async _callOllamaAPI(prompt, systemPrompt, numCtx, schema) {
+        const requestBody = {
+            model: this.model,
+            prompt: prompt,
+            system: systemPrompt,
+            stream: false,
+            format: schema,
+            options: {
+                temperature: config.aiTemperatureAnalysis,
+                top_p: 0.9,
+                repeat_penalty: 1.1,
+                top_k: 7,
+                // limits make no sense. imagine a doc with 50 pages
+                num_predict: -1,
+                num_ctx: 1024 * 32
+            }
         };
-      } catch (jsonError) {
-        console.warn('Error parsing JSON from response:', jsonError.message);
-        console.warn('Attempting to sanitize the JSON...');
 
-        // Sanitize the JSON
-        jsonStr = this._sanitizeJsonString(jsonStr);
+        // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
+        // unless explicitly enabled via OLLAMA_THINK=true
+        if (!config.ollama.think) {
+            requestBody.think = false;
+        }
 
-        try {
-          const sanitizedResult = JSON.parse(jsonStr);
-          return {
-            tags: Array.isArray(sanitizedResult.tags)
-              ? sanitizedResult.tags
-              : [],
-            correspondent: sanitizedResult.correspondent || null,
-            title: sanitizedResult.title || null,
-            document_date: sanitizedResult.document_date || null,
-            language: sanitizedResult.language || null,
-          };
-        } catch {
-          console.error(
-            'Final JSON parsing failed after sanitization. This happens when the JSON structure is too complex or invalid. That indicates an issue with the generated JSON string by Ollama. Switch to OpenAI for better results or fine tune your prompt.'
-          );
-          return { tags: [], correspondent: null };
+        const response = await this.client.post(`${this.apiUrl}/api/generate`, requestBody, {
+            headers: this._buildRequestHeaders()
+        });
+
+        if (!response.data) {
+            throw new Error('Invalid response from Ollama API');
+        }
+
+        return response.data;
+    }
+
+    /**
+     * Process Ollama API response
+     * @param {Object} responseData - Ollama API response data
+     * @returns {Object} Parsed response
+     */
+    _processOllamaResponse(responseData) {
+        // Check if we got a structured response or need to parse from text
+        if (responseData.response && typeof responseData.response === 'object') {
+            // We got a structured response directly
+            console.log('Using structured output response');
+            return {
+                tags: Array.isArray(responseData.response.tags) ? responseData.response.tags : [],
+                correspondent: responseData.response.correspondent || null,
+                title: responseData.response.title || null,
+                document_date: responseData.response.document_date || null,
+                document_type: responseData.response.document_type || null,
+                language: responseData.response.language || null,
+                custom_fields: responseData.response.custom_fields || null
+            };
+        } else if (responseData.response) {
+            console.log('No JS object yet, parsing JSON');
+            return this._parseResponse(responseData.response);
+        } else {
+            throw new Error('No response data from Ollama API');
         }
-      }
-    } catch (error) {
-      console.error('Error parsing Ollama response:', error.message);
-      return { tags: [], correspondent: null };
     }
-  }
-
-  /**
-   * Sanitize a JSON string
-   * @param {string} jsonStr - JSON string to sanitize
-   * @returns {string} Sanitized JSON string
-   */
-  _sanitizeJsonString(jsonStr) {
-    return jsonStr
-      .replace(/,\s*}/g, '}') // Remove trailing commas before closing braces
-      .replace(/,\s*]/g, ']') // Remove trailing commas before closing brackets
-      .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":'); // Ensure property names are quoted
-  }
-
-  /**
-   * Log prompt and response to file
-   * @param {string} prompt - Prompt text
-   * @param {Object} response - Response object
-   */
-  async _logPromptAndResponse(prompt, response) {
-    const content =
-      '================================================================================' +
-      prompt +
-      '\n\n' +
-      JSON.stringify(response) +
-      '\n\n' +
-      '================================================================================\n\n';
-
-    await writePromptToFile(content);
-  }
-
-  /**
-   * Generate text based on a prompt
-   * @param {string} prompt - The prompt to generate text from
-   * @returns {Promise<string>} - The generated text
-   */
-  async generateText(prompt) {
-    try {
-      // Calculate context window size based on prompt length
-      const promptTokenCount = this._calculatePromptTokenCount(prompt);
-      const numCtx = this._calculateNumCtx(
-        promptTokenCount,
-        Number(config.responseTokens)
-      );
-
-      // Simple system prompt for text generation
-      const systemPrompt = `You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.`;
-
-      // Call Ollama API without enforcing a specific response format
-      const generateTextBody = {
-        model: this.model,
-        prompt: prompt,
-        system: systemPrompt,
-        stream: false,
-        options: {
-          temperature: config.aiTemperatureGeneration,
-          top_p: 0.9,
-          num_predict: 1024,
-          num_ctx: numCtx,
-        },
-      };
-
-      // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
-      // unless explicitly enabled via OLLAMA_THINK=true
-      if (!config.ollama.think) {
-        generateTextBody.think = false;
-      }
-
-      const response = await this.client.post(
-        `${this.apiUrl}/api/generate`,
-        generateTextBody,
-        {
-          headers: this._buildRequestHeaders(),
+
+    /**
+     * Extract token usage metrics from an Ollama /api/generate response.
+     * @param {Object} responseData - Ollama API response data
+     * @returns {{promptTokens:number, completionTokens:number, totalTokens:number}}
+     */
+    _extractOllamaMetrics(responseData) {
+        const promptTokens = Number(responseData?.prompt_eval_count) || 0;
+        const completionTokens = Number(responseData?.eval_count) || 0;
+
+        return {
+            promptTokens,
+            completionTokens,
+            totalTokens: promptTokens + completionTokens
+        };
+    }
+
+    /**
+     * Parse text response to extract JSON
+     * @param {string} response - Response text
+     * @returns {Object} Parsed object
+     */
+    _parseResponse(response) {
+        try {
+            // Find JSON in response using regex
+            const jsonMatch = response.match(/\{[\s\S]*\}/);
+            if (!jsonMatch) {
+                return { tags: [], correspondent: null };
+            }
+
+            let jsonStr = jsonMatch[0];
+            console.log('Extracted JSON String:', jsonStr);
+
+            try {
+                // Attempt to parse the JSON
+                const result = JSON.parse(jsonStr);
+
+                // Validate and return the result
+                return {
+                    tags: Array.isArray(result.tags) ? result.tags : [],
+                    correspondent: result.correspondent || null,
+                    title: result.title || null,
+                    document_date: result.document_date || null,
+                    document_type: result.document_type || null,
+                    language: result.language || null,
+                    custom_fields: result.custom_fields || null
+                };
+
+            } catch (jsonError) {
+                console.warn('Error parsing JSON from response:', jsonError.message);
+                console.warn('Attempting to sanitize the JSON...');
+
+                // Sanitize the JSON
+                jsonStr = this._sanitizeJsonString(jsonStr);
+
+                try {
+                    const sanitizedResult = JSON.parse(jsonStr);
+                    return {
+                        tags: Array.isArray(sanitizedResult.tags) ? sanitizedResult.tags : [],
+                        correspondent: sanitizedResult.correspondent || null,
+                        title: sanitizedResult.title || null,
+                        document_date: sanitizedResult.document_date || null,
+                        language: sanitizedResult.language || null
+                    };
+                } catch (finalError) {
+                    console.error('Final JSON parsing failed after sanitization. This happens when the JSON structure is too complex or invalid. That indicates an issue with the generated JSON string by Ollama. Switch to OpenAI for better results or fine tune your prompt.');
+                    return { tags: [], correspondent: null };
+                }
+            }
+        } catch (error) {
+            console.error('Error parsing Ollama response:', error.message);
+            return { tags: [], correspondent: null };
         }
-      );
+    }
+
+    /**
+     * Sanitize a JSON string
+     * @param {string} jsonStr - JSON string to sanitize
+     * @returns {string} Sanitized JSON string
+     */
+    _sanitizeJsonString(jsonStr) {
+        return jsonStr
+            .replace(/,\s*}/g, '}') // Remove trailing commas before closing braces
+            .replace(/,\s*]/g, ']') // Remove trailing commas before closing brackets
+            .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":'); // Ensure property names are quoted
+    }
+
+    /**
+     * Log prompt and response to file
+     * @param {string} prompt - Prompt text
+     * @param {Object} response - Response object
+     */
+    async _logPromptAndResponse(prompt, response) {
+        const content = '================================================================================'
+            + prompt + "\n\n"
+            + JSON.stringify(response)
+            + '\n\n'
+            + '================================================================================\n\n';
+
+        await writePromptToFile(content);
+    }
+
+    /**
+     * Generate text based on a prompt
+     * @param {string} prompt - The prompt to generate text from
+     * @returns {Promise<string>} - The generated text
+     */
+    async generateText(prompt) {
+        try {
+            // Calculate context window size based on prompt length
+            const promptTokenCount = this._calculatePromptTokenCount(prompt);
+            const numCtx = this._calculateNumCtx(promptTokenCount, Number(config.responseTokens));
+
+            // Simple system prompt for text generation
+            const systemPrompt = `You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.`;
+
+            // Call Ollama API without enforcing a specific response format
+            const generateTextBody = {
+                model: this.model,
+                prompt: prompt,
+                system: systemPrompt,
+                stream: false,
+                options: {
+                    temperature: config.aiTemperatureGeneration,
+                    top_p: 0.9,
+                    num_predict: 1024,
+                    num_ctx: numCtx
+                }
+            };
+
+            // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
+            // unless explicitly enabled via OLLAMA_THINK=true
+            if (!config.ollama.think) {
+                generateTextBody.think = false;
+            }
 
-      if (!response.data || !response.data.response) {
-        throw new Error('Invalid response from Ollama API');
-      }
+            const response = await this.client.post(`${this.apiUrl}/api/generate`, generateTextBody, {
+                headers: this._buildRequestHeaders()
+            });
 
-      return response.data.response;
-    } catch (error) {
-      console.error(`Error generating text with Ollama: ${error.message}`);
-      console.debug(error);
-      throw error;
+            if (!response.data || !response.data.response) {
+                throw new Error('Invalid response from Ollama API');
+            }
+
+            return response.data.response;
+        } catch (error) {
+             console.error(`Error generating text with Ollama: ${error.message}`); console.debug(error);
+            throw error;
+        }
     }
-  }
-
-  /**
-   * Check if the Ollama service is running
-   * @returns {Promise<boolean>} - True if the service is running, false otherwise
-   */
-  async checkStatus() {
-    // use ollama status endpoint
-    try {
-      const response = await this.client.get(`${this.apiUrl}/api/ps`, {
-        headers: this._buildRequestHeaders(),
-      });
-      if (response.status === 200) {
-        const data = response.data;
-        // Ensure data is an array and has at least one model
-        let modelName = null;
-        if (Array.isArray(data.models) && data.models.length > 0) {
-          modelName = data.models[0].name;
+
+    /**
+     * Check if the Ollama service is running
+     * @returns {Promise<boolean>} - True if the service is running, false otherwise
+     */
+    async checkStatus() {
+        // use ollama status endpoint
+        try {
+            const response = await this.client.get(`${this.apiUrl}/api/ps`, {
+                headers: this._buildRequestHeaders()
+            });
+            if (response.status === 200) {
+                const data = response.data;
+                // Ensure data is an array and has at least one model
+                let modelName = null;
+                if (Array.isArray(data.models) && data.models.length > 0) {
+                    modelName = data.models[0].name;
+                }
+                console.log('Ollama model name:', modelName);
+                return { status: 'ok', model: modelName };
+            }
+        } catch (error) {
+            console.error(`Error checking Ollama service status: ${error.message}`);
+            console.debug(error);
         }
-        console.log('Ollama model name:', modelName);
-        return { status: 'ok', model: modelName };
-      }
-    } catch (error) {
-      console.error(`Error checking Ollama service status: ${error.message}`);
-      console.debug(error);
+        return { status: 'error' };
     }
-    return { status: 'error' };
-  }
 }
 
 module.exports = new OllamaService();
diff --git a/services/paperlessService.js b/services/paperlessService.js
index 2bd65896..91466ed8 100644
--- a/services/paperlessService.js
+++ b/services/paperlessService.js
@@ -271,6 +271,7 @@ class PaperlessService {
     }
   }
 
+
   // Aktualisiert den Tag-Cache, wenn er älter als CACHE_LIFETIME ist
   async ensureTagCache() {
     const now = Date.now();
@@ -279,12 +280,8 @@ class PaperlessService {
       if (this._refreshPromise) {
         return this._refreshPromise;
       }
-      const expireTime = new Date(
-        this.lastTagRefresh + this.CACHE_LIFETIME
-      ).toISOString();
-      console.log(
-        `[DEBUG] Tag cache expired (age: ${Math.floor(cacheAge / 1000)}s, TTL: ${Math.floor(this.CACHE_LIFETIME / 1000)}s, expired at: ${expireTime})`
-      );
+      const expireTime = new Date(this.lastTagRefresh + this.CACHE_LIFETIME).toISOString();
+      console.log(`[DEBUG] Tag cache expired (age: ${Math.floor(cacheAge / 1000)}s, TTL: ${Math.floor(this.CACHE_LIFETIME / 1000)}s, expired at: ${expireTime})`);
       // No race condition: synchronous code is never preempted in Node.js's
       // event loop, so no other call can reach here between the check above
       // and the assignment below.
@@ -308,92 +305,105 @@ class PaperlessService {
 
   // Lädt alle existierenden Tags
   async refreshTagCache() {
-    try {
-      console.log('[DEBUG] Refreshing tag cache...');
-      this.tagCache.clear();
-      let nextUrl = '/tags/';
-      while (nextUrl) {
-        const response = await this.client.get(nextUrl);
-
-        // Validate response structure
-        if (!response?.data?.results) {
-          console.error(
-            '[ERROR] Invalid response structure from API:',
-            response?.data
-          );
-          break;
-        }
+      try {
+        console.log('[DEBUG] Refreshing tag cache...');
+        this.tagCache.clear();
+        let nextUrl = '/tags/';
+        while (nextUrl) {
+          const response = await this.client.get(nextUrl);
+
+          // Validate response structure
+          if (!response?.data?.results) {
+            console.error('[ERROR] Invalid response structure from API:', response?.data);
+            break;
+          }
 
-        response.data.results.forEach((tag) => {
-          this.tagCache.set(tag.name.toLowerCase(), tag);
-        });
+          response.data.results.forEach(tag => {
+            this.tagCache.set(tag.name.toLowerCase(), tag);
+          });
 
-        // Safely extract relative path from next URL to prevent SSRF
-        if (response.data.next) {
-          nextUrl = this._safeExtractRelativePath(response.data.next);
-          if (nextUrl) {
-            console.log('[DEBUG] Next page URL:', nextUrl);
+          // Safely extract relative path from next URL to prevent SSRF
+          if (response.data.next) {
+            nextUrl = this._safeExtractRelativePath(response.data.next);
+            if (nextUrl) {
+              console.log('[DEBUG] Next page URL:', nextUrl);
+            }
+          } else {
+            nextUrl = null;
           }
-        } else {
-          nextUrl = null;
         }
+        this.lastTagRefresh = Date.now();
+        console.log(`[DEBUG] Tag cache refreshed. Found ${this.tagCache.size} tags.`);
+      } catch (error) {
+        console.error('[ERROR] refreshing tag cache:', error.message);
+        throw error;
       }
-      this.lastTagRefresh = Date.now();
-      console.log(
-        `[DEBUG] Tag cache refreshed. Found ${this.tagCache.size} tags.`
-      );
-    } catch (error) {
-      console.error('[ERROR] refreshing tag cache:', error.message);
-      throw error;
     }
-  }
 
   async initializeWithCredentials(apiUrl, apiToken) {
-    const baseUrl = stripTrailingSlashes(apiUrl);
     this.client = axios.create({
-      baseURL: baseUrl + '/api',
+      baseURL: apiUrl.replace(/\/+$/, '') + '/api',
       headers: {
-        Authorization: `Token ${apiToken}`,
-        'Content-Type': 'application/json',
-      },
-      beforeRedirect: createRedirectGuard(() => baseUrl),
+        'Authorization': `Token ${apiToken}`,
+        'Content-Type': 'application/json'
+      }
     });
-
+    
     // Test the connection
     try {
       await this.client.get('/');
       return true;
     } catch (error) {
-      console.error(
-        '[ERROR] Failed to initialize with credentials:',
-        error.message
-      );
+      console.error('[ERROR] Failed to initialize with credentials:', error.message);
       this.client = null;
       return false;
     }
   }
 
   async createCustomFieldSafely(fieldName, fieldType, default_currency) {
+    const normalizedFieldType = String(fieldType || '').trim();
+
+    const existingField = await this.findExistingCustomField(fieldName);
+    if (existingField) {
+      const existingFieldType = String(existingField.data_type || '').trim();
+      if (existingFieldType !== normalizedFieldType) {
+        const error = new Error(
+          `Custom field "${fieldName}" has type "${existingFieldType}" in Paperless, ` +
+          `but configuration requests "${normalizedFieldType}".`
+        );
+        error.code = 'CUSTOM_FIELD_TYPE_MISMATCH';
+        throw error;
+      }
+      return existingField;
+    }
+
     try {
       // Try to create the field first
-      const response = await this.client.post('/custom_fields/', {
+      const response = await this.client.post('/custom_fields/', { 
         name: fieldName,
-        data_type: fieldType,
+        data_type: normalizedFieldType,
         extra_data: {
-          default_currency: default_currency || null,
-        },
+          default_currency: default_currency || null
+        }
       });
       const newField = response.data;
-      console.log(
-        `[DEBUG] Successfully created custom field "${fieldName}" with ID ${newField.id}`
-      );
+      console.log(`[DEBUG] Successfully created custom field "${fieldName}" with ID ${newField.id}`);
       this.customFieldCache.set(fieldName.toLowerCase(), newField);
       return newField;
-    } catch (error) {
+    } catch (error) { 
       if (error.response?.status === 400) {
         await this.refreshCustomFieldCache();
         const existingField = await this.findExistingCustomField(fieldName);
         if (existingField) {
+          const existingFieldType = String(existingField.data_type || '').trim();
+          if (existingFieldType !== normalizedFieldType) {
+            const mismatchError = new Error(
+              `Custom field "${fieldName}" has type "${existingFieldType}", ` +
+              `but configuration requests "${normalizedFieldType}".`
+            );
+            mismatchError.code = 'CUSTOM_FIELD_TYPE_MISMATCH';
+            throw mismatchError;
+          }
           return existingField;
         }
       }
@@ -404,104 +414,87 @@ class PaperlessService {
   async getExistingCustomFields(documentId) {
     try {
       const response = await this.client.get(`/documents/${documentId}/`);
-      console.log(
-        '[DEBUG] Document response custom fields:',
-        response.data.custom_fields
-      );
+      console.log('[DEBUG] Document response custom fields:', response.data.custom_fields);
       return response.data.custom_fields || [];
     } catch (error) {
-      console.error('[ERROR] fetching document %s:', documentId, error.message);
+      console.error(`[ERROR] fetching document ${documentId}:`, error.message);
       return [];
     }
   }
-
+  
   async findExistingCustomField(fieldName) {
     const normalizedName = fieldName.toLowerCase();
-
+    
     const cachedField = this.customFieldCache.get(normalizedName);
     if (cachedField) {
-      console.log(
-        `[DEBUG] Found custom field "${fieldName}" in cache with ID ${cachedField.id}`
-      );
+      console.log(`[DEBUG] Found custom field "${fieldName}" in cache with ID ${cachedField.id}`);
       return cachedField;
     }
 
     try {
       const response = await this.client.get('/custom_fields/', {
         params: {
-          name__iexact: normalizedName, // Case-insensitive exact match
-        },
+          name__iexact: normalizedName  // Case-insensitive exact match
+        }
       });
 
       if (response.data.results.length > 0) {
         const foundField = response.data.results[0];
-        console.log(
-          `[DEBUG] Found existing custom field "${fieldName}" via API with ID ${foundField.id}`
-        );
+        console.log(`[DEBUG] Found existing custom field "${fieldName}" via API with ID ${foundField.id}`);
         this.customFieldCache.set(normalizedName, foundField);
         return foundField;
       }
     } catch (error) {
-      console.warn(
-        '[ERROR] searching for custom field "%s":',
-        fieldName,
-        error.message
-      );
+      console.warn(`[ERROR] searching for custom field "${fieldName}":`, error.message);
     }
 
     return null;
   }
 
   async refreshCustomFieldCache() {
-    try {
-      console.log('[DEBUG] Refreshing custom field cache...');
-      this.customFieldCache.clear();
-      let nextUrl = '/custom_fields/';
-      while (nextUrl) {
-        const response = await this.client.get(nextUrl);
-
-        // Validate response structure
-        if (!response?.data?.results) {
-          console.error(
-            '[ERROR] Invalid response structure from API:',
-            response?.data
-          );
-          break;
-        }
+      try {
+        console.log('[DEBUG] Refreshing custom field cache...');
+        this.customFieldCache.clear();
+        let nextUrl = '/custom_fields/';
+        while (nextUrl) {
+          const response = await this.client.get(nextUrl);
+
+          // Validate response structure
+          if (!response?.data?.results) {
+            console.error('[ERROR] Invalid response structure from API:', response?.data);
+            break;
+          }
 
-        response.data.results.forEach((field) => {
-          this.customFieldCache.set(field.name.toLowerCase(), field);
-        });
+          response.data.results.forEach(field => {
+            this.customFieldCache.set(field.name.toLowerCase(), field);
+          });
 
-        // Safely extract relative path from next URL to prevent SSRF
-        if (response.data.next) {
-          nextUrl = this._safeExtractRelativePath(response.data.next);
-          if (nextUrl) {
-            console.log('[DEBUG] Next page URL:', nextUrl);
+          // Safely extract relative path from next URL to prevent SSRF
+          if (response.data.next) {
+            nextUrl = this._safeExtractRelativePath(response.data.next);
+            if (nextUrl) {
+              console.log('[DEBUG] Next page URL:', nextUrl);
+            }
+          } else {
+            nextUrl = null;
           }
-        } else {
-          nextUrl = null;
         }
+        this.lastCustomFieldRefresh = Date.now();
+        console.log(`[DEBUG] Custom field cache refreshed. Found ${this.customFieldCache.size} fields.`);
+      } catch (error) {
+        console.error('[ERROR] refreshing custom field cache:', error.message);
+        throw error;
       }
-      this.lastCustomFieldRefresh = Date.now();
-      console.log(
-        `[DEBUG] Custom field cache refreshed. Found ${this.customFieldCache.size} fields.`
-      );
-    } catch (error) {
-      console.error('[ERROR] refreshing custom field cache:', error.message);
-      throw error;
     }
-  }
+
 
   async findExistingTag(tagName) {
     const normalizedName = tagName.toLowerCase();
-
+    
     // 1. Zuerst im Cache suchen
     const cachedTag = this.tagCache.get(normalizedName);
     if (cachedTag) {
-      console.log(
-        `[DEBUG] Found tag "${tagName}" in cache with ID ${cachedTag.id}`
-      );
+      console.log(`[DEBUG] Found tag "${tagName}" in cache with ID ${cachedTag.id}`);
       return cachedTag;
     }
 
@@ -509,20 +502,18 @@ class PaperlessService {
     try {
       const response = await this.client.get('/tags/', {
         params: {
-          name__iexact: normalizedName, // Case-insensitive exact match
-        },
+          name__iexact: normalizedName  // Case-insensitive exact match
+        }
       });
 
       if (response.data.results.length > 0) {
         const foundTag = response.data.results[0];
-        console.log(
-          `[DEBUG] Found existing tag "${tagName}" via API with ID ${foundTag.id}`
-        );
+        console.log(`[DEBUG] Found existing tag "${tagName}" via API with ID ${foundTag.id}`);
         this.tagCache.set(normalizedName, foundTag);
         return foundTag;
       }
     } catch (error) {
-      console.warn('[ERROR] searching for tag "%s":', tagName, error.message);
+      console.warn(`[ERROR] searching for tag "${tagName}":`, error.message);
     }
 
     return null;
@@ -530,14 +521,12 @@ class PaperlessService {
 
   async createTagSafely(tagName) {
     const normalizedName = tagName.toLowerCase();
-
+    
     try {
       // Versuche zuerst, den Tag zu erstellen
       const response = await this.client.post('/tags/', { name: tagName });
       const newTag = response.data;
-      console.log(
-        `[DEBUG] Successfully created tag "${tagName}" with ID ${newTag.id}`
-      );
+      console.log(`[DEBUG] Successfully created tag "${tagName}" with ID ${newTag.id}`);
       this.tagCache.set(normalizedName, newTag);
       // Invalidate cache after creating new tag to ensure consistency
       this.lastTagRefresh = 0;
@@ -547,7 +536,7 @@ class PaperlessService {
         // Bei einem 400er Fehler könnte der Tag bereits existieren
         // Aktualisiere den Cache und suche erneut
         await this.refreshTagCache();
-
+        
         // Suche nochmal nach dem Tag
         const existingTag = await this.findExistingTag(tagName);
         if (existingTag) {
@@ -562,14 +551,13 @@ class PaperlessService {
     try {
       this.initialize();
       await this.ensureTagCache();
-
+      
       // Check if we should restrict to existing tags
       // Explicitly check options first, then env var
-      const restrictToExistingTags =
-        options.restrictToExistingTags === true ||
-        (options.restrictToExistingTags === undefined &&
-          process.env.RESTRICT_TO_EXISTING_TAGS === 'yes');
-
+      const restrictToExistingTags = options.restrictToExistingTags === true || 
+                                   (options.restrictToExistingTags === undefined && 
+                                    process.env.RESTRICT_TO_EXISTING_TAGS === 'yes');
+      
       // Input validation
       if (!tagNames) {
         console.warn('[DEBUG] No tags provided to processTags');
@@ -577,26 +565,23 @@ class PaperlessService {
       }
 
       // Convert to array if string is passed
-      const tagsArray =
-        typeof tagNames === 'string'
-          ? [tagNames]
-          : Array.isArray(tagNames)
-            ? tagNames
-            : [];
+      const tagsArray = typeof tagNames === 'string' 
+        ? [tagNames]
+        : Array.isArray(tagNames) 
+          ? tagNames 
+          : [];
 
       if (tagsArray.length === 0) {
         console.warn('[DEBUG] No valid tags to process');
         return { tagIds: [], errors: [] };
       }
-
+  
       const tagIds = [];
       const errors = [];
       const processedTags = new Set(); // Prevent duplicates
-
-      console.log(
-        `[DEBUG] Processing tags with restrictToExistingTags=${restrictToExistingTags}`
-      );
-
+      
+      console.log(`[DEBUG] Processing tags with restrictToExistingTags=${restrictToExistingTags}`);
+  
       // Process regular tags
       for (const tagName of tagsArray) {
         if (!tagName || typeof tagName !== 'string') {
@@ -604,79 +589,64 @@ class PaperlessService {
           errors.push({ tagName, error: 'Invalid tag name' });
           continue;
         }
-
+  
         const normalizedName = tagName.toLowerCase().trim();
-
+        
         // Skip empty or already processed tags
         if (!normalizedName || processedTags.has(normalizedName)) {
           continue;
         }
-
+  
         try {
           // Search for existing tag first
           let tag = await this.findExistingTag(tagName);
-
+          
           // If no existing tag found and restrictions are not enabled, create new one
           if (!tag && !restrictToExistingTags) {
             tag = await this.createTagSafely(tagName);
           } else if (!tag && restrictToExistingTags) {
-            console.log(
-              `[DEBUG] Tag "${tagName}" does not exist and restrictions are enabled, skipping`
-            );
-            errors.push({
-              tagName,
-              error: 'Tag does not exist and restrictions are enabled',
-            });
+            console.log(`[DEBUG] Tag "${tagName}" does not exist and restrictions are enabled, skipping`);
+            errors.push({ tagName, error: 'Tag does not exist and restrictions are enabled' });
             continue;
           }
-
+  
           if (tag && tag.id) {
             tagIds.push(tag.id);
             processedTags.add(normalizedName);
           }
+  
         } catch (error) {
-          console.error('[ERROR] processing tag "%s":', tagName, error.message);
+          console.error(`[ERROR] processing tag "${tagName}":`, error.message);
           errors.push({ tagName, error: error.message });
         }
       }
-
+  
       // Add AI-Processed tag if enabled
-      if (
-        process.env.ADD_AI_PROCESSED_TAG === 'yes' &&
-        process.env.AI_PROCESSED_TAG_NAME
-      ) {
+      if (process.env.ADD_AI_PROCESSED_TAG === 'yes' && process.env.AI_PROCESSED_TAG_NAME) {
         try {
           const aiTagName = process.env.AI_PROCESSED_TAG_NAME;
           let aiTag = await this.findExistingTag(aiTagName);
-
+          
           if (!aiTag) {
             aiTag = await this.createTagSafely(aiTagName);
           }
-
+  
           if (aiTag && aiTag.id) {
             tagIds.push(aiTag.id);
           }
         } catch (error) {
-          console.error(
-            `[ERROR] processing AI tag "${process.env.AI_PROCESSED_TAG_NAME}":`,
-            error.message
-          );
-          errors.push({
-            tagName: process.env.AI_PROCESSED_TAG_NAME,
-            error: error.message,
-          });
+          console.error(`[ERROR] processing AI tag "${process.env.AI_PROCESSED_TAG_NAME}":`, error.message);
+          errors.push({ tagName: process.env.AI_PROCESSED_TAG_NAME, error: error.message });
         }
       }
-
-      return {
+  
+      return { 
         tagIds: [...new Set(tagIds)], // Remove any duplicates
-        errors,
-      };
+        errors 
+      };      
     } catch (error) {
       console.error('[ERROR] in processTags:', error);
-      throw new Error(`[ERROR] Failed to process tags: ${error.message}`, {
-        cause: error,
-      });
+      throw new Error(`[ERROR] Failed to process tags: ${error.message}`);
     }
   }
 
@@ -733,7 +703,8 @@ class PaperlessService {
         );
 
         // Kleine Verzögerung um die API nicht zu überlasten
-        await new Promise((resolve) => setTimeout(resolve, 100));
+        await new Promise(resolve => setTimeout(resolve, 100));
+
       } catch (error) {
         console.error(`[ERRRO] fetching tags page ${page}:`, error.message);
         if (error.response) {
@@ -808,7 +779,7 @@ class PaperlessService {
     let allCorrespondents = [];
     let page = 1;
     let hasNextPage = true;
-
+  
     try {
       while (hasNextPage) {
         const response = await this.client.get('/correspondents/', {
@@ -821,7 +792,7 @@ class PaperlessService {
         });
 
         const { results, next } = response.data;
-
+        
         // Füge die Ergebnisse der aktuellen Seite hinzu
         allCorrespondents = allCorrespondents.concat(
           results.map((correspondent) => ({
@@ -842,6 +813,7 @@ class PaperlessService {
       }
 
       return allCorrespondents;
+  
     } catch (error) {
       console.error('[ERROR] fetching correspondent names:', error.message);
       return [];
@@ -983,13 +955,9 @@ class PaperlessService {
         return true;
       }
 
-      const hasExcludedTag = document.tags.some((tag) => {
-        const normalizedTagId =
-          typeof tag === 'object' ? Number(tag?.id) : Number(tag);
-        return (
-          Number.isInteger(normalizedTagId) &&
-          excludedTagSet.has(normalizedTagId)
-        );
+      const hasExcludedTag = document.tags.some(tag => {
+        const normalizedTagId = typeof tag === 'object' ? Number(tag?.id) : Number(tag);
+        return Number.isInteger(normalizedTagId) && excludedTagSet.has(normalizedTagId);
       });
 
       return !hasExcludedTag;
@@ -1012,23 +980,16 @@ class PaperlessService {
     let documents = [];
     let page = 1;
     let hasMore = true;
-    const shouldFilterByTags =
-      applyFilters && process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
-    const includeTagNames = applyFilters
-      ? this.parseTagList(process.env.TAGS)
-      : [];
-    const excludeTagNames = applyFilters
-      ? this.parseTagList(process.env.IGNORE_TAGS)
-      : [];
+    const shouldFilterByTags = applyFilters && process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
+    const includeTagNames = applyFilters ? this.parseTagList(process.env.TAGS) : [];
+    const excludeTagNames = applyFilters ? this.parseTagList(process.env.IGNORE_TAGS) : [];
     let includeTagIds = [];
     let excludeTagIds = [];
 
     // Vorverarbeitung der Include-Tags, wenn Filter aktiv ist
     if (shouldFilterByTags) {
       if (includeTagNames.length === 0) {
-        console.warn(
-          '[DEBUG] PROCESS_PREDEFINED_DOCUMENTS is set to yes but no TAGS are defined'
-        );
+        console.warn('[DEBUG] PROCESS_PREDEFINED_DOCUMENTS is set to yes but no TAGS are defined');
         return [];
       }
 
@@ -1039,18 +1000,13 @@ class PaperlessService {
         return [];
       }
 
-      console.log(
-        '[DEBUG] Filtering documents for include tag IDs:',
-        includeTagIds
-      );
+      console.log('[DEBUG] Filtering documents for include tag IDs:', includeTagIds);
     }
 
     if (excludeTagNames.length > 0) {
       excludeTagIds = await this.resolveTagIdsByName(excludeTagNames);
       if (excludeTagIds.length === 0) {
-        console.warn(
-          '[DEBUG] IGNORE_TAGS configured but no matching tags were found'
-        );
+        console.warn('[DEBUG] IGNORE_TAGS configured but no matching tags were found');
       } else {
         console.log('[DEBUG] Excluding documents with tag IDs:', excludeTagIds);
       }
@@ -1061,10 +1017,8 @@ class PaperlessService {
         const params = {
           page,
           page_size: 100,
-          fields:
-            options.fields ||
-            'id,title,created,created_date,added,tags,correspondent',
-          ordering: 'id',
+          fields: options.fields || 'id,title,created,created_date,added,tags,correspondent',
+          ordering: 'id'
         };
 
         // Füge Tag-Filter hinzu, wenn Tags definiert sind
@@ -1073,7 +1027,7 @@ class PaperlessService {
         }
 
         const response = await this.client.get('/documents/', { params });
-
+        
         if (!response?.data?.results || !Array.isArray(response.data.results)) {
           console.error(`[DEBUG] Invalid API response on page ${page}`);
           break;
@@ -1084,17 +1038,15 @@ class PaperlessService {
         page++;
 
         console.log(
-          `[DEBUG] Fetched page ${page - 1}, got ${response.data.results.length} documents. ` +
-            `[DEBUG] Total so far: ${documents.length}`
+          `[DEBUG] Fetched page ${page-1}, got ${response.data.results.length} documents. ` +
+          `[DEBUG] Total so far: ${documents.length}`
         );
 
         // Kleine Verzögerung um die API nicht zu überlasten
-        await new Promise((resolve) => setTimeout(resolve, 100));
+        await new Promise(resolve => setTimeout(resolve, 100));
+
       } catch (error) {
-        console.error(
-          `[ERROR]  fetching documents page ${page}:`,
-          error.message
-        );
+        console.error(`[ERROR]  fetching documents page ${page}:`, error.message);
         if (error.response) {
           console.error('[ERROR] Response status:', error.response.status);
         }
@@ -1102,32 +1054,26 @@ class PaperlessService {
       }
     }
 
-    const filteredDocuments = this.filterDocumentsByExcludedTagIds(
-      documents,
-      excludeTagIds
-    );
+    const filteredDocuments = this.filterDocumentsByExcludedTagIds(documents, excludeTagIds);
     if (excludeTagIds.length > 0) {
       console.log(
         `[DEBUG] Exclude filter removed ${documents.length - filteredDocuments.length} documents. ` +
-          `[DEBUG] Remaining: ${filteredDocuments.length}`
+        `[DEBUG] Remaining: ${filteredDocuments.length}`
       );
     }
 
-    console.log(
-      `[DEBUG] Finished fetching. Found ${filteredDocuments.length} documents after filtering.`
-    );
+    console.log(`[DEBUG] Finished fetching. Found ${filteredDocuments.length} documents after filtering.`);
     return filteredDocuments;
-  }
+}
 
   async getEffectiveDocumentCount() {
-    const shouldFilterByTags =
-      process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
+    const shouldFilterByTags = process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
     const includeTagNames = this.parseTagList(process.env.TAGS);
     const excludeTagNames = this.parseTagList(process.env.IGNORE_TAGS);
     const cacheKey = JSON.stringify({
       shouldFilterByTags,
       includeTagNames,
-      excludeTagNames,
+      excludeTagNames
     });
 
     if (
@@ -1168,7 +1114,7 @@ class PaperlessService {
         try {
           effectiveCount = await this.getDocumentCountByParams({
             ...baseCountParams,
-            tags__id__none: excludeTagIds.join(','),
+            tags__id__none: excludeTagIds.join(',')
           });
           this._supportsTagsIdNone = true;
         } catch (error) {
@@ -1184,9 +1130,7 @@ class PaperlessService {
         if (excludeTagIds.length === 0) {
           effectiveCount = await this.getDocumentCountByParams(baseCountParams);
         } else {
-          const processableDocuments = await this.getAllDocuments({
-            fields: 'id,tags',
-          });
+          const processableDocuments = await this.getAllDocuments({ fields: 'id,tags' });
           effectiveCount = processableDocuments.length;
         }
       }
@@ -1194,23 +1138,44 @@ class PaperlessService {
       this._effectiveCountCache = {
         key: cacheKey,
         count: effectiveCount,
-        expiresAt: Date.now() + this._effectiveCountCacheTtlMs,
+        expiresAt: Date.now() + this._effectiveCountCacheTtlMs
       };
 
       return effectiveCount;
     } catch (error) {
-      console.error(
-        '[ERROR] fetching effective document count:',
-        error.message
-      );
+      console.error('[ERROR] fetching effective document count:', error.message);
       return 0;
     }
   }
 
+  async getAllDocumentIds() {
+    /**
+     * Get all Document IDs from the Paperless API.
+     * 
+     * @returns    An array of all Document IDs.
+     * @throws     An error if the request fails.
+     * @note       This method is used to get all Document IDs for further processing.
+     */
+    this.initialize();
+    try {
+      const response = await this.client.get('/documents/', {
+        params: { 
+          page,
+          page_size: 100,
+          fields: 'id',
+        }
+      });
+      return response.data.results.map(doc => doc.id);
+    } catch (error) {
+      console.error('[ERROR] fetching document IDs:', error.message);
+      return [];
+    }
+  }
+
   async getAllDocumentIdsScan() {
     /**
      * Get all Document IDs from the Paperless API.
-     *
+     * 
      * @returns    An array of all Document IDs.
      * @throws     An error if the request fails.
      * @note       This method is used to get all Document IDs for further processing.
@@ -1224,35 +1189,32 @@ class PaperlessService {
     let documents = [];
     let page = 1;
     let hasMore = true;
-    const shouldFilterByTags =
-      process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
+    const shouldFilterByTags = process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
     let tagIds = [];
 
     // Vorverarbeitung der Tags, wenn Filter aktiv ist
     if (shouldFilterByTags) {
       if (!process.env.TAGS) {
-        console.warn(
-          '[DEBUG] PROCESS_PREDEFINED_DOCUMENTS is set to yes but no TAGS are defined'
-        );
+        console.warn('[DEBUG] PROCESS_PREDEFINED_DOCUMENTS is set to yes but no TAGS are defined');
         return [];
       }
-
+      
       // Hole die Tag-IDs für die definierten Tags
-      const tagNames = process.env.TAGS.split(',').map((tag) => tag.trim());
+      const tagNames = process.env.TAGS.split(',').map(tag => tag.trim());
       await this.ensureTagCache();
-
+      
       for (const tagName of tagNames) {
         const tag = await this.findExistingTag(tagName);
         if (tag) {
           tagIds.push(tag.id);
         }
       }
-
+      
       if (tagIds.length === 0) {
         console.warn('[DEBUG] None of the specified tags were found');
         return [];
       }
-
+      
       console.log('[DEBUG] Filtering documents for tag IDs:', tagIds);
     }
 
@@ -1262,11 +1224,11 @@ class PaperlessService {
           page,
           page_size: 100,
           fields: 'id',
-          ordering: 'id',
+          ordering: 'id'
         };
 
         const response = await this.client.get('/documents/', { params });
-
+        
         if (!response?.data?.results || !Array.isArray(response.data.results)) {
           console.error(`[ERROR] Invalid API response on page ${page}`);
           break;
@@ -1277,17 +1239,15 @@ class PaperlessService {
         page++;
 
         console.log(
-          `[DEBUG] Fetched page ${page - 1}, got ${response.data.results.length} documents. ` +
-            `[DEBUG] Total so far: ${documents.length}`
+          `[DEBUG] Fetched page ${page-1}, got ${response.data.results.length} documents. ` +
+          `[DEBUG] Total so far: ${documents.length}`
         );
 
         // Kleine Verzögerung um die API nicht zu überlasten
-        await new Promise((resolve) => setTimeout(resolve, 100));
+        await new Promise(resolve => setTimeout(resolve, 100));
+
       } catch (error) {
-        console.error(
-          `[ERROR] fetching documents page ${page}:`,
-          error.message
-        );
+        console.error(`[ERROR] fetching documents page ${page}:`, error.message);
         if (error.response) {
           console.error('[DEBUG] Response status:', error.response.status);
         }
@@ -1295,473 +1255,122 @@ class PaperlessService {
       }
     }
 
-    console.log(
-      `[DEBUG] Finished fetching. Found ${documents.length} documents.`
-    );
+    console.log(`[DEBUG] Finished fetching. Found ${documents.length} documents.`);
     return documents;
   }
 
-  /**
-   * Normalize a mixed list of IDs into unique, positive integers.
-   * Guards against `Number(null) === 0`, which would otherwise turn documents
-   * without a correspondent into a lookup for the non-existent ID 0.
-   */
-  _normalizeEntityIds(ids) {
-    if (!Array.isArray(ids)) {
-      return [];
-    }
-
-    const normalized = new Set();
-    for (const rawId of ids) {
-      const id = typeof rawId === 'object' ? Number(rawId?.id) : Number(rawId);
-      if (Number.isInteger(id) && id > 0) {
-        normalized.add(id);
-      }
+  async getCorrespondentNameById(correspondentId) {
+    /**
+     * Get the Name of a Correspondent by its ID.
+     * 
+     * @param   id  The id of the correspondent.
+     * @returns    The name of the correspondent.
+     */
+    this.initialize();
+    try {
+      const response = await this.client.get(`/correspondents/${correspondentId}/`);
+      return response.data;
+    } catch (error) {
+      console.error(`[ERROR] fetching correspondent ${correspondentId}:`, error.message);
+      return null;
     }
-
-    return [...normalized];
   }
 
-  /**
-   * Resolve tag IDs to names using the shared tag cache.
-   *
-   * The cache is keyed by tag name, so the reverse lookup is built on the fly.
-   * This costs at most one paginated refresh per CACHE_LIFETIME instead of one
-   * detail request per tag.
-   *
-   * @param   tagIds  Tag IDs to resolve.
-   * @returns         A plain object mapping tag ID to tag name.
-   */
-  async getTagNamesByIds(tagIds = []) {
+  async getDocumentTypeNameById(documentTypeId) {
     this.initialize();
-    const uniqueTagIds = this._normalizeEntityIds(tagIds);
-    if (uniqueTagIds.length === 0 || !this.client) {
-      return {};
-    }
-
     try {
-      await this.ensureTagCache();
+      const response = await this.client.get(`/document_types/${documentTypeId}/`);
+      return response.data;
     } catch (error) {
-      console.error('[ERROR] resolving tag names from cache:', error.message);
-      return {};
-    }
-
-    const namesById = new Map();
-    for (const tag of this.tagCache.values()) {
-      const id = Number(tag?.id);
-      if (Number.isInteger(id) && typeof tag?.name === 'string') {
-        namesById.set(id, tag.name);
-      }
-    }
-
-    const resolved = {};
-    for (const tagId of uniqueTagIds) {
-      const name = namesById.get(tagId);
-      if (name) {
-        resolved[tagId] = name;
-      }
+      console.error(`[ERROR] fetching document type ${documentTypeId}:`, error.message);
+      return null;
     }
-
-    return resolved;
   }
 
-  /**
-   * Resolve correspondent IDs to names, batching every cache miss into a single
-   * filtered list request instead of one detail request per correspondent.
-   *
-   * @param   correspondentIds  Correspondent IDs to resolve.
-   * @returns                   A plain object mapping correspondent ID to name.
-   */
-  async getCorrespondentNamesByIds(correspondentIds = []) {
+  async getTagNameById(tagId) {
+    /**
+     * Get the Name of a Tag by its ID.
+     *
+     * @param   id  The id of the tag.
+     * @returns    The name of the tag.
+     */
     this.initialize();
-    const uniqueIds = this._normalizeEntityIds(correspondentIds);
-    if (uniqueIds.length === 0 || !this.client) {
-      return {};
-    }
-
-    // The cache holds a whole generation of names; drop it once it is stale
-    // rather than tracking a timestamp per entry.
-    if (Date.now() - this.lastCorrespondentRefresh > this.CACHE_LIFETIME) {
-      this.correspondentNameCache.clear();
-      this.lastCorrespondentRefresh = Date.now();
-    }
-
-    const missingIds = uniqueIds.filter(
-      (id) => !this.correspondentNameCache.has(id)
-    );
-
-    if (missingIds.length > 0) {
-      await this._loadCorrespondentNames(missingIds);
+    try {
+      const response = await this.client.get(`/tags/${tagId}/`);
+      return response.data.name;
+    } catch (error) {
+      console.error(`[ERROR] fetching tag name for ID ${tagId}:`, error.message);
+      return null;
     }
+  }
 
-    const resolved = {};
-    for (const id of uniqueIds) {
-      const name = this.correspondentNameCache.get(id);
-      if (name) {
-        resolved[id] = name;
-      }
+  async getDocumentsWithTitleTagsCorrespondentCreated () {
+    /**
+     * Get all documents with metadata (title, tags, correspondent, created date).
+     * 
+     * @returns    An array of documents with metadata.
+     * @throws     An error if the request fails.
+     * @note       This method is used to get all documents with metadata for further processing 
+     */
+    
+    this.initialize();
+    try {
+      const response = await this.client.get('/documents/', {
+        params: {
+          fields: 'id,title,tags,correspondent,created'
+        }
+      });
+      return response.data.results;
+    } catch (error) {
+      console.error('[ERROR] fetching documents with metadata:', error.message);
+      return [];
     }
-
-    return resolved;
   }
 
-  /**
-   * Populate the correspondent name cache for the given IDs.
-   * Prefers the `id__in` filter and falls back to a full listing when the
-   * Paperless-ngx instance does not support it.
-   */
-  async _loadCorrespondentNames(missingIds) {
-    let loaded;
+  async getRecentDocumentsWithMetadata(limit = 16) {
+    this.initialize();
 
-    if (this._supportsCorrespondentIdIn === false) {
-      loaded = await this._primeCorrespondentNameCache();
-    } else {
-      loaded = true;
-      for (let index = 0; index < missingIds.length; index += 100) {
-        const chunk = missingIds.slice(index, index + 100);
-        const applied = await this._fetchCorrespondentNameChunk(chunk);
-        if (!applied) {
-          // `id__in` is unsupported or failed - load everything once instead.
-          loaded = await this._primeCorrespondentNameCache();
-          break;
+    const safeLimit = Number.isInteger(Number(limit)) ? Math.max(1, Math.min(Number(limit), 100)) : 16;
+
+    try {
+      const response = await this.client.get('/documents/', {
+        params: {
+          fields: 'id,title,tags,correspondent,created',
+          page: 1,
+          page_size: safeLimit,
+          ordering: '-created'
         }
-      }
+      });
 
-      if (loaded && this._supportsCorrespondentIdIn !== false) {
-        this._supportsCorrespondentIdIn = true;
+      if (!Array.isArray(response?.data?.results)) {
+        return [];
       }
-    }
 
-    // Only remember misses when the lookup itself worked. A transient network
-    // error must not poison the cache for a whole TTL generation.
-    if (!loaded) {
-      return;
+      return response.data.results;
+    } catch (error) {
+      console.error('[ERROR] fetching recent documents with metadata:', error.message);
+      return [];
     }
+  }
 
-    // Cache the misses too, otherwise an ID Paperless-ngx cannot resolve is
-    // looked up again on every single search - which in the fallback path
-    // means listing all correspondents over and over.
-    for (const id of missingIds) {
-      if (!this.correspondentNameCache.has(id)) {
-        this.correspondentNameCache.set(id, null);
-      }
-    }
+  // Aktualisierte getDocuments Methode
+  async getDocuments() {
+    return this.getAllDocuments();
   }
 
-  /**
-   * Fetch one batch of correspondent names via `id__in`.
-   * @returns true when the filter was honoured, false when the caller must fall back.
-   */
-  async _fetchCorrespondentNameChunk(chunk) {
-    const requestedIds = new Set(chunk);
+  async getDocumentContent(documentId) {
+    this.initialize();
+    const response = await this.client.get(`/documents/${documentId}/`);
+    return response.data.content;
+  }
 
-    try {
-      const response = await this.client.get('/correspondents/', {
-        params: {
-          id__in: chunk.join(','),
-          page_size: chunk.length,
-          fields: 'id,name',
-        },
-      });
-
-      const results = response?.data?.results;
-      if (!Array.isArray(results)) {
-        return false;
-      }
-
-      // Unknown query params are silently ignored by Paperless-ngx, which would
-      // return an arbitrary page of correspondents instead of the requested
-      // ones. Any unexpected ID means the filter was not applied.
-      const filterHonoured = results.every((correspondent) =>
-        requestedIds.has(Number(correspondent?.id))
-      );
-      if (!filterHonoured) {
-        console.warn(
-          '[DEBUG] Paperless-ngx ignored the correspondent id__in filter, falling back to a full listing'
-        );
-        this._supportsCorrespondentIdIn = false;
-        return false;
-      }
-
-      for (const correspondent of results) {
-        const id = Number(correspondent?.id);
-        if (Number.isInteger(id) && typeof correspondent?.name === 'string') {
-          this.correspondentNameCache.set(id, correspondent.name);
-        }
-      }
-
-      return true;
-    } catch (error) {
-      console.error(
-        '[ERROR] fetching correspondent names by id:',
-        error.message
-      );
-      return false;
-    }
-  }
-
-  /**
-   * Load every correspondent name into the cache in one paginated sweep.
-   * Used as the fallback path so a single search never fans out into one
-   * request per correspondent.
-   *
-   * @returns true when the listing produced entries. `listCorrespondentsNames()`
-   *          swallows its own errors and returns an empty array, so an empty
-   *          result is treated as inconclusive rather than as a success.
-   */
-  async _primeCorrespondentNameCache() {
-    try {
-      const correspondents = await this.listCorrespondentsNames();
-      let cached = 0;
-      for (const correspondent of correspondents) {
-        const id = Number(correspondent?.id);
-        if (Number.isInteger(id) && typeof correspondent?.name === 'string') {
-          this.correspondentNameCache.set(id, correspondent.name);
-          cached += 1;
-        }
-      }
-      return cached > 0;
-    } catch (error) {
-      console.error('[ERROR] priming correspondent name cache:', error.message);
-      return false;
-    }
-  }
-
-  async getCorrespondentNameById(correspondentId) {
-    /**
-     * Get the Name of a Correspondent by its ID.
-     *
-     * @param   id  The id of the correspondent.
-     * @returns    An object holding the correspondent name, or null.
-     */
-    const id = Number(correspondentId);
-    if (!Number.isInteger(id) || id < 1) {
-      return null;
-    }
-
-    const names = await this.getCorrespondentNamesByIds([id]);
-    return names[id] ? { id, name: names[id] } : null;
-  }
-
-  async getTagNameById(tagId) {
-    /**
-     * Get the Name of a Tag by its ID.
-     *
-     * @param   id  The id of the tag.
-     * @returns    The name of the tag.
-     */
-    const id = Number(tagId);
-    if (!Number.isInteger(id) || id < 1) {
-      return null;
-    }
-
-    const names = await this.getTagNamesByIds([id]);
-    return names[id] || null;
-  }
-
-  async getDocumentsWithTitleTagsCorrespondentCreated() {
-    /**
-     * Get all documents with metadata (title, tags, correspondent, created date).
-     *
-     * @returns    An array of documents with metadata.
-     * @throws     An error if the request fails.
-     * @note       This method is used to get all documents with metadata for further processing
-     */
-
-    this.initialize();
-    try {
-      const response = await this.client.get('/documents/', {
-        params: {
-          fields: 'id,title,tags,correspondent,created',
-        },
-      });
-      return response.data.results;
-    } catch (error) {
-      console.error('[ERROR] fetching documents with metadata:', error.message);
-      return [];
-    }
-  }
-
-  async getRecentDocumentsWithMetadata(limit = 16) {
-    this.initialize();
-
-    const safeLimit = Number.isInteger(Number(limit))
-      ? Math.max(1, Math.min(Number(limit), 100))
-      : 16;
-
-    try {
-      const response = await this.client.get('/documents/', {
-        params: {
-          fields: 'id,title,tags,correspondent,created',
-          page: 1,
-          page_size: safeLimit,
-          ordering: '-created',
-        },
-      });
-
-      if (!Array.isArray(response?.data?.results)) {
-        return [];
-      }
-
-      return response.data.results;
-    } catch (error) {
-      console.error(
-        '[ERROR] fetching recent documents with metadata:',
-        error.message
-      );
-      return [];
-    }
-  }
-
-  async searchDocuments(query, limit = 100, mode = 'all') {
-    this.initialize();
-    if (!this.client) {
-      console.error('[DEBUG] Client not initialized');
-      return [];
-    }
-
-    const safeLimit = Number.isInteger(Number(limit))
-      ? Math.max(1, Math.min(Number(limit), 200))
-      : 100;
-    const normalizedQuery = String(query || '').trim();
-    const documentFields = 'id,title,tags,correspondent,created';
-
-    try {
-      // Explicit ID mode: exact lookup via GET /documents/{id}/
-      if (mode === 'id') {
-        const id = Number.parseInt(normalizedQuery, 10);
-        if (!Number.isInteger(id) || id < 1 || String(id) !== normalizedQuery) {
-          return [];
-        }
-
-        try {
-          // Request only the selector fields so the document content, which can
-          // be megabytes on scanned files, is not transferred.
-          const response = await this.client.get(`/documents/${id}/`, {
-            params: { fields: documentFields },
-          });
-          const doc = response?.data;
-          if (!doc || doc.id == null) {
-            return [];
-          }
-          return [
-            {
-              id: doc.id,
-              title: doc.title,
-              tags: doc.tags,
-              correspondent: doc.correspondent,
-              created: doc.created || doc.created_date || doc.added || null,
-            },
-          ];
-        } catch (error) {
-          if (error?.response?.status !== 404) {
-            console.error(
-              `[ERROR] searching document by id ${id}:`,
-              error.message
-            );
-          }
-          return [];
-        }
-      }
-
-      const params = {
-        fields: documentFields,
-        page: 1,
-        page_size: safeLimit,
-        ordering: '-created',
-      };
-
-      if (mode === 'title') {
-        params.title__icontains = normalizedQuery;
-      } else if (mode === 'tags') {
-        const tagIds = await this._findTagIdsByPartialName(normalizedQuery);
-        if (!tagIds.length) return [];
-        params.tags__id__in = tagIds.join(',');
-      } else if (mode === 'correspondent') {
-        const corrIds =
-          await this._findCorrespondentIdsByPartialName(normalizedQuery);
-        if (!corrIds.length) return [];
-        params.correspondent__id__in = corrIds.join(',');
-      } else {
-        params.query = normalizedQuery;
-      }
-
-      let response;
-      try {
-        response = await this.client.get('/documents/', { params });
-      } catch (error) {
-        // Full-text search needs the Paperless-ngx search index and rejects
-        // malformed query syntax. Keep the selector usable by retrying with a
-        // plain title filter instead of returning nothing.
-        if (mode !== 'all') {
-          throw error;
-        }
-
-        console.warn(
-          `[DEBUG] Full-text document search failed (${error.message}), retrying with a title filter`
-        );
-
-        const fallbackParams = { ...params };
-        delete fallbackParams.query;
-        fallbackParams.title__icontains = normalizedQuery;
-        response = await this.client.get('/documents/', {
-          params: fallbackParams,
-        });
-      }
-
-      if (!Array.isArray(response?.data?.results)) {
-        return [];
-      }
-
-      return response.data.results;
-    } catch (error) {
-      console.error('[ERROR] searching documents:', error.message);
-      return [];
-    }
-  }
-
-  async _findTagIdsByPartialName(query) {
-    try {
-      const response = await this.client.get('/tags/', {
-        params: { name__icontains: query, page_size: 50 },
-      });
-      return (response.data?.results || []).map((t) => t.id);
-    } catch (error) {
-      console.error('[ERROR] searching tags by name:', error.message);
-      return [];
-    }
-  }
-
-  async _findCorrespondentIdsByPartialName(query) {
-    try {
-      const response = await this.client.get('/correspondents/', {
-        params: { name__icontains: query, page_size: 50 },
-      });
-      return (response.data?.results || []).map((c) => c.id);
-    } catch (error) {
-      console.error('[ERROR] searching correspondents by name:', error.message);
-      return [];
-    }
-  }
-
-  // Aktualisierte getDocuments Methode
-  async getDocuments() {
-    return this.getAllDocuments();
-  }
-
-  async getDocumentContent(documentId) {
-    this.initialize();
-    const response = await this.client.get(`/documents/${documentId}/`);
-    return response.data.content;
-  }
-
-  async getDocument(documentId) {
-    this.initialize();
+  async getDocument(documentId) {
+    this.initialize();
     try {
       const response = await this.client.get(`/documents/${documentId}/`);
       return response.data;
     } catch (error) {
-      console.error('[ERROR] fetching document %s:', documentId, error.message);
+      console.error(`[ERROR] fetching document ${documentId}:`, error.message);
       throw error;
     }
   }
@@ -1769,324 +1378,259 @@ class PaperlessService {
   async searchForCorrespondentById(id) {
     try {
       const response = await this.client.get('/correspondents/', {
-        params: {
-          id: id,
-        },
+          params: {
+              id: id
+          }
       });
 
       const results = response.data.results;
-
+      
       if (results.length === 0) {
-        console.log(`[DEBUG] No correspondent with "${id}" found`);
-        return null;
+          console.log(`[DEBUG] No correspondent with "${id}" found`);
+          return null;
       }
-
+      
       if (results.length > 1) {
-        console.log(`[DEBUG] Multiple correspondents found:`);
-        results.forEach((c) => {
-          console.log(`- ID: ${c.id}, Name: ${c.name}`);
-        });
-        return results;
+          console.log(`[DEBUG] Multiple correspondents found:`);
+          results.forEach(c => {
+              console.log(`- ID: ${c.id}, Name: ${c.name}`);
+          });
+          return results;
       }
 
       // Genau ein Ergebnis gefunden
       return {
-        id: results[0].id,
-        name: results[0].name,
+          id: results[0].id,
+          name: results[0].name
       };
-    } catch (error) {
-      console.error(
-        '[ERROR] while seraching for existing correspondent:',
-        error.message
-      );
+
+  } catch (error) {
+      console.error('[ERROR] while seraching for existing correspondent:', error.message);
       throw error;
-    }
   }
+}
 
-  async searchForExistingCorrespondent(correspondent) {
-    try {
+async searchForExistingCorrespondent(correspondent) {
+  try {
       const response = await this.client.get('/correspondents/', {
-        params: {
-          name__icontains: correspondent,
-        },
+          params: {
+              name__icontains: correspondent
+          }
       });
 
       const results = response.data.results;
-
+      
       if (results.length === 0) {
-        console.log(
-          `[DEBUG] No correspondent with name "${correspondent}" found`
-        );
-        return null;
+          console.log(`[DEBUG] No correspondent with name "${correspondent}" found`);
+          return null;
       }
-
+      
       // Check for exact match in the results - thanks to @skius for the hint!
-      const exactMatch = results.find(
-        (c) => c.name.toLowerCase() === correspondent.toLowerCase()
-      );
+      const exactMatch = results.find(c => c.name.toLowerCase() === correspondent.toLowerCase());
       if (exactMatch) {
-        console.log(
-          `[DEBUG] Found exact match for correspondent "${correspondent}" with ID ${exactMatch.id}`
-        );
-        return {
-          id: exactMatch.id,
-          name: exactMatch.name,
-        };
+          console.log(`[DEBUG] Found exact match for correspondent "${correspondent}" with ID ${exactMatch.id}`);
+          return {
+              id: exactMatch.id,
+              name: exactMatch.name
+          };
       }
 
       // No exact match found, return null
       console.log(`[DEBUG] No exact match found for "${correspondent}"`);
       return null;
-    } catch (error) {
-      console.error(
-        '[ERROR] while searching for existing correspondent:',
-        error.message
-      );
+
+  } catch (error) {
+      console.error('[ERROR] while searching for existing correspondent:', error.message);
       throw error;
-    }
   }
+}
 
   async getOrCreateCorrespondent(name, options = {}) {
     this.initialize();
-
+    
     // Check if we should restrict to existing correspondents
     // Explicitly check options first, then env var
-    const restrictToExistingCorrespondents =
-      options.restrictToExistingCorrespondents === true ||
-      (options.restrictToExistingCorrespondents === undefined &&
-        process.env.RESTRICT_TO_EXISTING_CORRESPONDENTS === 'yes');
-
-    console.log(
-      `[DEBUG] Processing correspondent with restrictToExistingCorrespondents=${restrictToExistingCorrespondents}`
-    );
-
+    const restrictToExistingCorrespondents = options.restrictToExistingCorrespondents === true || 
+                                           (options.restrictToExistingCorrespondents === undefined && 
+                                            process.env.RESTRICT_TO_EXISTING_CORRESPONDENTS === 'yes');
+    
+    console.log(`[DEBUG] Processing correspondent with restrictToExistingCorrespondents=${restrictToExistingCorrespondents}`);
+  
     try {
-      // Search for the correspondent
-      const existingCorrespondent =
-        await this.searchForExistingCorrespondent(name);
-      console.log(
-        '[DEBUG] Response Correspondent Search: ',
-        existingCorrespondent
-      );
-
-      if (existingCorrespondent) {
-        console.log(
-          `[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}`
-        );
-        return existingCorrespondent;
-      }
-
-      // If we're restricting to existing correspondents and none was found, return null
-      if (restrictToExistingCorrespondents) {
-        console.log(
-          `[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null`
-        );
-        return null;
-      }
-
-      // Create new correspondent only if restrictions are not enabled
-      try {
-        const createResponse = await this.client.post('/correspondents/', {
-          name: name,
-        });
-        console.log(
-          `[DEBUG] Created new correspondent "${name}" with ID ${createResponse.data.id}`
-        );
-        return createResponse.data;
-      } catch (createError) {
-        if (
-          createError.response?.status === 400 &&
-          createError.response?.data?.error?.includes('unique constraint')
-        ) {
-          // Race condition check - another process might have created it
-          const retryResponse = await this.client.get('/correspondents/', {
-            params: { name: name },
-          });
-
-          const justCreatedCorrespondent = retryResponse.data.results.find(
-            (c) => c.name.toLowerCase() === name.toLowerCase()
-          );
-
-          if (justCreatedCorrespondent) {
-            console.log(
-              `[DEBUG] Retrieved correspondent "${name}" after constraint error with ID ${justCreatedCorrespondent.id}`
-            );
-            return justCreatedCorrespondent;
-          }
+        // Search for the correspondent
+        const existingCorrespondent = await this.searchForExistingCorrespondent(name);
+        console.log("[DEBUG] Response Correspondent Search: ", existingCorrespondent);
+    
+        if (existingCorrespondent) {
+            console.log(`[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}`);
+            return existingCorrespondent;
+        }
+        
+        // If we're restricting to existing correspondents and none was found, return null
+        if (restrictToExistingCorrespondents) {
+            console.log(`[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null`);
+            return null;
+        }
+    
+        // Create new correspondent only if restrictions are not enabled
+        try {
+            const createResponse = await this.client.post('/correspondents/', { 
+                name: name 
+            });
+            console.log(`[DEBUG] Created new correspondent "${name}" with ID ${createResponse.data.id}`);
+            return createResponse.data;
+        } catch (createError) {
+            if (createError.response?.status === 400 && 
+                createError.response?.data?.error?.includes('unique constraint')) {
+              
+                // Race condition check - another process might have created it
+                const retryResponse = await this.client.get('/correspondents/', {
+                    params: { name: name }
+                });
+              
+                const justCreatedCorrespondent = retryResponse.data.results.find(
+                    c => c.name.toLowerCase() === name.toLowerCase()
+                );
+              
+                if (justCreatedCorrespondent) {
+                    console.log(`[DEBUG] Retrieved correspondent "${name}" after constraint error with ID ${justCreatedCorrespondent.id}`);
+                    return justCreatedCorrespondent;
+                }
+            }
+            throw createError;
         }
-        throw createError;
-      }
     } catch (error) {
-      console.error(
-        '[ERROR] Failed to process correspondent "%s":',
-        name,
-        error.message
-      );
-      throw error;
+        console.error(`[ERROR] Failed to process correspondent "${name}":`, error.message);
+        throw error;
     }
-  }
+}
 
-  async searchForExistingDocumentType(documentType) {
-    try {
+async searchForExistingDocumentType(documentType) {
+  try {
       const response = await this.client.get('/document_types/', {
-        params: {
-          name__icontains: documentType,
-        },
+          params: {
+              name__icontains: documentType
+          }
       });
 
       const results = response.data.results;
-
+      
       if (results.length === 0) {
-        console.log(
-          `[DEBUG] No document type with name "${documentType}" found`
-        );
-        return null;
+          console.log(`[DEBUG] No document type with name "${documentType}" found`);
+          return null;
       }
-
+      
       // Check for exact match in the results
-      const exactMatch = results.find(
-        (dt) => dt.name.toLowerCase() === documentType.toLowerCase()
-      );
+      const exactMatch = results.find(dt => dt.name.toLowerCase() === documentType.toLowerCase());
       if (exactMatch) {
-        console.log(
-          `[DEBUG] Found exact match for document type "${documentType}" with ID ${exactMatch.id}`
-        );
-        return {
-          id: exactMatch.id,
-          name: exactMatch.name,
-        };
+          console.log(`[DEBUG] Found exact match for document type "${documentType}" with ID ${exactMatch.id}`);
+          return {
+              id: exactMatch.id,
+              name: exactMatch.name
+          };
       }
 
       // No exact match found, return null
       console.log(`[DEBUG] No exact match found for "${documentType}"`);
       return null;
-    } catch (error) {
-      console.error(
-        '[ERROR] while searching for existing document type:',
-        error.message
-      );
+
+  } catch (error) {
+      console.error('[ERROR] while searching for existing document type:', error.message);
       throw error;
-    }
   }
+}
 
-  async getOrCreateDocumentType(name, options = {}) {
-    this.initialize();
-
-    // Explicit option value wins; otherwise fall back to env config.
-    const restrictToExistingDocumentTypes =
-      options.restrictToExistingDocumentTypes === true ||
-      (options.restrictToExistingDocumentTypes === undefined &&
-        process.env.RESTRICT_TO_EXISTING_DOCUMENT_TYPES === 'yes');
+async getOrCreateDocumentType(name, options = {}) {
+  this.initialize();
 
-    console.log(
-      `[DEBUG] Processing document type with restrictToExistingDocumentTypes=${restrictToExistingDocumentTypes}`
-    );
+  // Explicit option value wins; otherwise fall back to env config.
+  const restrictToExistingDocumentTypes = options.restrictToExistingDocumentTypes === true ||
+                                        (options.restrictToExistingDocumentTypes === undefined &&
+                                         process.env.RESTRICT_TO_EXISTING_DOCUMENT_TYPES === 'yes');
 
-    try {
+  console.log(`[DEBUG] Processing document type with restrictToExistingDocumentTypes=${restrictToExistingDocumentTypes}`);
+  
+  try {
       // Suche nach existierendem document_type
       const existingDocType = await this.searchForExistingDocumentType(name);
-      console.log('[DEBUG] Response Document Type Search: ', existingDocType);
-
+      console.log("[DEBUG] Response Document Type Search: ", existingDocType);
+  
       if (existingDocType) {
-        console.log(
-          `[DEBUG] Found existing document type "${name}" with ID ${existingDocType.id}`
-        );
-        return existingDocType;
-      }
-
-      if (restrictToExistingDocumentTypes) {
-        console.log(
-          `[DEBUG] Document type "${name}" does not exist and restrictions are enabled, returning null`
-        );
-        return null;
+          console.log(`[DEBUG] Found existing document type "${name}" with ID ${existingDocType.id}`);
+          return existingDocType;
       }
 
+        if (restrictToExistingDocumentTypes) {
+          console.log(`[DEBUG] Document type "${name}" does not exist and restrictions are enabled, returning null`);
+          return null;
+        }
+  
       // Erstelle neuen document_type
       try {
-        const createResponse = await this.client.post('/document_types/', {
-          name: name,
-          matching_algorithm: 1, // 1 = ANY
-          match: '', // Optional: Kann später angepasst werden
-          is_insensitive: true,
-        });
-        console.log(
-          `[DEBUG] Created new document type "${name}" with ID ${createResponse.data.id}`
-        );
-        return createResponse.data;
-      } catch (createError) {
-        if (
-          createError.response?.status === 400 &&
-          createError.response?.data?.error?.includes('unique constraint')
-        ) {
-          // Race condition check
-          const retryResponse = await this.client.get('/document_types/', {
-            params: { name: name },
+          const createResponse = await this.client.post('/document_types/', { 
+              name: name,
+              matching_algorithm: 1, // 1 = ANY
+              match: "",  // Optional: Kann später angepasst werden
+              is_insensitive: true
           });
-
-          const justCreatedDocType = retryResponse.data.results.find(
-            (dt) => dt.name.toLowerCase() === name.toLowerCase()
-          );
-
-          if (justCreatedDocType) {
-            console.log(
-              `[DEBUG] Retrieved document type "${name}" after constraint error with ID ${justCreatedDocType.id}`
-            );
-            return justCreatedDocType;
+          console.log(`[DEBUG] Created new document type "${name}" with ID ${createResponse.data.id}`);
+          return createResponse.data;
+      } catch (createError) {
+          if (createError.response?.status === 400 && 
+              createError.response?.data?.error?.includes('unique constraint')) {
+            
+              // Race condition check
+              const retryResponse = await this.client.get('/document_types/', {
+                  params: { name: name }
+              });
+            
+              const justCreatedDocType = retryResponse.data.results.find(
+                  dt => dt.name.toLowerCase() === name.toLowerCase()
+              );
+            
+              if (justCreatedDocType) {
+                  console.log(`[DEBUG] Retrieved document type "${name}" after constraint error with ID ${justCreatedDocType.id}`);
+                  return justCreatedDocType;
+              }
           }
-        }
-        throw createError;
+          throw createError;
       }
-    } catch (error) {
-      console.error(
-        `[ERROR] Failed to process document type "${name}":`,
-        error.message
-      );
+  } catch (error) {
+      console.error(`[ERROR] Failed to process document type "${name}":`, error.message);
       throw error;
-    }
   }
+}
 
   async removeUnusedTagsFromDocument(documentId, keepTagIds) {
     this.initialize();
     if (!this.client) return;
-
+  
     try {
-      console.log(
-        '[DEBUG] Removing unused tags from document %s, keeping tags:',
-        documentId,
-        keepTagIds
-      );
-
+      console.log(`[DEBUG] Removing unused tags from document ${documentId}, keeping tags:`, keepTagIds);
+      
       // Hole aktuelles Dokument
       const currentDoc = await this.getDocument(documentId);
-
+      
       // Finde Tags die entfernt werden sollen (die nicht in keepTagIds sind)
-      const tagsToRemove = currentDoc.tags.filter(
-        (tagId) => !keepTagIds.includes(tagId)
-      );
-
+      const tagsToRemove = currentDoc.tags.filter(tagId => !keepTagIds.includes(tagId));
+      
       if (tagsToRemove.length === 0) {
         console.log('[DEBUG] No tags to remove');
         return currentDoc;
       }
-
+  
       // Update das Dokument mit nur den zu behaltenden Tags
       const updateData = {
-        tags: keepTagIds,
+        tags: keepTagIds
       };
-
+  
       // Führe das Update durch
       await this.client.patch(`/documents/${documentId}/`, updateData);
-      console.log(
-        `[DEBUG] Successfully removed ${tagsToRemove.length} tags from document ${documentId}`
-      );
-
+      console.log(`[DEBUG] Successfully removed ${tagsToRemove.length} tags from document ${documentId}`);
+      
       return await this.getDocument(documentId);
     } catch (error) {
-      console.error(
-        '[ERROR] Error removing unused tags from document %s:',
-        documentId,
-        error.message
-      );
+      console.error(`[ERROR] Error removing unused tags from document ${documentId}:`, error.message);
       throw error;
     }
   }
@@ -2097,92 +1641,36 @@ class PaperlessService {
       const response = await this.client.get(`/tags/${tagId}/`);
       return response.data.name;
     } catch (error) {
-      console.error(
-        `[ERROR] fetching tag text for ID ${tagId}:`,
-        error.message
-      );
+      console.error(`[ERROR] fetching tag text for ID ${tagId}:`, error.message);
       return null;
     }
   }
 
-  /**
-   * Lightweight probe against the configured Paperless-ngx instance.
-   *
-   * Used by the startup retry loop and by every scan run: most read helpers in
-   * this service swallow transport errors and return empty results, which makes
-   * an unreachable Paperless indistinguishable from "nothing to do". This call
-   * makes that distinction explicit and never throws.
-   *
-   * @returns {Promise<{reachable: boolean, authorized: boolean, status: number|null, error: string|null}>}
-   *   reachable: an HTTP response was received (the host answered).
-   *   authorized: the API token is accepted (not 401/403).
-   */
-  async checkConnection() {
-    this.initialize();
-
-    if (!this.client) {
-      return {
-        reachable: false,
-        authorized: false,
-        status: null,
-        error: 'Paperless-ngx client is not configured',
-      };
-    }
-
-    try {
-      const response = await this.client.get('/users/', {
-        params: { current_user: true, page_size: 1 },
-        timeout: CONNECTION_PROBE_TIMEOUT_MS,
-      });
-
-      return {
-        reachable: true,
-        authorized: true,
-        status: response.status || 200,
-        error: null,
-      };
-    } catch (error) {
-      const status = error.response?.status ?? null;
-
-      // A status code means the host answered, so it is reachable — only the
-      // credentials or permissions may be wrong. Without a status the request
-      // never got through, so authorization is unknown and reported as false.
-      return {
-        reachable: status !== null,
-        authorized: status !== null && status !== 401 && status !== 403,
-        status,
-        error: error.message || 'Unknown connection error',
-      };
-    }
-  }
-
   async getOwnUserID() {
     this.initialize();
     try {
-      const response = await this.client.get('/users/', {
-        params: {
-          current_user: true,
-          full_perms: true,
-        },
-      });
-
-      if (response.data.results && response.data.results.length > 0) {
-        const userInfo = response.data.results;
-        //filter for username by process.env.PAPERLESS_USERNAME
-        const user = userInfo.find(
-          (user) => user.username === process.env.PAPERLESS_USERNAME
-        );
-        if (user) {
-          console.log(`[DEBUG] Found own user ID: ${user.id}`);
-          return user.id;
+        const response = await this.client.get('/users/', {
+            params: {
+                current_user: true,
+                full_perms: true
+            }
+        });
+        
+        if (response.data.results && response.data.results.length > 0) {
+            const userInfo = response.data.results;
+            //filter for username by process.env.PAPERLESS_USERNAME
+            const user = userInfo.find(user => user.username === process.env.PAPERLESS_USERNAME);
+            if (user) {
+                console.log(`[DEBUG] Found own user ID: ${user.id}`);
+                return user.id;
+            }
         }
-      }
-      return null;
+        return null;
     } catch (error) {
-      console.error('[ERROR] fetching own user ID:', error.message);
-      return null;
+        console.error('[ERROR] fetching own user ID:', error.message);
+        return null;
     }
-  }
+}
   //Remove if not needed?
   async getOwnerOfDocument(documentId) {
     this.initialize();
@@ -2190,10 +1678,7 @@ class PaperlessService {
       const response = await this.client.get(`/documents/${documentId}/`);
       return response.data.owner;
     } catch (error) {
-      console.error(
-        `[ERROR] fetching owner of document ${documentId}:`,
-        error.message
-      );
+      console.error(`[ERROR] fetching owner of document ${documentId}:`, error.message);
       return null;
     }
   }
@@ -2205,42 +1690,31 @@ class PaperlessService {
       const response = await this.client.get(`/documents/${documentId}/`);
       return response.data.user_can_change;
     } catch (error) {
-      console.error(
-        `[ERROR] No Permission to edit document ${documentId}:`,
-        error.message
-      );
+      console.error(`[ERROR] No Permission to edit document ${documentId}:`, error.message);
       return null;
     }
   }
 
+
   async updateDocument(documentId, updates) {
     this.initialize();
     if (!this.client) return;
     try {
       const currentDoc = await this.getDocument(documentId);
-
+      
       if (updates.tags) {
-        console.log(
-          '[DEBUG] Current tags for document %s:',
-          documentId,
-          currentDoc.tags
-        );
+        console.log(`[DEBUG] Current tags for document ${documentId}:`, currentDoc.tags);
         console.log(`[DEBUG] Adding new tags:`, updates.tags);
         console.log(`[DEBUG] Current correspondent:`, currentDoc.correspondent);
         console.log(`[DEBUG] New correspondent:`, updates.correspondent);
-
-        const combinedTags = [
-          ...new Set([...currentDoc.tags, ...updates.tags]),
-        ];
+                
+        const combinedTags = [...new Set([...currentDoc.tags, ...updates.tags])];
         updates.tags = combinedTags;
-
+        
         console.log(`[DEBUG] Combined tags:`, combinedTags);
       }
 
-      if (
-        updates.correspondent === null ||
-        updates.correspondent === undefined
-      ) {
+      if (updates.correspondent === null || updates.correspondent === undefined) {
         // Keep existing correspondent when no new value is provided
         delete updates.correspondent;
       }
@@ -2264,14 +1738,10 @@ class PaperlessService {
           }
 
           if (!isValid(dateObject)) {
-            console.warn(
-              `[WARN] Invalid date format: ${updates.created}, skipping date update`
-            );
+            console.warn(`[WARN] Invalid date format: ${updates.created}, skipping date update`);
             delete updates.created;
           } else if (dateObject > new Date()) {
-            console.warn(
-              `[WARN] AI returned future date ${format(dateObject, 'yyyy-MM-dd')}, skipping date update`
-            );
+            console.warn(`[WARN] AI returned future date ${format(dateObject, 'yyyy-MM-dd')}, skipping date update`);
             delete updates.created;
           } else {
             updates.created = format(dateObject, 'yyyy-MM-dd');
@@ -2301,26 +1771,20 @@ class PaperlessService {
       //     await this.client.patch(`/documents/${documentId}/`, { custom_fields: [] });
       //   }
       // }
-
+      
       // Validate title length before sending to API
       if (updateData.title && updateData.title.length > 128) {
         updateData.title = updateData.title.substring(0, 124) + '…';
-        console.warn(
-          `[WARN] Title truncated to 128 characters for document ${documentId}`
-        );
+        console.warn(`[WARN] Title truncated to 128 characters for document ${documentId}`);
       }
-
+      
       console.log('[DEBUG] Final update data:', updateData);
       await this.client.patch(`/documents/${documentId}/`, updateData);
-      console.log(
-        '[SUCCESS] Updated document %s with:',
-        documentId,
-        updateData
-      );
+      console.log(`[SUCCESS] Updated document ${documentId} with:`, updateData);
       return await this.getDocument(documentId);
     } catch (error) {
       console.log(error);
-      console.error('[ERROR] updating document %s:', documentId, error.message);
+      console.error(`[ERROR] updating document ${documentId}:`, error.message);
       return null;
     }
   }
@@ -2337,18 +1801,13 @@ class PaperlessService {
     if (!this.client) return null;
     try {
       const patch = {};
-      if (Array.isArray(original.tags)) patch.tags = original.tags;
-      if (original.title != null) patch.title = original.title;
-      if (original.correspondent !== undefined)
-        patch.correspondent = original.correspondent;
-      if (original.documentType !== undefined)
-        patch.document_type = original.documentType;
-      if (original.language != null) patch.language = original.language;
+      if (Array.isArray(original.tags))          patch.tags          = original.tags;
+      if (original.title != null)                 patch.title         = original.title;
+      if (original.correspondent !== undefined)   patch.correspondent = original.correspondent;
+      if (original.documentType   !== undefined)  patch.document_type = original.documentType;
+      if (original.language       != null)        patch.language      = original.language;
 
-      console.log(
-        `[DEBUG] Restoring document ${documentId} to original state:`,
-        patch
-      );
+      console.log(`[DEBUG] Restoring document ${documentId} to original state:`, patch);
       await this.client.patch(`/documents/${documentId}/`, patch);
       console.log(`[SUCCESS] Restored document ${documentId}`);
       return await this.getDocument(documentId);
@@ -2359,4 +1818,5 @@ class PaperlessService {
   }
 }
 
+
 module.exports = new PaperlessService();
diff --git a/views/settings.ejs b/views/settings.ejs
index ba431fcf..1d0b558f 100644
--- a/views/settings.ejs
+++ b/views/settings.ejs
@@ -666,7 +666,8 @@
                                     <div class="col-span-5"><input type="text" id="newFieldName" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg shadow-sm focus:ring-2 focus:ring-blue-500 focus:border-blue-500 bg-white dark:bg-gray-800 text-gray-900 dark:text-gray-100" placeholder="Enter field name..."></div>
                                     <div class="col-span-3">
                                         <select id="newFieldType" onchange="toggleCurrencySelect()" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg shadow-sm focus:ring-2 focus:ring-blue-500 focus:border-blue-500 bg-white dark:bg-gray-800 text-gray-900 dark:text-gray-100">
-                                            <option value="string">Text</option>
+                                            <option value="string">Text (up to 128 letters)</option>
+                                            <option value="long-string">Long Text</option>
                                             <option value="float">Number</option>
                                             <option value="integer">Integer</option>
                                             <option value="monetary">Currency</option>

From 78f1d6cfc9a1fa12f92da76e59dbe8dfcef98fc3 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sat, 25 Jul 2026 11:30:11 +0200
Subject: [PATCH 02/53] use names not implying a problem where is none.

---
 services/ollamaService.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/services/ollamaService.js b/services/ollamaService.js
index 2aef95ae..53616489 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -620,10 +620,8 @@ class OllamaService {
      * @returns {Object} Parsed response
      */
     _processOllamaResponse(responseData) {
-        // Check if we got a structured response or need to parse from text
         if (responseData.response && typeof responseData.response === 'object') {
-            // We got a structured response directly
-            console.log('Using structured output response');
+            console.log('Using response JS object directly');
             return {
                 tags: Array.isArray(responseData.response.tags) ? responseData.response.tags : [],
                 correspondent: responseData.response.correspondent || null,

From 0dc0ea093e25d5e28f7b8410a36dcacc1886d96e Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 9 Aug 2026 23:38:53 +0200
Subject: [PATCH 03/53] adopt paperless-ngx datatype longtext

# Conflicts:
#	services/paperlessService.js
---
 services/paperlessService.js | 7 +++++--
 views/settings.ejs           | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/services/paperlessService.js b/services/paperlessService.js
index 91466ed8..3b1f01ac 100644
--- a/services/paperlessService.js
+++ b/services/paperlessService.js
@@ -363,6 +363,9 @@ class PaperlessService {
   async createCustomFieldSafely(fieldName, fieldType, default_currency) {
     const normalizedFieldType = String(fieldType || '').trim();
 
+    // Check an already existing field before attempting creation. Paperless
+    // returns 400 for duplicate names, so without this check a configured
+    // type mismatch (e.g. string vs. longtext) would be silently ignored.
     const existingField = await this.findExistingCustomField(fieldName);
     if (existingField) {
       const existingFieldType = String(existingField.data_type || '').trim();
@@ -996,7 +999,7 @@ class PaperlessService {
       includeTagIds = await this.resolveTagIdsByName(includeTagNames);
 
       if (includeTagIds.length === 0) {
-        console.warn('[DEBUG] None of the specified tags were found');
+        console.warn(`[DEBUG] None of the specified tags ${includeTagNames} were found`);
         return [];
       }
 
@@ -1211,7 +1214,7 @@ class PaperlessService {
       }
       
       if (tagIds.length === 0) {
-        console.warn('[DEBUG] None of the specified tags were found');
+        console.warn(`[DEBUG] None of the specified tags ${tagNames} were found`);
         return [];
       }
       
diff --git a/views/settings.ejs b/views/settings.ejs
index 1d0b558f..369a81a3 100644
--- a/views/settings.ejs
+++ b/views/settings.ejs
@@ -667,7 +667,7 @@
                                     <div class="col-span-3">
                                         <select id="newFieldType" onchange="toggleCurrencySelect()" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg shadow-sm focus:ring-2 focus:ring-blue-500 focus:border-blue-500 bg-white dark:bg-gray-800 text-gray-900 dark:text-gray-100">
                                             <option value="string">Text (up to 128 letters)</option>
-                                            <option value="long-string">Long Text</option>
+                                            <option value="longtext">Long Text</option>
                                             <option value="float">Number</option>
                                             <option value="integer">Integer</option>
                                             <option value="monetary">Currency</option>

From 9065588e852e52d1a7f045e61b0bd50a9c9fa6ae Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 27 Jul 2026 10:56:05 +0200
Subject: [PATCH 04/53] prevent to publish IDE configuration

---
 .idea/.gitignore | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 .idea/.gitignore

diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 00000000..902d1820
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,12 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Ignored default folder with query files
+/queries/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Jetbrains IDE
+/.idea/
\ No newline at end of file

From e41a13c976766bc065b25727b38fb12bc16567db Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 27 Jul 2026 10:58:58 +0200
Subject: [PATCH 05/53] prevent to publish IDE configuration

---
 .gitignore | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 655e1560..eb0fd26a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,9 +32,11 @@ dist/
 .local/
 *.local.*
 .vscode/
+.idea/
 # AI context files (auto-generated)
 COPILOT.md
 .DS_Store
 GEMINI.md
 CVE/*
-.todo/*
\ No newline at end of file
+.todo/*
+.ai/

From 8627823c14d312682050ab05def08fa9657b8e22 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 9 Aug 2026 22:44:08 +0200
Subject: [PATCH 06/53] Cleanup

---
 services/paperlessService.js | 1149 ++++++++++++++++++----------------
 1 file changed, 607 insertions(+), 542 deletions(-)

diff --git a/services/paperlessService.js b/services/paperlessService.js
index 3b1f01ac..58c5c9cb 100644
--- a/services/paperlessService.js
+++ b/services/paperlessService.js
@@ -10,6 +10,8 @@ const {
 
 /** Timeout for the connectivity probe so a hanging host cannot stall a scan. */
 const CONNECTION_PROBE_TIMEOUT_MS = 10000;
+const {CachedNameIdEnum} = require('./CachedNameIdEnum');
+const {safeExtractRelativePath} = require('./serviceUtils');
 
 class PaperlessService {
   constructor() {
@@ -33,6 +35,44 @@ class PaperlessService {
     // Dynamic cache lifetime from config (default: 5 minutes)
     // Lazy load to avoid circular dependency
     this._cacheTTL = null;
+    this._correspondents = null;
+    this._documentTypes = null;
+  }
+
+  get correspondents() {
+    if (this._correspondents === null) {
+      const aiServiceFactory = require('./aiServiceFactory');
+      const aiService = aiServiceFactory.getService();
+      this._correspondents = new CachedNameIdEnum(
+        'correspondent',
+        '/correspondents/',
+        'Ignore case and consider typos, misspellings, and alternative spellings, misplaced Address data, seasonal ads,' +
+          'abbreviations of organization form etc.. ',
+        () => {
+          return this.CACHE_LIFETIME;
+        },
+        aiService
+      );
+    }
+    return this._correspondents;
+  }
+
+  get documentTypes() {
+    if (this._documentTypes === null) {
+      const aiServiceFactory = require('./aiServiceFactory');
+      const aiService = aiServiceFactory.getService();
+      this._documentTypes = new CachedNameIdEnum(
+        'document_type',
+        '/document_types/',
+        'Ignore case. Ignore special characters. consider typos, misspellings, alternative spellings, ' +
+          'same words in a different language. Especially check if it is a synonyme for a existing document type',
+        () => {
+          return this.CACHE_LIFETIME;
+        },
+        aiService
+      );
+    }
+    return this._documentTypes;
   }
 
   get CACHE_LIFETIME() {
@@ -271,7 +311,7 @@ class PaperlessService {
     }
   }
 
-
+  //todo
   // Aktualisiert den Tag-Cache, wenn er älter als CACHE_LIFETIME ist
   async ensureTagCache() {
     const now = Date.now();
@@ -280,8 +320,12 @@ class PaperlessService {
       if (this._refreshPromise) {
         return this._refreshPromise;
       }
-      const expireTime = new Date(this.lastTagRefresh + this.CACHE_LIFETIME).toISOString();
-      console.log(`[DEBUG] Tag cache expired (age: ${Math.floor(cacheAge / 1000)}s, TTL: ${Math.floor(this.CACHE_LIFETIME / 1000)}s, expired at: ${expireTime})`);
+      const expireTime = new Date(
+        this.lastTagRefresh + this.CACHE_LIFETIME
+      ).toISOString();
+      console.log(
+        `[DEBUG] Tag cache expired (age: ${Math.floor(cacheAge / 1000)}s, TTL: ${Math.floor(this.CACHE_LIFETIME / 1000)}s, expired at: ${expireTime})`
+      );
       // No race condition: synchronous code is never preempted in Node.js's
       // event loop, so no other call can reach here between the check above
       // and the assignment below.
@@ -297,64 +341,74 @@ class PaperlessService {
    * Useful for forcing a refresh after external tag modifications.
    */
   clearTagCache() {
+    this.initialize()
     console.log('[DEBUG] Manually clearing tag cache...');
     this.tagCache.clear();
+    this.correspondents.flush_cache(this.client);
     this.lastTagRefresh = 0;
     console.log('[DEBUG] Tag cache cleared.');
   }
 
-  // Lädt alle existierenden Tags
+  // Lädt alle existierenden Tags Todo
   async refreshTagCache() {
-      try {
-        console.log('[DEBUG] Refreshing tag cache...');
-        this.tagCache.clear();
-        let nextUrl = '/tags/';
-        while (nextUrl) {
-          const response = await this.client.get(nextUrl);
-
-          // Validate response structure
-          if (!response?.data?.results) {
-            console.error('[ERROR] Invalid response structure from API:', response?.data);
-            break;
-          }
+    try {
+      console.log('[DEBUG] Refreshing tag cache...');
+      this.tagCache.clear();
+      let nextUrl = '/tags/';
+      while (nextUrl) {
+        const response = await this.client.get(nextUrl);
+
+        // Validate response structure
+        if (!response?.data?.results) {
+          console.error(
+            '[ERROR] Invalid response structure from API:',
+            response?.data
+          );
+          break;
+        }
 
-          response.data.results.forEach(tag => {
-            this.tagCache.set(tag.name.toLowerCase(), tag);
-          });
+        response.data.results.forEach((tag) => {
+          this.tagCache.set(tag.name.toLowerCase(), tag);
+        });
 
-          // Safely extract relative path from next URL to prevent SSRF
-          if (response.data.next) {
-            nextUrl = this._safeExtractRelativePath(response.data.next);
-            if (nextUrl) {
-              console.log('[DEBUG] Next page URL:', nextUrl);
-            }
-          } else {
-            nextUrl = null;
+        // Safely extract relative path from next URL to prevent SSRF
+        if (response.data.next) {
+          nextUrl = safeExtractRelativePath(response.data.next, this.client);
+          if (nextUrl) {
+            console.log('[DEBUG] Next page URL:', nextUrl);
           }
+        } else {
+          nextUrl = null;
         }
-        this.lastTagRefresh = Date.now();
-        console.log(`[DEBUG] Tag cache refreshed. Found ${this.tagCache.size} tags.`);
-      } catch (error) {
-        console.error('[ERROR] refreshing tag cache:', error.message);
-        throw error;
       }
+      this.lastTagRefresh = Date.now();
+      console.log(
+        `[DEBUG] Tag cache refreshed. Found ${this.tagCache.size} tags.`
+      );
+    } catch (error) {
+      console.error('[ERROR] refreshing tag cache:', error.message);
+      throw error;
     }
+  }
 
   async initializeWithCredentials(apiUrl, apiToken) {
     this.client = axios.create({
       baseURL: apiUrl.replace(/\/+$/, '') + '/api',
       headers: {
-        'Authorization': `Token ${apiToken}`,
-        'Content-Type': 'application/json'
-      }
+        Authorization: `Token ${apiToken}`,
+        'Content-Type': 'application/json',
+      },
     });
-    
+
     // Test the connection
     try {
       await this.client.get('/');
       return true;
     } catch (error) {
-      console.error('[ERROR] Failed to initialize with credentials:', error.message);
+      console.error(
+        '[ERROR] Failed to initialize with credentials:',
+        error.message
+      );
       this.client = null;
       return false;
     }
@@ -372,7 +426,7 @@ class PaperlessService {
       if (existingFieldType !== normalizedFieldType) {
         const error = new Error(
           `Custom field "${fieldName}" has type "${existingFieldType}" in Paperless, ` +
-          `but configuration requests "${normalizedFieldType}".`
+            `but configuration requests "${normalizedFieldType}".`
         );
         error.code = 'CUSTOM_FIELD_TYPE_MISMATCH';
         throw error;
@@ -382,23 +436,27 @@ class PaperlessService {
 
     try {
       // Try to create the field first
-      const response = await this.client.post('/custom_fields/', { 
+      const response = await this.client.post('/custom_fields/', {
         name: fieldName,
         data_type: normalizedFieldType,
         extra_data: {
-          default_currency: default_currency || null
-        }
+          default_currency: default_currency || null,
+        },
       });
       const newField = response.data;
-      console.log(`[DEBUG] Successfully created custom field "${fieldName}" with ID ${newField.id}`);
+      console.log(
+        `[DEBUG] Successfully created custom field "${fieldName}" with ID ${newField.id}`
+      );
       this.customFieldCache.set(fieldName.toLowerCase(), newField);
       return newField;
-    } catch (error) { 
+    } catch (error) {
       if (error.response?.status === 400) {
         await this.refreshCustomFieldCache();
         const existingField = await this.findExistingCustomField(fieldName);
         if (existingField) {
-          const existingFieldType = String(existingField.data_type || '').trim();
+          const existingFieldType = String(
+            existingField.data_type || ''
+          ).trim();
           if (existingFieldType !== normalizedFieldType) {
             const mismatchError = new Error(
               `Custom field "${fieldName}" has type "${existingFieldType}", ` +
@@ -417,87 +475,104 @@ class PaperlessService {
   async getExistingCustomFields(documentId) {
     try {
       const response = await this.client.get(`/documents/${documentId}/`);
-      console.log('[DEBUG] Document response custom fields:', response.data.custom_fields);
+      console.log(
+        '[DEBUG] Document response custom fields:',
+        response.data.custom_fields
+      );
       return response.data.custom_fields || [];
     } catch (error) {
       console.error(`[ERROR] fetching document ${documentId}:`, error.message);
       return [];
     }
   }
-  
+
   async findExistingCustomField(fieldName) {
     const normalizedName = fieldName.toLowerCase();
-    
+
     const cachedField = this.customFieldCache.get(normalizedName);
     if (cachedField) {
-      console.log(`[DEBUG] Found custom field "${fieldName}" in cache with ID ${cachedField.id}`);
+      console.log(
+        `[DEBUG] Found custom field "${fieldName}" in cache with ID ${cachedField.id}`
+      );
       return cachedField;
     }
 
     try {
       const response = await this.client.get('/custom_fields/', {
         params: {
-          name__iexact: normalizedName  // Case-insensitive exact match
-        }
+          name__iexact: normalizedName, // Case-insensitive exact match
+        },
       });
 
       if (response.data.results.length > 0) {
         const foundField = response.data.results[0];
-        console.log(`[DEBUG] Found existing custom field "${fieldName}" via API with ID ${foundField.id}`);
+        console.log(
+          `[DEBUG] Found existing custom field "${fieldName}" via API with ID ${foundField.id}`
+        );
         this.customFieldCache.set(normalizedName, foundField);
         return foundField;
       }
     } catch (error) {
-      console.warn(`[ERROR] searching for custom field "${fieldName}":`, error.message);
+      console.warn(
+        `[ERROR] searching for custom field "${fieldName}":`,
+        error.message
+      );
     }
 
     return null;
   }
 
   async refreshCustomFieldCache() {
-      try {
-        console.log('[DEBUG] Refreshing custom field cache...');
-        this.customFieldCache.clear();
-        let nextUrl = '/custom_fields/';
-        while (nextUrl) {
-          const response = await this.client.get(nextUrl);
-
-          // Validate response structure
-          if (!response?.data?.results) {
-            console.error('[ERROR] Invalid response structure from API:', response?.data);
-            break;
-          }
+    try {
+      console.log('[DEBUG] Refreshing custom field cache...');
+      this.customFieldCache.clear();
+      let nextUrl = '/custom_fields/';
+      while (nextUrl) {
+        const response = await this.client.get(nextUrl);
+
+        // Validate response structure
+        if (!response?.data?.results) {
+          console.error(
+            '[ERROR] Invalid response structure from API:',
+            response?.data
+          );
+          break;
+        }
 
-          response.data.results.forEach(field => {
-            this.customFieldCache.set(field.name.toLowerCase(), field);
-          });
+        response.data.results.forEach((field) => {
+          this.customFieldCache.set(field.name.toLowerCase(), field);
+        });
 
-          // Safely extract relative path from next URL to prevent SSRF
-          if (response.data.next) {
-            nextUrl = this._safeExtractRelativePath(response.data.next);
-            if (nextUrl) {
-              console.log('[DEBUG] Next page URL:', nextUrl);
-            }
-          } else {
-            nextUrl = null;
+        // Safely extract relative path from next URL to prevent SSRF
+        if (response.data.next) {
+          nextUrl = this._safeExtractRelativePath(response.data.next);
+          if (nextUrl) {
+            console.log('[DEBUG] Next page URL:', nextUrl);
           }
+        } else {
+          nextUrl = null;
         }
-        this.lastCustomFieldRefresh = Date.now();
-        console.log(`[DEBUG] Custom field cache refreshed. Found ${this.customFieldCache.size} fields.`);
-      } catch (error) {
-        console.error('[ERROR] refreshing custom field cache:', error.message);
-        throw error;
       }
+      this.lastCustomFieldRefresh = Date.now();
+      console.log(
+        `[DEBUG] Custom field cache refreshed. Found ${this.customFieldCache.size} fields.`
+      );
+    } catch (error) {
+      console.error('[ERROR] refreshing custom field cache:', error.message);
+      throw error;
     }
+  }
 
-
+  // todo
   async findExistingTag(tagName) {
     const normalizedName = tagName.toLowerCase();
-    
+
     // 1. Zuerst im Cache suchen
     const cachedTag = this.tagCache.get(normalizedName);
     if (cachedTag) {
-      console.log(`[DEBUG] Found tag "${tagName}" in cache with ID ${cachedTag.id}`);
+      console.log(
+        `[DEBUG] Found tag "${tagName}" in cache with ID ${cachedTag.id}`
+      );
       return cachedTag;
     }
 
@@ -505,13 +580,15 @@ class PaperlessService {
     try {
       const response = await this.client.get('/tags/', {
         params: {
-          name__iexact: normalizedName  // Case-insensitive exact match
-        }
+          name__iexact: normalizedName, // Case-insensitive exact match
+        },
       });
 
       if (response.data.results.length > 0) {
         const foundTag = response.data.results[0];
-        console.log(`[DEBUG] Found existing tag "${tagName}" via API with ID ${foundTag.id}`);
+        console.log(
+          `[DEBUG] Found existing tag "${tagName}" via API with ID ${foundTag.id}`
+        );
         this.tagCache.set(normalizedName, foundTag);
         return foundTag;
       }
@@ -524,12 +601,14 @@ class PaperlessService {
 
   async createTagSafely(tagName) {
     const normalizedName = tagName.toLowerCase();
-    
+
     try {
       // Versuche zuerst, den Tag zu erstellen
       const response = await this.client.post('/tags/', { name: tagName });
       const newTag = response.data;
-      console.log(`[DEBUG] Successfully created tag "${tagName}" with ID ${newTag.id}`);
+      console.log(
+        `[DEBUG] Successfully created tag "${tagName}" with ID ${newTag.id}`
+      );
       this.tagCache.set(normalizedName, newTag);
       // Invalidate cache after creating new tag to ensure consistency
       this.lastTagRefresh = 0;
@@ -539,7 +618,7 @@ class PaperlessService {
         // Bei einem 400er Fehler könnte der Tag bereits existieren
         // Aktualisiere den Cache und suche erneut
         await this.refreshTagCache();
-        
+
         // Suche nochmal nach dem Tag
         const existingTag = await this.findExistingTag(tagName);
         if (existingTag) {
@@ -554,13 +633,14 @@ class PaperlessService {
     try {
       this.initialize();
       await this.ensureTagCache();
-      
+
       // Check if we should restrict to existing tags
       // Explicitly check options first, then env var
-      const restrictToExistingTags = options.restrictToExistingTags === true || 
-                                   (options.restrictToExistingTags === undefined && 
-                                    process.env.RESTRICT_TO_EXISTING_TAGS === 'yes');
-      
+      const restrictToExistingTags =
+        options.restrictToExistingTags === true ||
+        (options.restrictToExistingTags === undefined &&
+          process.env.RESTRICT_TO_EXISTING_TAGS === 'yes');
+
       // Input validation
       if (!tagNames) {
         console.warn('[DEBUG] No tags provided to processTags');
@@ -568,23 +648,26 @@ class PaperlessService {
       }
 
       // Convert to array if string is passed
-      const tagsArray = typeof tagNames === 'string' 
-        ? [tagNames]
-        : Array.isArray(tagNames) 
-          ? tagNames 
-          : [];
+      const tagsArray =
+        typeof tagNames === 'string'
+          ? [tagNames]
+          : Array.isArray(tagNames)
+            ? tagNames
+            : [];
 
       if (tagsArray.length === 0) {
         console.warn('[DEBUG] No valid tags to process');
         return { tagIds: [], errors: [] };
       }
-  
+
       const tagIds = [];
       const errors = [];
       const processedTags = new Set(); // Prevent duplicates
-      
-      console.log(`[DEBUG] Processing tags with restrictToExistingTags=${restrictToExistingTags}`);
-  
+
+      console.log(
+        `[DEBUG] Processing tags with restrictToExistingTags=${restrictToExistingTags}`
+      );
+
       // Process regular tags
       for (const tagName of tagsArray) {
         if (!tagName || typeof tagName !== 'string') {
@@ -592,67 +675,84 @@ class PaperlessService {
           errors.push({ tagName, error: 'Invalid tag name' });
           continue;
         }
-  
+
         const normalizedName = tagName.toLowerCase().trim();
-        
+
         // Skip empty or already processed tags
         if (!normalizedName || processedTags.has(normalizedName)) {
           continue;
         }
-  
+
         try {
           // Search for existing tag first
           let tag = await this.findExistingTag(tagName);
-          
+
           // If no existing tag found and restrictions are not enabled, create new one
           if (!tag && !restrictToExistingTags) {
             tag = await this.createTagSafely(tagName);
           } else if (!tag && restrictToExistingTags) {
-            console.log(`[DEBUG] Tag "${tagName}" does not exist and restrictions are enabled, skipping`);
-            errors.push({ tagName, error: 'Tag does not exist and restrictions are enabled' });
+            console.log(
+              `[DEBUG] Tag "${tagName}" does not exist and restrictions are enabled, skipping`
+            );
+            errors.push({
+              tagName,
+              error: 'Tag does not exist and restrictions are enabled',
+            });
             continue;
           }
-  
+
           if (tag && tag.id) {
             tagIds.push(tag.id);
             processedTags.add(normalizedName);
           }
-  
         } catch (error) {
           console.error(`[ERROR] processing tag "${tagName}":`, error.message);
           errors.push({ tagName, error: error.message });
         }
       }
-  
+
       // Add AI-Processed tag if enabled
-      if (process.env.ADD_AI_PROCESSED_TAG === 'yes' && process.env.AI_PROCESSED_TAG_NAME) {
+      if (
+        process.env.ADD_AI_PROCESSED_TAG === 'yes' &&
+        process.env.AI_PROCESSED_TAG_NAME
+      ) {
         try {
           const aiTagName = process.env.AI_PROCESSED_TAG_NAME;
           let aiTag = await this.findExistingTag(aiTagName);
-          
+
           if (!aiTag) {
             aiTag = await this.createTagSafely(aiTagName);
           }
-  
+
           if (aiTag && aiTag.id) {
             tagIds.push(aiTag.id);
           }
         } catch (error) {
-          console.error(`[ERROR] processing AI tag "${process.env.AI_PROCESSED_TAG_NAME}":`, error.message);
-          errors.push({ tagName: process.env.AI_PROCESSED_TAG_NAME, error: error.message });
+          console.error(
+            `[ERROR] processing AI tag "${process.env.AI_PROCESSED_TAG_NAME}":`,
+            error.message
+          );
+          errors.push({
+            tagName: process.env.AI_PROCESSED_TAG_NAME,
+            error: error.message,
+          });
         }
       }
-  
-      return { 
+
+      return {
         tagIds: [...new Set(tagIds)], // Remove any duplicates
-        errors 
-      };      
+        errors,
+      };
     } catch (error) {
       console.error('[ERROR] in processTags:', error);
-      throw new Error(`[ERROR] Failed to process tags: ${error.message}`);
+      // contradicting hints: 0/1 argument vs root cause missing todo
+      throw new Error('[ERROR] Failed to process tags: $.{error.message}', {
+        cause: error,
+      });
     }
   }
 
+  // todo
   async getTags() {
     this.initialize();
     if (!this.client) {
@@ -665,6 +765,7 @@ class PaperlessService {
     return Array.from(this.tagCache.values());
   }
 
+  //todo
   /**
    * Fetch tags directly from API, bypassing cache.
    * Use only when fresh data is absolutely required.
@@ -706,8 +807,7 @@ class PaperlessService {
         );
 
         // Kleine Verzögerung um die API nicht zu überlasten
-        await new Promise(resolve => setTimeout(resolve, 100));
-
+        await new Promise((resolve) => setTimeout(resolve, 100));
       } catch (error) {
         console.error(`[ERRRO] fetching tags page ${page}:`, error.message);
         if (error.response) {
@@ -721,6 +821,7 @@ class PaperlessService {
     return tags;
   }
 
+  //todo
   async getTagCount() {
     this.initialize();
     try {
@@ -736,15 +837,7 @@ class PaperlessService {
 
   async getCorrespondentCount() {
     this.initialize();
-    try {
-      const response = await this.client.get('/correspondents/', {
-        params: { count: true },
-      });
-      return response.data.count;
-    } catch (error) {
-      console.error('[ERROR] fetching correspondent count:', error.message);
-      return 0;
-    }
+    return await this.correspondents.getCount(this.client);
   }
 
   async getDocumentCount() {
@@ -779,44 +872,11 @@ class PaperlessService {
 
   async listCorrespondentsNames() {
     this.initialize();
-    let allCorrespondents = [];
-    let page = 1;
-    let hasNextPage = true;
-  
     try {
-      while (hasNextPage) {
-        const response = await this.client.get('/correspondents/', {
-          params: {
-            fields: 'id,name',
-            count: true,
-            page: page,
-            page_size: 100,
-          },
-        });
-
-        const { results, next } = response.data;
-        
-        // Füge die Ergebnisse der aktuellen Seite hinzu
-        allCorrespondents = allCorrespondents.concat(
-          results.map((correspondent) => ({
-            name: correspondent.name,
-            id: correspondent.id,
-            document_count: correspondent.document_count,
-          }))
-        );
-
-        // Prüfe, ob es eine nächste Seite gibt
-        hasNextPage = next !== null;
-        page++;
-
-        // Optional: Füge eine kleine Verzögerung hinzu, um die API nicht zu überlasten
-        if (hasNextPage) {
-          await new Promise((resolve) => setTimeout(resolve, 100));
-        }
-      }
-
-      return allCorrespondents;
-  
+      return await this.correspondents.visitAllContainedElements(
+        this.client,
+        (correspondent, result) => result.add(correspondent.name)
+      );
     } catch (error) {
       console.error('[ERROR] fetching correspondent names:', error.message);
       return [];
@@ -825,44 +885,18 @@ class PaperlessService {
 
   async listDocumentTypesNames() {
     this.initialize();
-    let allDocumentTypes = [];
-    let page = 1;
-    let hasNextPage = true;
-
     try {
-      while (hasNextPage) {
-        const response = await this.client.get('/document_types/', {
-          params: {
-            fields: 'id,name',
-            count: true,
-            page: page,
-          },
-        });
-
-        const { results, next } = response.data;
-
-        allDocumentTypes = allDocumentTypes.concat(
-          results.map((docType) => ({
-            name: docType.name,
-            id: docType.id,
-          }))
-        );
-
-        hasNextPage = next !== null;
-        page++;
-
-        if (hasNextPage) {
-          await new Promise((resolve) => setTimeout(resolve, 100));
-        }
-      }
-
-      return allDocumentTypes;
+      return await this.documentTypes.visitAllContainedElements(
+        this.client,
+        (doc_type, result) => result.add(doc_type.name)
+      );
     } catch (error) {
       console.error('[ERROR] fetching document type names:', error.message);
       return [];
     }
   }
 
+  // todo
   async listTagNames() {
     this.initialize();
     let allTags = [];
@@ -918,6 +952,7 @@ class PaperlessService {
     return [];
   }
 
+  // todo
   async resolveTagIdsByName(tagNames) {
     const normalizedTagNames = this.parseTagList(tagNames);
     if (normalizedTagNames.length === 0) {
@@ -958,9 +993,13 @@ class PaperlessService {
         return true;
       }
 
-      const hasExcludedTag = document.tags.some(tag => {
-        const normalizedTagId = typeof tag === 'object' ? Number(tag?.id) : Number(tag);
-        return Number.isInteger(normalizedTagId) && excludedTagSet.has(normalizedTagId);
+      const hasExcludedTag = document.tags.some((tag) => {
+        const normalizedTagId =
+          typeof tag === 'object' ? Number(tag?.id) : Number(tag);
+        return (
+          Number.isInteger(normalizedTagId) &&
+          excludedTagSet.has(normalizedTagId)
+        );
       });
 
       return !hasExcludedTag;
@@ -983,33 +1022,47 @@ class PaperlessService {
     let documents = [];
     let page = 1;
     let hasMore = true;
-    const shouldFilterByTags = applyFilters && process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
-    const includeTagNames = applyFilters ? this.parseTagList(process.env.TAGS) : [];
-    const excludeTagNames = applyFilters ? this.parseTagList(process.env.IGNORE_TAGS) : [];
+    const shouldFilterByTags =
+      applyFilters && process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
+    const includeTagNames = applyFilters
+      ? this.parseTagList(process.env.TAGS)
+      : [];
+    const excludeTagNames = applyFilters
+      ? this.parseTagList(process.env.IGNORE_TAGS)
+      : [];
     let includeTagIds = [];
     let excludeTagIds = [];
 
     // Vorverarbeitung der Include-Tags, wenn Filter aktiv ist
     if (shouldFilterByTags) {
       if (includeTagNames.length === 0) {
-        console.warn('[DEBUG] PROCESS_PREDEFINED_DOCUMENTS is set to yes but no TAGS are defined');
+        console.warn(
+          '[DEBUG] PROCESS_PREDEFINED_DOCUMENTS is set to yes but no TAGS are defined'
+        );
         return [];
       }
 
       includeTagIds = await this.resolveTagIdsByName(includeTagNames);
 
       if (includeTagIds.length === 0) {
-        console.warn(`[DEBUG] None of the specified tags ${includeTagNames} were found`);
+        console.warn(
+          `[DEBUG] None of the specified tags ${includeTagNames} were found`
+        );
         return [];
       }
 
-      console.log('[DEBUG] Filtering documents for include tag IDs:', includeTagIds);
+      console.log(
+        '[DEBUG] Filtering documents for include tag IDs:',
+        includeTagIds
+      );
     }
 
     if (excludeTagNames.length > 0) {
       excludeTagIds = await this.resolveTagIdsByName(excludeTagNames);
       if (excludeTagIds.length === 0) {
-        console.warn('[DEBUG] IGNORE_TAGS configured but no matching tags were found');
+        console.warn(
+          '[DEBUG] IGNORE_TAGS configured but no matching tags were found'
+        );
       } else {
         console.log('[DEBUG] Excluding documents with tag IDs:', excludeTagIds);
       }
@@ -1020,8 +1073,10 @@ class PaperlessService {
         const params = {
           page,
           page_size: 100,
-          fields: options.fields || 'id,title,created,created_date,added,tags,correspondent',
-          ordering: 'id'
+          fields:
+            options.fields ||
+            'id,title,created,created_date,added,tags,correspondent',
+          ordering: 'id',
         };
 
         // Füge Tag-Filter hinzu, wenn Tags definiert sind
@@ -1030,7 +1085,7 @@ class PaperlessService {
         }
 
         const response = await this.client.get('/documents/', { params });
-        
+
         if (!response?.data?.results || !Array.isArray(response.data.results)) {
           console.error(`[DEBUG] Invalid API response on page ${page}`);
           break;
@@ -1041,15 +1096,17 @@ class PaperlessService {
         page++;
 
         console.log(
-          `[DEBUG] Fetched page ${page-1}, got ${response.data.results.length} documents. ` +
-          `[DEBUG] Total so far: ${documents.length}`
+          `[DEBUG] Fetched page ${page - 1}, got ${response.data.results.length} documents. ` +
+            `[DEBUG] Total so far: ${documents.length}`
         );
 
         // Kleine Verzögerung um die API nicht zu überlasten
-        await new Promise(resolve => setTimeout(resolve, 100));
-
+        await new Promise((resolve) => setTimeout(resolve, 100));
       } catch (error) {
-        console.error(`[ERROR]  fetching documents page ${page}:`, error.message);
+        console.error(
+          `[ERROR]  fetching documents page ${page}:`,
+          error.message
+        );
         if (error.response) {
           console.error('[ERROR] Response status:', error.response.status);
         }
@@ -1057,26 +1114,32 @@ class PaperlessService {
       }
     }
 
-    const filteredDocuments = this.filterDocumentsByExcludedTagIds(documents, excludeTagIds);
+    const filteredDocuments = this.filterDocumentsByExcludedTagIds(
+      documents,
+      excludeTagIds
+    );
     if (excludeTagIds.length > 0) {
       console.log(
         `[DEBUG] Exclude filter removed ${documents.length - filteredDocuments.length} documents. ` +
-        `[DEBUG] Remaining: ${filteredDocuments.length}`
+          `[DEBUG] Remaining: ${filteredDocuments.length}`
       );
     }
 
-    console.log(`[DEBUG] Finished fetching. Found ${filteredDocuments.length} documents after filtering.`);
+    console.log(
+      `[DEBUG] Finished fetching. Found ${filteredDocuments.length} documents after filtering.`
+    );
     return filteredDocuments;
-}
+  }
 
   async getEffectiveDocumentCount() {
-    const shouldFilterByTags = process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
+    const shouldFilterByTags =
+      process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
     const includeTagNames = this.parseTagList(process.env.TAGS);
     const excludeTagNames = this.parseTagList(process.env.IGNORE_TAGS);
     const cacheKey = JSON.stringify({
       shouldFilterByTags,
       includeTagNames,
-      excludeTagNames
+      excludeTagNames,
     });
 
     if (
@@ -1117,7 +1180,7 @@ class PaperlessService {
         try {
           effectiveCount = await this.getDocumentCountByParams({
             ...baseCountParams,
-            tags__id__none: excludeTagIds.join(',')
+            tags__id__none: excludeTagIds.join(','),
           });
           this._supportsTagsIdNone = true;
         } catch (error) {
@@ -1133,7 +1196,9 @@ class PaperlessService {
         if (excludeTagIds.length === 0) {
           effectiveCount = await this.getDocumentCountByParams(baseCountParams);
         } else {
-          const processableDocuments = await this.getAllDocuments({ fields: 'id,tags' });
+          const processableDocuments = await this.getAllDocuments({
+            fields: 'id,tags',
+          });
           effectiveCount = processableDocuments.length;
         }
       }
@@ -1141,44 +1206,23 @@ class PaperlessService {
       this._effectiveCountCache = {
         key: cacheKey,
         count: effectiveCount,
-        expiresAt: Date.now() + this._effectiveCountCacheTtlMs
+        expiresAt: Date.now() + this._effectiveCountCacheTtlMs,
       };
 
       return effectiveCount;
     } catch (error) {
-      console.error('[ERROR] fetching effective document count:', error.message);
+      console.error(
+        '[ERROR] fetching effective document count:',
+        error.message
+      );
       return 0;
     }
   }
 
-  async getAllDocumentIds() {
-    /**
-     * Get all Document IDs from the Paperless API.
-     * 
-     * @returns    An array of all Document IDs.
-     * @throws     An error if the request fails.
-     * @note       This method is used to get all Document IDs for further processing.
-     */
-    this.initialize();
-    try {
-      const response = await this.client.get('/documents/', {
-        params: { 
-          page,
-          page_size: 100,
-          fields: 'id',
-        }
-      });
-      return response.data.results.map(doc => doc.id);
-    } catch (error) {
-      console.error('[ERROR] fetching document IDs:', error.message);
-      return [];
-    }
-  }
-
   async getAllDocumentIdsScan() {
     /**
      * Get all Document IDs from the Paperless API.
-     * 
+     *
      * @returns    An array of all Document IDs.
      * @throws     An error if the request fails.
      * @note       This method is used to get all Document IDs for further processing.
@@ -1192,32 +1236,37 @@ class PaperlessService {
     let documents = [];
     let page = 1;
     let hasMore = true;
-    const shouldFilterByTags = process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
+    const shouldFilterByTags =
+      process.env.PROCESS_PREDEFINED_DOCUMENTS === 'yes';
     let tagIds = [];
 
     // Vorverarbeitung der Tags, wenn Filter aktiv ist
     if (shouldFilterByTags) {
       if (!process.env.TAGS) {
-        console.warn('[DEBUG] PROCESS_PREDEFINED_DOCUMENTS is set to yes but no TAGS are defined');
+        console.warn(
+          '[DEBUG] PROCESS_PREDEFINED_DOCUMENTS is set to yes but no TAGS are defined'
+        );
         return [];
       }
-      
+
       // Hole die Tag-IDs für die definierten Tags
-      const tagNames = process.env.TAGS.split(',').map(tag => tag.trim());
+      const tagNames = process.env.TAGS.split(',').map((tag) => tag.trim());
       await this.ensureTagCache();
-      
+
       for (const tagName of tagNames) {
         const tag = await this.findExistingTag(tagName);
         if (tag) {
           tagIds.push(tag.id);
         }
       }
-      
+
       if (tagIds.length === 0) {
-        console.warn(`[DEBUG] None of the specified tags ${tagNames} were found`);
+        console.warn(
+          `[DEBUG] None of the specified tags ${tagNames} were found`
+        );
         return [];
       }
-      
+
       console.log('[DEBUG] Filtering documents for tag IDs:', tagIds);
     }
 
@@ -1227,11 +1276,11 @@ class PaperlessService {
           page,
           page_size: 100,
           fields: 'id',
-          ordering: 'id'
+          ordering: 'id',
         };
 
         const response = await this.client.get('/documents/', { params });
-        
+
         if (!response?.data?.results || !Array.isArray(response.data.results)) {
           console.error(`[ERROR] Invalid API response on page ${page}`);
           break;
@@ -1242,15 +1291,17 @@ class PaperlessService {
         page++;
 
         console.log(
-          `[DEBUG] Fetched page ${page-1}, got ${response.data.results.length} documents. ` +
-          `[DEBUG] Total so far: ${documents.length}`
+          `[DEBUG] Fetched page ${page - 1}, got ${response.data.results.length} documents. ` +
+            `[DEBUG] Total so far: ${documents.length}`
         );
 
         // Kleine Verzögerung um die API nicht zu überlasten
-        await new Promise(resolve => setTimeout(resolve, 100));
-
+        await new Promise((resolve) => setTimeout(resolve, 100));
       } catch (error) {
-        console.error(`[ERROR] fetching documents page ${page}:`, error.message);
+        console.error(
+          `[ERROR] fetching documents page ${page}:`,
+          error.message
+        );
         if (error.response) {
           console.error('[DEBUG] Response status:', error.response.status);
         }
@@ -1258,34 +1309,39 @@ class PaperlessService {
       }
     }
 
-    console.log(`[DEBUG] Finished fetching. Found ${documents.length} documents.`);
+    console.log(
+      `[INFO] Finished fetching. Found ${documents.length} documents.`
+    );
     return documents;
   }
 
+  /**
+   * Get the Name of a Correspondent by its ID.
+   *
+   * @param   correspondentId  The id of the correspondent.
+   * @returns    The name of the correspondent.
+   */
   async getCorrespondentNameById(correspondentId) {
-    /**
-     * Get the Name of a Correspondent by its ID.
-     * 
-     * @param   id  The id of the correspondent.
-     * @returns    The name of the correspondent.
-     */
     this.initialize();
-    try {
-      const response = await this.client.get(`/correspondents/${correspondentId}/`);
-      return response.data;
-    } catch (error) {
-      console.error(`[ERROR] fetching correspondent ${correspondentId}:`, error.message);
-      return null;
-    }
+    return this.correspondents.getById(this.client, correspondentId)
   }
 
   async getDocumentTypeNameById(documentTypeId) {
     this.initialize();
     try {
-      const response = await this.client.get(`/document_types/${documentTypeId}/`);
-      return response.data;
+      let documentType = this.documentTypes.getById(this.client, documentTypeId);
+      if (documentType) {
+        return documentType.name;
+      }
+      console.log(
+        `[DEBUG] Document type with ID #${documentTypeId} does not exist yet.`
+      );
+      return null;
     } catch (error) {
-      console.error(`[ERROR] fetching document type ${documentTypeId}:`, error.message);
+      console.error(
+        `[ERROR] fetching document type #${documentTypeId}:`,
+        error.message
+      );
       return null;
     }
   }
@@ -1302,26 +1358,29 @@ class PaperlessService {
       const response = await this.client.get(`/tags/${tagId}/`);
       return response.data.name;
     } catch (error) {
-      console.error(`[ERROR] fetching tag name for ID ${tagId}:`, error.message);
+      console.error(
+        `[ERROR] fetching tag name for ID ${tagId}:`,
+        error.message
+      );
       return null;
     }
   }
 
-  async getDocumentsWithTitleTagsCorrespondentCreated () {
+  async getDocumentsWithTitleTagsCorrespondentCreated() {
     /**
      * Get all documents with metadata (title, tags, correspondent, created date).
-     * 
+     *
      * @returns    An array of documents with metadata.
      * @throws     An error if the request fails.
-     * @note       This method is used to get all documents with metadata for further processing 
+     * @note       This method is used to get all documents with metadata for further processing
      */
-    
+
     this.initialize();
     try {
       const response = await this.client.get('/documents/', {
         params: {
-          fields: 'id,title,tags,correspondent,created'
-        }
+          fields: 'id,title,tags,correspondent,created',
+        },
       });
       return response.data.results;
     } catch (error) {
@@ -1333,7 +1392,9 @@ class PaperlessService {
   async getRecentDocumentsWithMetadata(limit = 16) {
     this.initialize();
 
-    const safeLimit = Number.isInteger(Number(limit)) ? Math.max(1, Math.min(Number(limit), 100)) : 16;
+    const safeLimit = Number.isInteger(Number(limit))
+      ? Math.max(1, Math.min(Number(limit), 100))
+      : 16;
 
     try {
       const response = await this.client.get('/documents/', {
@@ -1341,8 +1402,8 @@ class PaperlessService {
           fields: 'id,title,tags,correspondent,created',
           page: 1,
           page_size: safeLimit,
-          ordering: '-created'
-        }
+          ordering: '-created',
+        },
       });
 
       if (!Array.isArray(response?.data?.results)) {
@@ -1351,7 +1412,10 @@ class PaperlessService {
 
       return response.data.results;
     } catch (error) {
-      console.error('[ERROR] fetching recent documents with metadata:', error.message);
+      console.error(
+        '[ERROR] fetching recent documents with metadata:',
+        error.message
+      );
       return [];
     }
   }
@@ -1379,272 +1443,247 @@ class PaperlessService {
   }
 
   async searchForCorrespondentById(id) {
+    this.initialize();
     try {
-      const response = await this.client.get('/correspondents/', {
-          params: {
-              id: id
-          }
-      });
-
-      const results = response.data.results;
-      
-      if (results.length === 0) {
-          console.log(`[DEBUG] No correspondent with "${id}" found`);
-          return null;
-      }
-      
-      if (results.length > 1) {
-          console.log(`[DEBUG] Multiple correspondents found:`);
-          results.forEach(c => {
-              console.log(`- ID: ${c.id}, Name: ${c.name}`);
-          });
-          return results;
-      }
-
-      // Genau ein Ergebnis gefunden
-      return {
-          id: results[0].id,
-          name: results[0].name
-      };
-
-  } catch (error) {
-      console.error('[ERROR] while seraching for existing correspondent:', error.message);
+      return await this.correspondents.getById(this.client, id);
+    } catch (error) {
+      console.error(
+        '[ERROR] while searching for existing correspondent:',
+        error.message
+      );
       throw error;
+    }
   }
-}
-
-async searchForExistingCorrespondent(correspondent) {
-  try {
-      const response = await this.client.get('/correspondents/', {
-          params: {
-              name__icontains: correspondent
-          }
-      });
-
-      const results = response.data.results;
-      
-      if (results.length === 0) {
-          console.log(`[DEBUG] No correspondent with name "${correspondent}" found`);
-          return null;
-      }
-      
-      // Check for exact match in the results - thanks to @skius for the hint!
-      const exactMatch = results.find(c => c.name.toLowerCase() === correspondent.toLowerCase());
-      if (exactMatch) {
-          console.log(`[DEBUG] Found exact match for correspondent "${correspondent}" with ID ${exactMatch.id}`);
-          return {
-              id: exactMatch.id,
-              name: exactMatch.name
-          };
-      }
-
-      // No exact match found, return null
-      console.log(`[DEBUG] No exact match found for "${correspondent}"`);
-      return null;
 
-  } catch (error) {
-      console.error('[ERROR] while searching for existing correspondent:', error.message);
+  async searchForExistingCorrespondent(correspondent) {
+    try {
+      this.initialize();
+      return await this.correspondents.getByName(this.client, correspondent);
+    } catch (error) {
+      console.error(
+        '[ERROR] while searching for existing correspondent:',
+        error.message
+      );
       throw error;
+    }
   }
-}
 
+
+  // todo
   async getOrCreateCorrespondent(name, options = {}) {
     this.initialize();
-    
+
     // Check if we should restrict to existing correspondents
     // Explicitly check options first, then env var
-    const restrictToExistingCorrespondents = options.restrictToExistingCorrespondents === true || 
-                                           (options.restrictToExistingCorrespondents === undefined && 
-                                            process.env.RESTRICT_TO_EXISTING_CORRESPONDENTS === 'yes');
-    
-    console.log(`[DEBUG] Processing correspondent with restrictToExistingCorrespondents=${restrictToExistingCorrespondents}`);
-  
-    try {
-        // Search for the correspondent
-        const existingCorrespondent = await this.searchForExistingCorrespondent(name);
-        console.log("[DEBUG] Response Correspondent Search: ", existingCorrespondent);
-    
-        if (existingCorrespondent) {
-            console.log(`[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}`);
-            return existingCorrespondent;
-        }
-        
-        // If we're restricting to existing correspondents and none was found, return null
-        if (restrictToExistingCorrespondents) {
-            console.log(`[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null`);
-            return null;
-        }
-    
-        // Create new correspondent only if restrictions are not enabled
-        try {
-            const createResponse = await this.client.post('/correspondents/', { 
-                name: name 
-            });
-            console.log(`[DEBUG] Created new correspondent "${name}" with ID ${createResponse.data.id}`);
-            return createResponse.data;
-        } catch (createError) {
-            if (createError.response?.status === 400 && 
-                createError.response?.data?.error?.includes('unique constraint')) {
-              
-                // Race condition check - another process might have created it
-                const retryResponse = await this.client.get('/correspondents/', {
-                    params: { name: name }
-                });
-              
-                const justCreatedCorrespondent = retryResponse.data.results.find(
-                    c => c.name.toLowerCase() === name.toLowerCase()
-                );
-              
-                if (justCreatedCorrespondent) {
-                    console.log(`[DEBUG] Retrieved correspondent "${name}" after constraint error with ID ${justCreatedCorrespondent.id}`);
-                    return justCreatedCorrespondent;
-                }
-            }
-            throw createError;
-        }
-    } catch (error) {
-        console.error(`[ERROR] Failed to process correspondent "${name}":`, error.message);
-        throw error;
-    }
-}
+    const restrictToExistingCorrespondents =
+      options.restrictToExistingCorrespondents === true ||
+      (options.restrictToExistingCorrespondents === undefined &&
+        process.env.RESTRICT_TO_EXISTING_CORRESPONDENTS === 'yes');
 
-async searchForExistingDocumentType(documentType) {
-  try {
-      const response = await this.client.get('/document_types/', {
-          params: {
-              name__icontains: documentType
-          }
-      });
+    console.log(
+      `[DEBUG] Processing correspondent with restrictToExistingCorrespondents=${restrictToExistingCorrespondents}`
+    );
 
-      const results = response.data.results;
-      
-      if (results.length === 0) {
-          console.log(`[DEBUG] No document type with name "${documentType}" found`);
-          return null;
+    try {
+      // Search for the correspondent
+      const existingCorrespondent = await this.searchForExistingCorrespondent(name);
+
+      if (existingCorrespondent) {
+        console.log(`[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}`);
+        return existingCorrespondent;
       }
-      
-      // Check for exact match in the results
-      const exactMatch = results.find(dt => dt.name.toLowerCase() === documentType.toLowerCase());
-      if (exactMatch) {
-          console.log(`[DEBUG] Found exact match for document type "${documentType}" with ID ${exactMatch.id}`);
-          return {
-              id: exactMatch.id,
-              name: exactMatch.name
-          };
+
+      // If we're restricting to existing correspondents and none was found, return null
+      if (restrictToExistingCorrespondents) {
+        console.log(`[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null`);
+        return null;
       }
 
-      // No exact match found, return null
-      console.log(`[DEBUG] No exact match found for "${documentType}"`);
-      return null;
+      // Create new correspondent only if restrictions are not enabled
+      try {
+        const createResponse = await this.client.post('/correspondents/', {
+          name: name,
+        });
+        console.log(
+          `[DEBUG] Created new correspondent "${name}" with ID ${createResponse.data.id}`
+        );
+        return createResponse.data;
+      } catch (createError) {
+        if (
+          createError.response?.status === 400 &&
+          createError.response?.data?.error?.includes('unique constraint')
+        ) {
+          // Race condition check - another process might have created it
+          const retryResponse = await this.client.get('/correspondents/', {
+            params: { name: name },
+          });
+
+          const justCreatedCorrespondent = retryResponse.data.results.find(
+            (c) => c.name.toLowerCase() === name.toLowerCase()
+          );
 
-  } catch (error) {
-      console.error('[ERROR] while searching for existing document type:', error.message);
+          if (justCreatedCorrespondent) {
+            console.log(
+              `[DEBUG] Retrieved correspondent "${name}" after constraint error with ID ${justCreatedCorrespondent.id}`
+            );
+            return justCreatedCorrespondent;
+          }
+        }
+        throw createError;
+      }
+    } catch (error) {
+      console.error(
+        `[ERROR] Failed to process correspondent "${name}":`,
+        error.message
+      );
       throw error;
+    }
   }
-}
 
-async getOrCreateDocumentType(name, options = {}) {
-  this.initialize();
+  async searchForExistingDocumentType(documentType) {
+    this.initialize();
+    try {
+      return await this.documentTypes.getByName(this.client, documentType);
+    } catch (error) {
+      console.error(
+        '[ERROR] while searching for existing document type:',
+        error.message
+      );
+      throw error;
+    }
+  }
 
-  // Explicit option value wins; otherwise fall back to env config.
-  const restrictToExistingDocumentTypes = options.restrictToExistingDocumentTypes === true ||
-                                        (options.restrictToExistingDocumentTypes === undefined &&
-                                         process.env.RESTRICT_TO_EXISTING_DOCUMENT_TYPES === 'yes');
+  async getOrCreateDocumentType(name, options = {}) {
+    this.initialize();
 
-  console.log(`[DEBUG] Processing document type with restrictToExistingDocumentTypes=${restrictToExistingDocumentTypes}`);
-  
-  try {
+    // Explicit option value wins; otherwise fall back to env config.
+    const restrictToExistingDocumentTypes =
+      options.restrictToExistingDocumentTypes === true ||
+      (options.restrictToExistingDocumentTypes === undefined &&
+        process.env.RESTRICT_TO_EXISTING_DOCUMENT_TYPES === 'yes');
+
+    console.log(
+      `[DEBUG] Processing document type with restrictToExistingDocumentTypes=${restrictToExistingDocumentTypes}`
+    );
+
+    try {
       // Suche nach existierendem document_type
       const existingDocType = await this.searchForExistingDocumentType(name);
-      console.log("[DEBUG] Response Document Type Search: ", existingDocType);
-  
+      console.log('[DEBUG] Response Document Type Search: ', existingDocType);
+
       if (existingDocType) {
-          console.log(`[DEBUG] Found existing document type "${name}" with ID ${existingDocType.id}`);
-          return existingDocType;
+        console.log(
+          `[DEBUG] Found existing document type "${name}" with ID ${existingDocType.id}`
+        );
+        return existingDocType;
+      }
+
+      if (restrictToExistingDocumentTypes) {
+        console.log(
+          `[DEBUG] Document type "${name}" does not exist and restrictions are enabled, returning null`
+        );
+        return null;
       }
 
-        if (restrictToExistingDocumentTypes) {
-          console.log(`[DEBUG] Document type "${name}" does not exist and restrictions are enabled, returning null`);
-          return null;
-        }
-  
       // Erstelle neuen document_type
       try {
-          const createResponse = await this.client.post('/document_types/', { 
-              name: name,
-              matching_algorithm: 1, // 1 = ANY
-              match: "",  // Optional: Kann später angepasst werden
-              is_insensitive: true
-          });
-          console.log(`[DEBUG] Created new document type "${name}" with ID ${createResponse.data.id}`);
-          return createResponse.data;
+        const createResponse = await this.client.post('/document_types/', {
+          name: name,
+          matching_algorithm: 1, // 1 = ANY
+          match: '', // Optional: Kann später angepasst werden
+          is_insensitive: true,
+        });
+        console.log(
+          `[DEBUG] Created new document type "${name}" with ID ${createResponse.data.id}`
+        );
+        return createResponse.data;
       } catch (createError) {
-          if (createError.response?.status === 400 && 
-              createError.response?.data?.error?.includes('unique constraint')) {
-            
-              // Race condition check
-              const retryResponse = await this.client.get('/document_types/', {
-                  params: { name: name }
-              });
-            
-              const justCreatedDocType = retryResponse.data.results.find(
-                  dt => dt.name.toLowerCase() === name.toLowerCase()
-              );
-            
-              if (justCreatedDocType) {
-                  console.log(`[DEBUG] Retrieved document type "${name}" after constraint error with ID ${justCreatedDocType.id}`);
-                  return justCreatedDocType;
-              }
+        if (
+          createError.response?.status === 400 &&
+          createError.response?.data?.error?.includes('unique constraint')
+        ) {
+          // Race condition check
+          const retryResponse = await this.client.get('/document_types/', {
+            params: { name: name },
+          });
+
+          const justCreatedDocType = retryResponse.data.results.find(
+            (dt) => dt.name.toLowerCase() === name.toLowerCase()
+          );
+
+          if (justCreatedDocType) {
+            console.log(
+              `[DEBUG] Retrieved document type "${name}" after constraint error with ID ${justCreatedDocType.id}`
+            );
+            return justCreatedDocType;
           }
-          throw createError;
+        }
+        throw createError;
       }
-  } catch (error) {
-      console.error(`[ERROR] Failed to process document type "${name}":`, error.message);
+    } catch (error) {
+      console.error(
+        `[ERROR] Failed to process document type "${name}":`,
+        error.message
+      );
       throw error;
+    }
   }
-}
 
   async removeUnusedTagsFromDocument(documentId, keepTagIds) {
     this.initialize();
     if (!this.client) return;
-  
+
     try {
-      console.log(`[DEBUG] Removing unused tags from document ${documentId}, keeping tags:`, keepTagIds);
-      
+      console.log(
+        `[DEBUG] Removing unused tags from document ${documentId}, keeping tags:`,
+        keepTagIds
+      );
+
       // Hole aktuelles Dokument
       const currentDoc = await this.getDocument(documentId);
-      
+
       // Finde Tags die entfernt werden sollen (die nicht in keepTagIds sind)
-      const tagsToRemove = currentDoc.tags.filter(tagId => !keepTagIds.includes(tagId));
-      
+      const tagsToRemove = currentDoc.tags.filter(
+        (tagId) => !keepTagIds.includes(tagId)
+      );
+
       if (tagsToRemove.length === 0) {
         console.log('[DEBUG] No tags to remove');
         return currentDoc;
       }
-  
+
       // Update das Dokument mit nur den zu behaltenden Tags
       const updateData = {
-        tags: keepTagIds
+        tags: keepTagIds,
       };
-  
+
       // Führe das Update durch
       await this.client.patch(`/documents/${documentId}/`, updateData);
-      console.log(`[DEBUG] Successfully removed ${tagsToRemove.length} tags from document ${documentId}`);
-      
+      console.log(
+        `[DEBUG] Successfully removed ${tagsToRemove.length} tags from document ${documentId}`
+      );
+
       return await this.getDocument(documentId);
     } catch (error) {
-      console.error(`[ERROR] Error removing unused tags from document ${documentId}:`, error.message);
+      console.error(
+        `[ERROR] Error removing unused tags from document ${documentId}:`,
+        error.message
+      );
       throw error;
     }
   }
 
+  // todo
   async getTagTextFromId(tagId) {
     this.initialize();
     try {
-      const response = await this.client.get(`/tags/${tagId}/`);
-      return response.data.name;
+        const response = await this.client.get(`/tags/${tagId}/`);
+        return response.data.name;
     } catch (error) {
-      console.error(`[ERROR] fetching tag text for ID ${tagId}:`, error.message);
+      console.error(
+        `[ERROR] fetching tag text for ID ${tagId}:`,
+        error.message
+      );
       return null;
     }
   }
@@ -1652,28 +1691,30 @@ async getOrCreateDocumentType(name, options = {}) {
   async getOwnUserID() {
     this.initialize();
     try {
-        const response = await this.client.get('/users/', {
-            params: {
-                current_user: true,
-                full_perms: true
-            }
-        });
-        
-        if (response.data.results && response.data.results.length > 0) {
-            const userInfo = response.data.results;
-            //filter for username by process.env.PAPERLESS_USERNAME
-            const user = userInfo.find(user => user.username === process.env.PAPERLESS_USERNAME);
-            if (user) {
-                console.log(`[DEBUG] Found own user ID: ${user.id}`);
-                return user.id;
-            }
+      const response = await this.client.get('/users/', {
+        params: {
+          current_user: true,
+          full_perms: true,
+        },
+      });
+
+      if (response.data.results && response.data.results.length > 0) {
+        const userInfo = response.data.results;
+        //filter for username by process.env.PAPERLESS_USERNAME
+        const user = userInfo.find(
+          (user) => user.username === process.env.PAPERLESS_USERNAME
+        );
+        if (user) {
+          console.log(`[DEBUG] Found own user ID: ${user.id}`);
+          return user.id;
         }
-        return null;
+      }
+      return null;
     } catch (error) {
-        console.error('[ERROR] fetching own user ID:', error.message);
-        return null;
+      console.error('[ERROR] fetching own user ID:', error.message);
+      return null;
     }
-}
+  }
   //Remove if not needed?
   async getOwnerOfDocument(documentId) {
     this.initialize();
@@ -1681,7 +1722,10 @@ async getOrCreateDocumentType(name, options = {}) {
       const response = await this.client.get(`/documents/${documentId}/`);
       return response.data.owner;
     } catch (error) {
-      console.error(`[ERROR] fetching owner of document ${documentId}:`, error.message);
+      console.error(
+        `[ERROR] fetching owner of document ${documentId}:`,
+        error.message
+      );
       return null;
     }
   }
@@ -1693,31 +1737,41 @@ async getOrCreateDocumentType(name, options = {}) {
       const response = await this.client.get(`/documents/${documentId}/`);
       return response.data.user_can_change;
     } catch (error) {
-      console.error(`[ERROR] No Permission to edit document ${documentId}:`, error.message);
+      console.error(
+        `[ERROR] No Permission to edit document ${documentId}:`,
+        error.message
+      );
       return null;
     }
   }
 
-
   async updateDocument(documentId, updates) {
     this.initialize();
     if (!this.client) return;
     try {
       const currentDoc = await this.getDocument(documentId);
-      
+
       if (updates.tags) {
-        console.log(`[DEBUG] Current tags for document ${documentId}:`, currentDoc.tags);
+        console.log(
+          `[DEBUG] Current tags for document ${documentId}:`,
+          currentDoc.tags
+        );
         console.log(`[DEBUG] Adding new tags:`, updates.tags);
         console.log(`[DEBUG] Current correspondent:`, currentDoc.correspondent);
         console.log(`[DEBUG] New correspondent:`, updates.correspondent);
-                
-        const combinedTags = [...new Set([...currentDoc.tags, ...updates.tags])];
+
+        const combinedTags = [
+          ...new Set([...currentDoc.tags, ...updates.tags]),
+        ];
         updates.tags = combinedTags;
-        
+
         console.log(`[DEBUG] Combined tags:`, combinedTags);
       }
 
-      if (updates.correspondent === null || updates.correspondent === undefined) {
+      if (
+        updates.correspondent === null ||
+        updates.correspondent === undefined
+      ) {
         // Keep existing correspondent when no new value is provided
         delete updates.correspondent;
       }
@@ -1741,10 +1795,14 @@ async getOrCreateDocumentType(name, options = {}) {
           }
 
           if (!isValid(dateObject)) {
-            console.warn(`[WARN] Invalid date format: ${updates.created}, skipping date update`);
+            console.warn(
+              `[WARN] Invalid date format: ${updates.created}, skipping date update`
+            );
             delete updates.created;
           } else if (dateObject > new Date()) {
-            console.warn(`[WARN] AI returned future date ${format(dateObject, 'yyyy-MM-dd')}, skipping date update`);
+            console.warn(
+              `[WARN] AI returned future date ${format(dateObject, 'yyyy-MM-dd')}, skipping date update`
+            );
             delete updates.created;
           } else {
             updates.created = format(dateObject, 'yyyy-MM-dd');
@@ -1774,16 +1832,18 @@ async getOrCreateDocumentType(name, options = {}) {
       //     await this.client.patch(`/documents/${documentId}/`, { custom_fields: [] });
       //   }
       // }
-      
+
       // Validate title length before sending to API
       if (updateData.title && updateData.title.length > 128) {
         updateData.title = updateData.title.substring(0, 124) + '…';
-        console.warn(`[WARN] Title truncated to 128 characters for document ${documentId}`);
+        console.warn(
+          `[WARN] Title truncated to 128 characters for document ${documentId}`
+        );
       }
-      
+
       console.log('[DEBUG] Final update data:', updateData);
       await this.client.patch(`/documents/${documentId}/`, updateData);
-      console.log(`[SUCCESS] Updated document ${documentId} with:`, updateData);
+      console.log(`[SUCCESS] Updated document ${documentId}`);
       return await this.getDocument(documentId);
     } catch (error) {
       console.log(error);
@@ -1804,13 +1864,18 @@ async getOrCreateDocumentType(name, options = {}) {
     if (!this.client) return null;
     try {
       const patch = {};
-      if (Array.isArray(original.tags))          patch.tags          = original.tags;
-      if (original.title != null)                 patch.title         = original.title;
-      if (original.correspondent !== undefined)   patch.correspondent = original.correspondent;
-      if (original.documentType   !== undefined)  patch.document_type = original.documentType;
-      if (original.language       != null)        patch.language      = original.language;
+      if (Array.isArray(original.tags)) patch.tags = original.tags;
+      if (original.title != null) patch.title = original.title;
+      if (original.correspondent !== undefined)
+        patch.correspondent = original.correspondent;
+      if (original.documentType !== undefined)
+        patch.document_type = original.documentType;
+      if (original.language != null) patch.language = original.language;
 
-      console.log(`[DEBUG] Restoring document ${documentId} to original state:`, patch);
+      console.log(
+        `[DEBUG] Restoring document ${documentId} to original state:`,
+        patch
+      );
       await this.client.patch(`/documents/${documentId}/`, patch);
       console.log(`[SUCCESS] Restored document ${documentId}`);
       return await this.getDocument(documentId);

From 7274e73d4fb6dd09218b0d4ed149ccc5c1e788a6 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 9 Aug 2026 22:44:28 +0200
Subject: [PATCH 07/53] prevent to publish IDE configuration

---
 build.sh                          |   0
 services/CachedNameIdEnum.js      | 357 ++++++++++++++++++++++++++++++
 tests/test-cached-name-id-enum.js | 261 ++++++++++++++++++++++
 3 files changed, 618 insertions(+)
 mode change 100755 => 100644 build.sh
 create mode 100644 services/CachedNameIdEnum.js
 create mode 100644 tests/test-cached-name-id-enum.js

diff --git a/build.sh b/build.sh
old mode 100755
new mode 100644
diff --git a/services/CachedNameIdEnum.js b/services/CachedNameIdEnum.js
new file mode 100644
index 00000000..1db0f5cc
--- /dev/null
+++ b/services/CachedNameIdEnum.js
@@ -0,0 +1,357 @@
+const { safeExtractRelativePath } = require('./serviceUtils');
+
+class CachedNameIdEnum {
+  constructor(
+    element_type_name,
+    url_prefix,
+    similarity_prompt,
+    cache_ttl_access,
+    ai_service
+  ) {
+    this.ai_service = ai_service;
+    this.element_type_name = element_type_name;
+    this.url_prefix = url_prefix;
+    this.similarity_prompt = similarity_prompt;
+    this.cache_ttl_access = cache_ttl_access;
+    this.cache_last_fetch_time = 0;
+    this.cache_refresh_promise = null;
+    this.elements_by_id = new Map();
+    this.elements_by_name = new Map();
+    this.element_list = [];
+  }
+
+  _clear() {
+    this.elements_by_id.clear();
+    this.elements_by_name.clear();
+    this.element_list = [];
+  }
+
+  _add_all(elements) {
+    this.element_list = this.element_list.concat(elements);
+    this.element_list.forEach((element) => {
+      this.elements_by_id.set(element.id, element);
+      this.elements_by_name.set(element.name.toLowerCase(), element);
+    });
+  }
+
+  async _cache_refill(client) {
+    this._clear();
+    this._add_all(await this._fetchAll(client));
+  }
+
+  _result_element_from_element(element) {
+    return {
+      id: element.id,
+      name: element.name,
+    };
+  }
+
+  async _fetchAll(client) {
+    this._checkClient(client);
+    let nextUrl = this.url_prefix;
+    let existing_elements = [];
+    while (nextUrl) {
+      const response = await client.get(nextUrl, { page_size: 100 });
+      existing_elements = existing_elements.concat(response.data.results);
+      // Safely extract relative path from next URL to prevent SSRF
+      if (response.data.next) {
+        nextUrl = safeExtractRelativePath(response.data.next, client);
+        if (nextUrl) {
+          console.log(
+            `[DEBUG] Next page URL: ${nextUrl} elements fetched so far: ${existing_elements.length}`
+          );
+        } else {
+          console.warn(`[WARN] Response has data.next but Next page URL is 
+            empty. elements fetched so far: ${existing_elements.length}`);
+        }
+      } else {
+        nextUrl = null;
+        this.cache_last_fetch_time = Date.now();
+        console.log(
+          `[DEBUG] Fetched ${existing_elements.length} ${this.element_type_name}s`
+        );
+        console.log(
+          `[DEBUG] Cache contains these ${this.element_type_name}s now: 
+          ${existing_elements.map((element) => element.name).join(', ')}`
+        );
+      }
+    }
+    return existing_elements;
+  }
+
+  async _check_cache_is_not_outdated(
+    client,
+    cache_ttl = this.cache_ttl_access()
+  ) {
+    if (this.element_list.length === 0) {
+      await this.flush_cache(client);
+      return;
+    }
+    if (cache_ttl === 0) return; // No timeout configured, refresh only on cache miss
+    if (cache_ttl > 0) {
+      // check for timeout and refresh if necessary
+      const cacheAge = Date.now() - this.cache_last_fetch_time;
+      if (cacheAge < cache_ttl) return;
+      const ageSeconds = Math.floor(cacheAge / 1000);
+      const ttlSeconds = Math.floor(cache_ttl / 1000);
+      const expireTime = new Date(
+        this.cache_last_fetch_time + cache_ttl
+      ).toISOString();
+      console.log(
+        `[DEBUG] ${this.element_type_name} cache expired (age: ${ageSeconds}s, TTL: ${ttlSeconds}s, expired at: ${expireTime})`
+      );
+    }
+    // cache_ttl < 0 or expired
+    await this.flush_cache(client);
+  }
+
+  async _getByName(name, client, cache_ttl = this.cache_ttl_access()) {
+    await this._check_cache_is_not_outdated(client, cache_ttl);
+    let match = this.elements_by_name.get(name.toLowerCase());
+    if (match) {
+      console.log(
+        `[DEBUG] Found exact match for ${this.element_type_name} "${match.name}" with ID ${match.id}`
+      );
+      return this._result_element_from_element(match);
+    }
+    match = await this._findSimilar(name);
+    if (match) {
+      console.log(
+        `[DEBUG] Found similar match for ${this.element_type_name} "${name}": "${match.name}" with ID ${match.id}`
+      );
+      return this._result_element_from_element(match);
+    }
+    console.log(
+      `[DEBUG] No exact nor similar match found for ${this.element_type_name} "${name}"`
+    );
+    return null;
+  }
+
+  async _getById(id, client, cache_ttl = this.cache_ttl_access()) {
+    if (id === 0) {
+      console.error(`[ERROR] Id cannot be 0` + new Error().stack);
+      return null;
+    }
+    await this._check_cache_is_not_outdated(client, cache_ttl);
+    let match = this.elements_by_id.get(id);
+    if (match) {
+      console.log(
+        `[DEBUG] Found ${this.element_type_name} by ID #${match.id}: "${match.name}"`
+      );
+      return this._result_element_from_element(match);
+    }
+    console.log(`[DEBUG] No ${this.element_type_name} with ID #${id} found`);
+    return null;
+  }
+
+  async _findSemanticMatch(name) {
+    let selectPrompt =
+      `These ${this.element_type_name}s are known: ` +
+      `${this.element_list.map((e) => `${e.name} (ID: ${e.id})`).join(', ')}. ` +
+      `Is ${name} contained in this list? ${this.similarity_prompt}`;
+
+    const format = {
+      type: 'object',
+      properties: {
+        match: {
+          type: ['object', 'null'],
+          properties: {
+            id: { type: 'number' },
+            name: { type: 'string' },
+          },
+          required: ['id', 'name'],
+        },
+      },
+      required: ['match'],
+    };
+    try {
+      let aiResult = await this.ai_service.generateAnswer(
+        selectPrompt,
+        undefined,
+        300,
+        format
+      );
+      console.debug(`[DEBUG] AI result: ${aiResult}`);
+      let parsedResult =
+        typeof aiResult === 'string' ? JSON.parse(aiResult) : aiResult;
+      if (!parsedResult.match) {
+        console.debug(
+          `[DEBUG] No ${this.element_type_name} found similar to "${name}"`
+        );
+        return null;
+      }
+      const matchedElement = parsedResult.match;
+      console.debug(
+        `[DEBUG] Found similar match for ${this.element_type_name} "${name}": "${matchedElement.name}" with ID ${matchedElement.id}`
+      );
+      return this._result_element_from_element(matchedElement);
+    } catch (error) {
+      console.error(
+        '[ERROR] while searching for existing correspondent:',
+        error.message
+      );
+    }
+
+    return null;
+  }
+
+  async _findMatchByFilter(name, filter, filterName) {
+    let elementResultList = await this.element_list.filter(filter);
+    if (elementResultList.length === 0) {
+      console.log(
+        `[DEBUG] No ${this.element_type_name} found with ${filterName} "${name}"`
+      );
+      return null;
+    }
+    if (elementResultList.length > 1) {
+      console.log(
+        `[DEBUG] Multiple ${this.element_type_name} found with a ${filterName} "${name}"`,
+        `[DEBUG] "${elementResultList.map((element) => element.name).join('", "')}"`
+      );
+      return null;
+    }
+    const matchedElement = elementResultList[0];
+    console.log(
+      `[DEBUG] Found "${filterName}" match for ${this.element_type_name} "${name}": "${matchedElement.name}" with ID ${matchedElement.id}`
+    );
+    return this._result_element_from_element(matchedElement);
+  }
+
+  async _findSubString(name) {
+    return this._findMatchByFilter(
+      name,
+      (element) => element.name.toLowerCase().includes(name.toLowerCase()),
+      'substring in'
+    );
+  }
+
+  _splitToTerms(name) {
+    let terms = name
+      .toLowerCase()
+      .replace(/[\s-_.:]+/g, ' ')
+      .split(' ');
+    terms = terms.filter((term) => term.length > 0);
+    console.debug(`[debug] ${name} in terms: ${terms.join(', ')}`);
+    return terms;
+  }
+
+  _isMatchBySubterms(name, candidate) {
+    let nameTerms = this._splitToTerms(name);
+    let candidateTerms = this._splitToTerms(candidate);
+    const termsToMatch = candidateTerms.length;
+    let matchNo = 0;
+    for (let candidateTerm of candidateTerms) {
+      if (nameTerms.includes(candidateTerm)) {
+        matchNo++;
+      }
+    }
+    let result = termsToMatch > 0 && matchNo === termsToMatch;
+    console.debug(
+      `[debug] ${name} is ${result ? '' : 'not'} in: ${candidateTerms.join(', ')}`
+    );
+    return result;
+  }
+
+  async _findByTermMatch(name) {
+    return this._findMatchByFilter(
+      name,
+      (element) => this._isMatchBySubterms(name, element.name),
+      'has all terms'
+    );
+  }
+
+  async _findSimilar(name) {
+    // Todo: implement methods to find similar element by matching with single word permutations and with something
+    //  like hemming distance before asking ai to spare tokens
+    let match = await this._findSubString(name);
+    if (match) {
+      return match;
+    }
+    match = await this._findByTermMatch(name);
+    if (match) {
+      return match;
+    }
+    return this._findSemanticMatch(name);
+  }
+
+  _checkClient(client) {
+    if (!client) {
+      throw new Error('No client provided.');
+    }
+  }
+
+  _checkDefined(name, value) {
+    if (value === undefined || value === null) {
+      throw new Error(`Mandatory argument ${name} is undefined or null.`);
+    }
+  }
+
+  /**************************************************************************
+   *
+   * Public API
+   *
+   *************************************************************************/
+
+  flush_cache(client) {
+    this._checkClient(client);
+    if (this.cache_refresh_promise) {
+      return this.cache_refresh_promise;
+    }
+    // No race condition: synchronous code is never preempted in Node.js's
+    // event loop, so no other call can reach here between the check above
+    // and the assignment below.
+    this.cache_refresh_promise = this._cache_refill(client).finally(() => {
+      this.cache_refresh_promise = null;
+    });
+    return this.cache_refresh_promise;
+  }
+
+  async getByName(client, name) {
+    this._checkClient(client);
+    this._checkDefined('name', name);
+    let match = await this._getByName(name, client);
+    if (!match) {
+      console.log(
+        `[DEBUG] Cache miss for ${this.element_type_name} "${name}", refreshing cache`
+      );
+      match = await this._getByName(name, client, -1);
+    }
+    return match;
+  }
+
+  async getById(client, id) {
+    this._checkClient(client);
+    this._checkDefined('id', id);
+    if (id === 0) {
+      return this._getById(id, client);
+    }
+    let match = await this._getById(id, client);
+    if (!match && id > Math.max(...this.elements_by_id.keys())) {
+      console.log(
+        `[DEBUG] Cache miss for ${this.element_type_name} #${id}, refreshing cache`
+      );
+      match = await this._getById(id, client, -1);
+    }
+    return match;
+  }
+
+  async visitAllContainedElements(client, visitor) {
+    this._checkClient(client);
+    await this._check_cache_is_not_outdated(client);
+    let result_set = new Set();
+    for (let element of this.element_list.values()) {
+      visitor(element, result_set);
+    }
+    return result_set;
+  }
+
+  async getCount(client) {
+    this._checkClient(client);
+    await this._check_cache_is_not_outdated(client);
+    return this.element_list.length;
+  }
+}
+
+module.exports = {
+  CachedNameIdEnum,
+};
diff --git a/tests/test-cached-name-id-enum.js b/tests/test-cached-name-id-enum.js
new file mode 100644
index 00000000..0d5d2c6d
--- /dev/null
+++ b/tests/test-cached-name-id-enum.js
@@ -0,0 +1,261 @@
+const assert = require('assert');
+const { CachedNameIdEnum } = require('../services/CachedNameIdEnum');
+
+function createEnum(
+  aiService = { generateAnswer: async () => JSON.stringify({ match: null }) },
+  cacheTtl = () => 0
+) {
+  return new CachedNameIdEnum(
+    'correspondent',
+    '/correspondents/',
+    'Use the closest match.',
+    cacheTtl,
+    aiService
+  );
+}
+
+function createClient(pages) {
+  const calls = [];
+  let pageIndex = 0;
+  const client = {
+    defaults: { baseURL: 'https://paperless.example/api' },
+    get: async (url, options) => {
+      calls.push({ url, options });
+      const results = pages[pageIndex++] || [];
+      const next =
+        pageIndex < pages.length
+          ? `https://paperless.example/api/page-${pageIndex}`
+          : null;
+      return { data: { results, next } };
+    },
+  };
+  client.getCalls = () => calls;
+  return client;
+}
+
+async function run() {
+  await assert.rejects(
+    () => createEnum().getCount(),
+    /No client provided\./,
+    'getCount should require a client'
+  );
+
+  const enumWithPages = createEnum();
+  const pagedClient = createClient([
+    [{ id: '1', name: 'Acme' }],
+    [{ id: '2', name: 'Tax Office' }],
+  ]);
+  const refresh = enumWithPages.flush_cache(pagedClient);
+  assert.strictEqual(typeof refresh.then, 'function');
+  await refresh;
+  assert.deepStrictEqual(enumWithPages.element_list, [
+    { id: '1', name: 'Acme' },
+    { id: '2', name: 'Tax Office' },
+  ]);
+  assert.deepStrictEqual(
+    pagedClient.getCalls().map(({ url, options }) => [url, options]),
+    [
+      ['/correspondents/', { page_size: 100 }],
+      ['/page-1', { page_size: 100 }],
+    ]
+  );
+
+  const exactClient = createClient([
+    [
+      { id: '1', name: 'Acme' },
+      { id: '2', name: 'Tax Office' },
+    ],
+  ]);
+  const exactEnum = createEnum();
+  assert.deepStrictEqual(await exactEnum.getByName(exactClient, 'ACME'), {
+    id: '1',
+    name: 'Acme',
+  });
+  assert.deepStrictEqual(await exactEnum.getById(exactClient, '2'), {
+    id: '2',
+    name: 'Tax Office',
+  });
+  assert.strictEqual(await exactEnum.getByName(exactClient, 'Unknown'), null);
+  assert.strictEqual(await exactEnum.getById(exactClient, 'missing'), null);
+
+  const similarityEnum = createEnum({
+    generateAnswer: async () =>
+      JSON.stringify({
+        match: { id: '42', name: 'Acme Corporation' },
+      }),
+  });
+  const similarityClient = createClient([
+    [{ id: '42', name: 'Acme Corporation' }],
+  ]);
+  assert.deepStrictEqual(
+    await similarityEnum.getByName(similarityClient, 'The Acme business'),
+    { id: '42', name: 'Acme Corporation' },
+    'semantic matches must unwrap the match property from the AI response'
+  );
+
+  const missEnum = createEnum();
+  const missClient = createClient([
+    [{ id: '1', name: 'Old' }],
+    [{ id: '2', name: 'New' }],
+  ]);
+  assert.deepStrictEqual(await missEnum.getByName(missClient, 'New'), {
+    id: '2',
+    name: 'New',
+  });
+  assert.strictEqual(missClient.getCalls().length, 2);
+
+  const visitorEnum = createEnum();
+  const visitorClient = createClient([
+    [
+      { id: '1', name: 'Acme' },
+      { id: '2', name: 'Tax Office' },
+    ],
+  ]);
+  const visited = [];
+  const resultSet = await visitorEnum.visitAllContainedElements(
+    visitorClient,
+    (element, set) => {
+      visited.push(element.name);
+      set.add(element.id);
+    }
+  );
+  assert.deepStrictEqual(visited, ['Acme', 'Tax Office']);
+  assert.deepStrictEqual([...resultSet], ['1', '2']);
+  assert.strictEqual(await visitorEnum.getCount(visitorClient), 2);
+
+  await assert.rejects(
+    () => createEnum().getByName(exactClient, undefined),
+    /Mandatory argument name is undefined or null\./
+  );
+  await assert.rejects(
+    () => createEnum().getByName(exactClient, null),
+    /Mandatory argument name is undefined or null\./
+  );
+  await assert.rejects(
+    () => createEnum().getById(exactClient, undefined),
+    /Mandatory argument id is undefined or null\./
+  );
+  await assert.rejects(
+    () => createEnum().getById(exactClient, null),
+    /Mandatory argument id is undefined or null\./
+  );
+
+  const zeroIdEnum = createEnum();
+  const zeroIdClient = createClient([[{ id: 0, name: 'Invalid' }]]);
+  assert.strictEqual(await zeroIdEnum.getById(zeroIdClient, 0), null);
+  assert.strictEqual(zeroIdClient.getCalls().length, 0);
+
+  const semanticCalls = [];
+  const semanticObjectEnum = createEnum({
+    generateAnswer: async (...args) => {
+      semanticCalls.push(args);
+      return { match: { id: 42, name: 'Acme Corporation' } };
+    },
+  });
+  const semanticObjectClient = createClient([
+    [{ id: 42, name: 'Acme Corporation' }],
+  ]);
+  assert.deepStrictEqual(
+    await semanticObjectEnum.getByName(
+      semanticObjectClient,
+      'Completely different'
+    ),
+    { id: 42, name: 'Acme Corporation' }
+  );
+  assert.strictEqual(semanticCalls.length, 1);
+  assert.match(semanticCalls[0][0], /Acme Corporation \(ID: 42\)/);
+  assert.strictEqual(semanticCalls[0][2], 300);
+  assert.deepStrictEqual(semanticCalls[0][3], {
+    type: 'object',
+    properties: {
+      match: {
+        type: ['object', 'null'],
+        properties: { id: { type: 'number' }, name: { type: 'string' } },
+        required: ['id', 'name'],
+      },
+    },
+    required: ['match'],
+  });
+
+  const failingAiEnum = createEnum({
+    generateAnswer: async () => {
+      throw new Error('AI unavailable');
+    },
+  });
+  const failingAiClient = createClient([[{ id: 1, name: 'Acme' }]]);
+  assert.strictEqual(
+    await failingAiEnum.getByName(failingAiClient, 'No such correspondent'),
+    null,
+    'AI errors should be treated as a similarity miss'
+  );
+
+  const refreshEnum = createEnum();
+  let resolveRefresh;
+  const refreshClient = {
+    defaults: { baseURL: 'https://paperless.example/api' },
+    getCalls: () => refreshCalls,
+    get: async (...args) => {
+      refreshCalls.push(args);
+      await new Promise((resolve) => {
+        resolveRefresh = resolve;
+      });
+      return { data: { results: [{ id: 7, name: 'Shared' }], next: null } };
+    },
+  };
+  const refreshCalls = [];
+  const firstRefresh = refreshEnum.flush_cache(refreshClient);
+  const secondRefresh = refreshEnum.flush_cache(refreshClient);
+  assert.strictEqual(firstRefresh, secondRefresh);
+  resolveRefresh();
+  await firstRefresh;
+  assert.strictEqual(refreshCalls.length, 1);
+  assert.strictEqual(await refreshEnum.getCount(refreshClient), 1);
+
+  const expiringEnum = createEnum(undefined, () => 1);
+  let expiringFetchNo = 0;
+  const expiringCalls = [];
+  const expiringClient = {
+    defaults: { baseURL: 'https://paperless.example/api' },
+    get: async (url, options) => {
+      expiringCalls.push({ url, options });
+      expiringFetchNo += 1;
+      return {
+        data: {
+          results: [
+            expiringFetchNo === 1
+              ? { id: 1, name: 'First' }
+              : { id: 2, name: 'Second' },
+          ],
+          next: null,
+        },
+      };
+    },
+    getCalls: () => expiringCalls,
+  };
+  await expiringEnum.getCount(expiringClient);
+  assert.strictEqual(expiringClient.getCalls().length, 1);
+  expiringEnum.cache_last_fetch_time = Date.now() - 10;
+  assert.strictEqual(await expiringEnum.getCount(expiringClient), 1);
+  assert.strictEqual(expiringClient.getCalls().length, 2);
+
+  assert.throws(() => createEnum().flush_cache(), /No client provided\./);
+  await assert.rejects(
+    () => createEnum().getByName(null, 'Acme'),
+    /No client provided\./
+  );
+  await assert.rejects(
+    () => createEnum().getById(null, '1'),
+    /No client provided\./
+  );
+  await assert.rejects(
+    () => createEnum().visitAllContainedElements(null, () => {}),
+    /No client provided\./
+  );
+
+  console.log('✅ test-cached-name-id-enum passed');
+}
+
+run().catch((error) => {
+  console.error('❌ test-cached-name-id-enum failed:', error);
+  process.exit(1);
+});

From 8de987047b9ca3fc120a59ae4efacbbd6e5d3546 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 9 Aug 2026 22:56:40 +0200
Subject: [PATCH 08/53] Defer Paperless service loading in Azure service

---
 services/azureService.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/services/azureService.js b/services/azureService.js
index 0b3044bb..119af2fc 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -9,7 +9,6 @@ const {
 const axios = require('axios');
 const AzureOpenAI = require('openai').AzureOpenAI;
 const config = require('../config/config');
-const paperlessService = require('./paperlessService');
 const fs = require('fs').promises;
 const path = require('path');
 const {
@@ -68,7 +67,7 @@ class AzureOpenAIService {
         console.log('[DEBUG] Thumbnail already cached');
       } catch {
         console.log('Thumbnail not cached, fetching from Paperless');
-
+        const paperlessService = require('./paperlessService');
         const thumbnailData = await paperlessService.getThumbnailImage(id);
 
         if (!thumbnailData) {

From fa2ccde2e881d9784776b80119815fbc5a6d8866 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Wed, 26 Aug 2026 12:47:38 +0200
Subject: [PATCH 09/53] fix merge conflict resolution bugs

---
 scripts/run-tests.js         |   1 +
 services/ollamaService.js    |  31 +--
 services/paperlessService.js | 383 ++++++++++++++++++++++++++++++++++-
 services/serviceUtils.js     |  28 +++
 4 files changed, 419 insertions(+), 24 deletions(-)

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index 1596dac9..828f4be3 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -31,6 +31,7 @@ function formatStatus(status) {
 }
 
 const TESTS = {
+  'cached-name-id-enum': 'test-cached-name-id-enum.js',
   'dashboard-stats-cache': 'test-dashboard-stats-cache.js',
   'document-type-restriction': 'test-document-type-restriction.js',
   'effective-document-count-cache': 'test-effective-document-count-cache.js',
diff --git a/services/ollamaService.js b/services/ollamaService.js
index c40eb658..2c439ba1 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -1,5 +1,6 @@
 const {
-    writePromptToFile
+    writePromptToFile,
+    toNameList
 } = require('./serviceUtils');
 const axios = require('axios');
 const config = require('../config/config');
@@ -302,27 +303,13 @@ class OllamaService {
         // Get system prompt based on configuration
         if (config.useExistingData === 'yes' && config.restrictToExistingTags === 'no' && config.restrictToExistingCorrespondents === 'no') {
             // Format existing tags
-            const existingTagsList = existingTags.join(', ');
-
-            // Format existing correspondents - handle both array of objects and array of strings
-            const existingCorrespondentList = correspondentList
-                .filter(Boolean)  // Remove any null/undefined entries
-                .map(correspondent => {
-                    if (typeof correspondent === 'string') return correspondent;
-                    return correspondent?.name || '';
-                })
-                .filter(name => name.length > 0)  // Remove empty strings
-                .join(', ');
-
-            // Format existing document types - handle both array of objects and array of strings
-            const existingDocumentTypesList = existingDocumentTypes
-                .filter(Boolean)  // Remove any null/undefined entries
-                .map(docType => {
-                    if (typeof docType === 'string') return docType;
-                    return docType?.name || '';
-                })
-                .filter(name => name.length > 0)  // Remove empty strings
-                .join(', ');
+            const existingTagsList = toNameList(existingTags).join(', ');
+
+            // Format existing correspondents
+            const existingCorrespondentList = toNameList(correspondentList).join(', ');
+
+            // Format existing document types
+            const existingDocumentTypesList = toNameList(existingDocumentTypes).join(', ');
 
             systemPrompt = `
             Pre-existing tags: ${existingTagsList}\n\n
diff --git a/services/paperlessService.js b/services/paperlessService.js
index f0bd8c8d..b366a1d0 100644
--- a/services/paperlessService.js
+++ b/services/paperlessService.js
@@ -945,10 +945,13 @@ class PaperlessService {
   async listCorrespondentsNames() {
     this.initialize();
     try {
-      return await this.correspondents.visitAllContainedElements(
+      // visitAllContainedElements collects into a Set; every caller of this
+      // method expects an array (map/length/spread), so flatten it here.
+      const names = await this.correspondents.visitAllContainedElements(
         this.client,
         (correspondent, result) => result.add(correspondent.name)
       );
+      return Array.from(names);
     } catch (error) {
       console.error('[ERROR] fetching correspondent names:', error.message);
       return [];
@@ -958,10 +961,11 @@ class PaperlessService {
   async listDocumentTypesNames() {
     this.initialize();
     try {
-      return await this.documentTypes.visitAllContainedElements(
+      const names = await this.documentTypes.visitAllContainedElements(
         this.client,
         (doc_type, result) => result.add(doc_type.name)
       );
+      return Array.from(names);
     } catch (error) {
       console.error('[ERROR] fetching document type names:', error.message);
       return [];
@@ -1406,6 +1410,250 @@ class PaperlessService {
     return documents;
   }
 
+  /**
+   * Normalize a list of entity IDs (plain numbers or `{ id }` objects) into
+   * unique positive integers.
+   *
+   * @param   ids  Raw IDs as handed over by callers or API responses.
+   * @returns      Unique positive integer IDs.
+   */
+  _normalizeEntityIds(ids) {
+    if (!Array.isArray(ids)) {
+      return [];
+    }
+
+    const normalized = new Set();
+    for (const rawId of ids) {
+      const id = typeof rawId === 'object' ? Number(rawId?.id) : Number(rawId);
+      if (Number.isInteger(id) && id > 0) {
+        normalized.add(id);
+      }
+    }
+
+    return [...normalized];
+  }
+
+  /**
+   * Resolve tag IDs to names using the shared tag cache.
+   *
+   * The cache is keyed by tag name, so the reverse lookup is built on the fly.
+   * This costs at most one paginated refresh per CACHE_LIFETIME instead of one
+   * detail request per tag.
+   *
+   * @param   tagIds  Tag IDs to resolve.
+   * @returns         A plain object mapping tag ID to tag name.
+   */
+  async getTagNamesByIds(tagIds = []) {
+    this.initialize();
+    const uniqueTagIds = this._normalizeEntityIds(tagIds);
+    if (uniqueTagIds.length === 0 || !this.client) {
+      return {};
+    }
+
+    try {
+      await this.ensureTagCache();
+    } catch (error) {
+      console.error('[ERROR] resolving tag names from cache:', error.message);
+      return {};
+    }
+
+    const namesById = new Map();
+    for (const tag of this.tagCache.values()) {
+      const id = Number(tag?.id);
+      if (Number.isInteger(id) && typeof tag?.name === 'string') {
+        namesById.set(id, tag.name);
+      }
+    }
+
+    const resolved = {};
+    for (const tagId of uniqueTagIds) {
+      const name = namesById.get(tagId);
+      if (name) {
+        resolved[tagId] = name;
+      }
+    }
+
+    return resolved;
+  }
+
+  /**
+   * Resolve correspondent IDs to names, batching every cache miss into a single
+   * filtered list request instead of one detail request per correspondent.
+   *
+   * @param   correspondentIds  Correspondent IDs to resolve.
+   * @returns                   A plain object mapping correspondent ID to name.
+   */
+  async getCorrespondentNamesByIds(correspondentIds = []) {
+    this.initialize();
+    const uniqueIds = this._normalizeEntityIds(correspondentIds);
+    if (uniqueIds.length === 0 || !this.client) {
+      return {};
+    }
+
+    // The cache holds a whole generation of names; drop it once it is stale
+    // rather than tracking a timestamp per entry.
+    if (Date.now() - this.lastCorrespondentRefresh > this.CACHE_LIFETIME) {
+      this.correspondentNameCache.clear();
+      this.lastCorrespondentRefresh = Date.now();
+    }
+
+    const missingIds = uniqueIds.filter(
+      (id) => !this.correspondentNameCache.has(id)
+    );
+
+    if (missingIds.length > 0) {
+      await this._loadCorrespondentNames(missingIds);
+    }
+
+    const resolved = {};
+    for (const id of uniqueIds) {
+      const name = this.correspondentNameCache.get(id);
+      if (name) {
+        resolved[id] = name;
+      }
+    }
+
+    return resolved;
+  }
+
+  /**
+   * Populate the correspondent name cache for the given IDs.
+   * Prefers the `id__in` filter and falls back to a full listing when the
+   * Paperless-ngx instance does not support it.
+   */
+  async _loadCorrespondentNames(missingIds) {
+    let loaded;
+
+    if (this._supportsCorrespondentIdIn === false) {
+      loaded = await this._primeCorrespondentNameCache();
+    } else {
+      loaded = true;
+      for (let index = 0; index < missingIds.length; index += 100) {
+        const chunk = missingIds.slice(index, index + 100);
+        const applied = await this._fetchCorrespondentNameChunk(chunk);
+        if (!applied) {
+          // `id__in` is unsupported or failed - load everything once instead.
+          loaded = await this._primeCorrespondentNameCache();
+          break;
+        }
+      }
+
+      if (loaded && this._supportsCorrespondentIdIn !== false) {
+        this._supportsCorrespondentIdIn = true;
+      }
+    }
+
+    // Only remember misses when the lookup itself worked. A transient network
+    // error must not poison the cache for a whole TTL generation.
+    if (!loaded) {
+      return;
+    }
+
+    // Cache the misses too, otherwise an ID Paperless-ngx cannot resolve is
+    // looked up again on every single search - which in the fallback path
+    // means listing all correspondents over and over.
+    for (const id of missingIds) {
+      if (!this.correspondentNameCache.has(id)) {
+        this.correspondentNameCache.set(id, null);
+      }
+    }
+  }
+
+  /**
+   * Fetch one batch of correspondent names via `id__in`.
+   * @returns true when the filter was honoured, false when the caller must fall back.
+   */
+  async _fetchCorrespondentNameChunk(chunk) {
+    const requestedIds = new Set(chunk);
+
+    try {
+      const response = await this.client.get('/correspondents/', {
+        params: {
+          id__in: chunk.join(','),
+          page_size: chunk.length,
+          fields: 'id,name',
+        },
+      });
+
+      const results = response?.data?.results;
+      if (!Array.isArray(results)) {
+        return false;
+      }
+
+      // Unknown query params are silently ignored by Paperless-ngx, which would
+      // return an arbitrary page of correspondents instead of the requested
+      // ones. Any unexpected ID means the filter was not applied.
+      const filterHonoured = results.every((correspondent) =>
+        requestedIds.has(Number(correspondent?.id))
+      );
+      if (!filterHonoured) {
+        console.warn(
+          '[DEBUG] Paperless-ngx ignored the correspondent id__in filter, falling back to a full listing'
+        );
+        this._supportsCorrespondentIdIn = false;
+        return false;
+      }
+
+      for (const correspondent of results) {
+        const id = Number(correspondent?.id);
+        if (Number.isInteger(id) && typeof correspondent?.name === 'string') {
+          this.correspondentNameCache.set(id, correspondent.name);
+        }
+      }
+
+      return true;
+    } catch (error) {
+      console.error(
+        '[ERROR] fetching correspondent names by id:',
+        error.message
+      );
+      return false;
+    }
+  }
+
+  /**
+   * Load every correspondent name into the cache in one paginated sweep.
+   * Used as the fallback path so a single search never fans out into one
+   * request per correspondent.
+   *
+   * @returns true when the listing produced entries. `listCorrespondentsNames()`
+   *          swallows its own errors and returns an empty array, so an empty
+   *          result is treated as inconclusive rather than as a success.
+   */
+  async _primeCorrespondentNameCache() {
+    try {
+      let page = 1;
+      let hasNextPage = true;
+      let cached = 0;
+
+      // Paginate directly through the axios client rather than through the
+      // correspondent enum: its element cache would answer from memory and a
+      // first lookup of an unresolvable ID would never reach Paperless-ngx.
+      while (hasNextPage) {
+        const response = await this.client.get('/correspondents/', {
+          params: { fields: 'id,name', count: true, page, page_size: 100 },
+        });
+        const results = response?.data?.results;
+        if (!Array.isArray(results)) {
+          break;
+        }
+        for (const correspondent of results) {
+          const id = Number(correspondent?.id);
+          if (Number.isInteger(id) && typeof correspondent?.name === 'string') {
+            this.correspondentNameCache.set(id, correspondent.name);
+            cached += 1;
+          }
+        }
+        hasNextPage = Boolean(response?.data?.next);
+        page += 1;
+      }
+      return cached > 0;
+    } catch (error) {
+      console.error('[ERROR] priming correspondent name cache:', error.message);
+      return false;
+    }
+  }
+
   /**
    * Get the Name of a Correspondent by its ID.
    *
@@ -1516,6 +1764,137 @@ class PaperlessService {
     return this.getAllDocuments();
   }
 
+  async searchDocuments(query, limit = 100, mode = 'all') {
+    this.initialize();
+    if (!this.client) {
+      console.error('[DEBUG] Client not initialized');
+      return [];
+    }
+
+    const safeLimit = Number.isInteger(Number(limit))
+      ? Math.max(1, Math.min(Number(limit), 200))
+      : 100;
+    const normalizedQuery = String(query || '').trim();
+    const documentFields = 'id,title,tags,correspondent,created';
+
+    try {
+      // Explicit ID mode: exact lookup via GET /documents/{id}/
+      if (mode === 'id') {
+        const id = Number.parseInt(normalizedQuery, 10);
+        if (!Number.isInteger(id) || id < 1 || String(id) !== normalizedQuery) {
+          return [];
+        }
+
+        try {
+          // Request only the selector fields so the document content, which can
+          // be megabytes on scanned files, is not transferred.
+          const response = await this.client.get(`/documents/${id}/`, {
+            params: { fields: documentFields },
+          });
+          const doc = response?.data;
+          if (!doc || doc.id == null) {
+            return [];
+          }
+          return [
+            {
+              id: doc.id,
+              title: doc.title,
+              tags: doc.tags,
+              correspondent: doc.correspondent,
+              created: doc.created || doc.created_date || doc.added || null,
+            },
+          ];
+        } catch (error) {
+          if (error?.response?.status !== 404) {
+            console.error(
+              `[ERROR] searching document by id ${id}:`,
+              error.message
+            );
+          }
+          return [];
+        }
+      }
+
+      const params = {
+        fields: documentFields,
+        page: 1,
+        page_size: safeLimit,
+        ordering: '-created',
+      };
+
+      if (mode === 'title') {
+        params.title__icontains = normalizedQuery;
+      } else if (mode === 'tags') {
+        const tagIds = await this._findTagIdsByPartialName(normalizedQuery);
+        if (!tagIds.length) return [];
+        params.tags__id__in = tagIds.join(',');
+      } else if (mode === 'correspondent') {
+        const corrIds =
+          await this._findCorrespondentIdsByPartialName(normalizedQuery);
+        if (!corrIds.length) return [];
+        params.correspondent__id__in = corrIds.join(',');
+      } else {
+        params.query = normalizedQuery;
+      }
+
+      let response;
+      try {
+        response = await this.client.get('/documents/', { params });
+      } catch (error) {
+        // Full-text search needs the Paperless-ngx search index and rejects
+        // malformed query syntax. Keep the selector usable by retrying with a
+        // plain title filter instead of returning nothing.
+        if (mode !== 'all') {
+          throw error;
+        }
+
+        console.warn(
+          `[DEBUG] Full-text document search failed (${error.message}), retrying with a title filter`
+        );
+
+        const fallbackParams = { ...params };
+        delete fallbackParams.query;
+        fallbackParams.title__icontains = normalizedQuery;
+        response = await this.client.get('/documents/', {
+          params: fallbackParams,
+        });
+      }
+
+      if (!Array.isArray(response?.data?.results)) {
+        return [];
+      }
+
+      return response.data.results;
+    } catch (error) {
+      console.error('[ERROR] searching documents:', error.message);
+      return [];
+    }
+  }
+
+  async _findTagIdsByPartialName(query) {
+    try {
+      const response = await this.client.get('/tags/', {
+        params: { name__icontains: query, page_size: 50 },
+      });
+      return (response.data?.results || []).map((t) => t.id);
+    } catch (error) {
+      console.error('[ERROR] searching tags by name:', error.message);
+      return [];
+    }
+  }
+
+  async _findCorrespondentIdsByPartialName(query) {
+    try {
+      const response = await this.client.get('/correspondents/', {
+        params: { name__icontains: query, page_size: 50 },
+      });
+      return (response.data?.results || []).map((c) => c.id);
+    } catch (error) {
+      console.error('[ERROR] searching correspondents by name:', error.message);
+      return [];
+    }
+  }
+
   async getDocumentContent(documentId) {
     this.initialize();
     const response = await this.client.get(`/documents/${documentId}/`);
diff --git a/services/serviceUtils.js b/services/serviceUtils.js
index f5637442..f9ad2dea 100644
--- a/services/serviceUtils.js
+++ b/services/serviceUtils.js
@@ -601,6 +601,33 @@ function validateUrlAgainstBase(urlToValidate, expectedBaseUrl) {
   };
 }
 
+/**
+ * Safely extract the relative path from a pagination `next` URL, validating it
+ * against the base URL of the axios client that received it. This prevents SSRF
+ * by ensuring a server-controlled link cannot point anywhere but the configured
+ * Paperless-ngx origin.
+ *
+ * @param {string} nextUrl - The next URL from an API response
+ * @param {Object} client - Axios client whose defaults.baseURL is the expected origin
+ * @returns {string|null} The relative path if valid, null otherwise
+ */
+function safeExtractRelativePath(nextUrl, client) {
+  if (!nextUrl || !client?.defaults?.baseURL) {
+    return null;
+  }
+
+  const validation = validateUrlAgainstBase(
+    nextUrl,
+    client.defaults.baseURL
+  );
+  if (!validation.valid) {
+    console.error(`[ERROR] URL validation failed: ${validation.error}`);
+    return null;
+  }
+
+  return validation.relativePath;
+}
+
 /**
  * Builds an axios `beforeRedirect` handler that keeps redirects on the host the
  * request was validated against.
@@ -932,6 +959,7 @@ module.exports = {
   validateUrl,
   validateApiUrl,
   validateUrlAgainstBase,
+  safeExtractRelativePath,
   createRedirectGuard,
   validateCustomFieldValue,
   assertCompletionNotTruncated,

From ac8ef470816443d82f1a4738befb247bfd3d3f77 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Wed, 26 Aug 2026 13:28:36 +0200
Subject: [PATCH 10/53] use CachedNameIdEnum also for tag cache

---
 services/CachedNameIdEnum.js      |  38 ++++-
 services/paperlessService.js      | 233 +++++++++++-------------------
 tests/test-cached-name-id-enum.js |   6 +-
 tests/test-tag-cache-page-size.js |  31 ++--
 4 files changed, 147 insertions(+), 161 deletions(-)

diff --git a/services/CachedNameIdEnum.js b/services/CachedNameIdEnum.js
index 1db0f5cc..e9e12fe9 100644
--- a/services/CachedNameIdEnum.js
+++ b/services/CachedNameIdEnum.js
@@ -6,13 +6,17 @@ class CachedNameIdEnum {
     url_prefix,
     similarity_prompt,
     cache_ttl_access,
-    ai_service
+    ai_service,
+    page_size = 100
   ) {
     this.ai_service = ai_service;
     this.element_type_name = element_type_name;
     this.url_prefix = url_prefix;
     this.similarity_prompt = similarity_prompt;
     this.cache_ttl_access = cache_ttl_access;
+    // Asked for on every page request. DRF clamps to its own max_page_size
+    // rather than rejecting, so a server that allows less simply pages more.
+    this.page_size = page_size;
     this.cache_last_fetch_time = 0;
     this.cache_refresh_promise = null;
     this.elements_by_id = new Map();
@@ -51,7 +55,11 @@ class CachedNameIdEnum {
     let nextUrl = this.url_prefix;
     let existing_elements = [];
     while (nextUrl) {
-      const response = await client.get(nextUrl, { page_size: 100 });
+      // page_size must ride in `params` — a bare property on the axios config
+      // is silently dropped so that the server answers with its default page size.
+      const response = await client.get(nextUrl, {
+        params: { page_size: this.page_size },
+      });
       existing_elements = existing_elements.concat(response.data.results);
       // Safely extract relative path from next URL to prevent SSRF
       if (response.data.next) {
@@ -335,6 +343,21 @@ class CachedNameIdEnum {
     return match;
   }
 
+  /**
+   * Insert a single element (e.g., one just created on the server) into the
+   * cache without a full refresh.
+   */
+  add(element) {
+    if (!element || element.id == null || !element.name) {
+      return;
+    }
+    this.elements_by_id.set(element.id, element);
+    this.elements_by_name.set(element.name.toLowerCase(), element);
+    if (!this.element_list.some((e) => e.id === element.id)) {
+      this.element_list.push(element);
+    }
+  }
+
   async visitAllContainedElements(client, visitor) {
     this._checkClient(client);
     await this._check_cache_is_not_outdated(client);
@@ -350,6 +373,17 @@ class CachedNameIdEnum {
     await this._check_cache_is_not_outdated(client);
     return this.element_list.length;
   }
+
+  /**
+   * All cached elements as `{id, name}` objects, refreshing when stale.
+   */
+  async getAll(client) {
+    this._checkClient(client);
+    await this._check_cache_is_not_outdated(client);
+    return this.element_list.map((element) =>
+      this._result_element_from_element(element)
+    );
+  }
 }
 
 module.exports = {
diff --git a/services/paperlessService.js b/services/paperlessService.js
index b366a1d0..ef648243 100644
--- a/services/paperlessService.js
+++ b/services/paperlessService.js
@@ -7,8 +7,8 @@ const {
   stripTrailingSlashes,
   createRedirectGuard,
 } = require('./serviceUtils');
-const {CachedNameIdEnum} = require('./CachedNameIdEnum');
-const {safeExtractRelativePath} = require('./serviceUtils');
+const { CachedNameIdEnum } = require('./CachedNameIdEnum');
+const { safeExtractRelativePath } = require('./serviceUtils');
 
 /** Timeout for the connectivity probe so a hanging host cannot stall a scan. */
 const CONNECTION_PROBE_TIMEOUT_MS = 10000;
@@ -57,13 +57,11 @@ function requestTimeoutMs() {
 class PaperlessService {
   constructor() {
     this.client = null;
-    this.tagCache = new Map();
     this.customFieldCache = new Map();
     this.correspondentNameCache = new Map();
-    this.lastTagRefresh = 0;
+    this.lastCorrespondentRefresh = 0;
     this.lastCorrespondentRefresh = 0;
     this._supportsCorrespondentIdIn = null;
-    this._refreshPromise = null;
     this._effectiveCountCache = null;
     this._effectiveCountCacheTtlMs = 60 * 1000;
     this._supportsTagsIdNone = null;
@@ -78,6 +76,7 @@ class PaperlessService {
     this._cacheTTL = null;
     this._correspondents = null;
     this._documentTypes = null;
+    this._tags = null;
   }
 
   get correspondents() {
@@ -116,6 +115,29 @@ class PaperlessService {
     return this._documentTypes;
   }
 
+  // Tags share the enum machinery with correspondents and document types.
+  // The larger page size matters here: tags are the one all-or-nothing read
+  // everything else queues behind, and a large library at the DRF default of
+  // 25 per page once stalled startup for a minute (54 sequential requests).
+  get tags() {
+    if (this._tags === null) {
+      const aiServiceFactory = require('./aiServiceFactory');
+      const aiService = aiServiceFactory.getService();
+      this._tags = new CachedNameIdEnum(
+        'tag',
+        '/tags/',
+        'Ignore case and consider typos, misspellings, singular vs plural forms, ' +
+          'abbreviations and compound words. Only match a tag that clearly means the same thing',
+        () => {
+          return this.CACHE_LIFETIME;
+        },
+        aiService,
+        TAG_PAGE_SIZE
+      );
+    }
+    return this._tags;
+  }
+
   get CACHE_LIFETIME() {
     if (this._cacheTTL === null) {
       // Re-require config at runtime to pick up dynamic configuration values without shadowing the top-level import
@@ -353,92 +375,36 @@ class PaperlessService {
     }
   }
 
-  //todo
-  // Aktualisiert den Tag-Cache, wenn er älter als CACHE_LIFETIME ist
+  /**
+   * Make sure the tag cache is populated and not expired. Kept as a named
+   * seam because several call sites and tests speak in these terms; the
+   * freshness logic itself lives in CachedNameIdEnum.
+   */
   async ensureTagCache() {
-    const now = Date.now();
-    const cacheAge = now - this.lastTagRefresh;
-    if (this.tagCache.size === 0 || cacheAge > this.CACHE_LIFETIME) {
-      if (this._refreshPromise) {
-        return this._refreshPromise;
-      }
-      const ttlSeconds = Math.floor(this.CACHE_LIFETIME / 1000);
-      // A cache that was never filled has no age and no expiry. Dating it from
-      // the epoch reported "age: 1786738904s ... expired at: 1970-01-01" on
-      // every cold start, which reads as a clock problem rather than a first
-      // run.
-      console.log(
-        this.lastTagRefresh === 0
-          ? `[DEBUG] Tag cache empty, building it (TTL: ${ttlSeconds}s)`
-          : `[DEBUG] Tag cache expired (age: ${Math.floor(cacheAge / 1000)}s, TTL: ${ttlSeconds}s, expired at: ${new Date(
-              this.lastTagRefresh + this.CACHE_LIFETIME
-            ).toISOString()})`
-      );
-      // No race condition: synchronous code is never preempted in Node.js's
-      // event loop, so no other call can reach here between the check above
-      // and the assignment below.
-      this._refreshPromise = this.refreshTagCache().finally(() => {
-        this._refreshPromise = null;
-      });
-      return this._refreshPromise;
+    if (!this.client) {
+      return;
     }
+    await this.tags.getCount(this.client);
   }
 
   /**
-   * Manually clear the tag cache.
+   * Manually clear (and rebuild on next access) the tag cache.
    * Useful for forcing a refresh after external tag modifications.
    */
   clearTagCache() {
-    this.initialize()
+    this.initialize();
     console.log('[DEBUG] Manually clearing tag cache...');
-    this.tagCache.clear();
-    this.correspondents.flush_cache(this.client);
-    this.lastTagRefresh = 0;
+    if (this.client) {
+      this.tags.flush_cache(this.client);
+      this.correspondents.flush_cache(this.client);
+    }
     console.log('[DEBUG] Tag cache cleared.');
   }
 
-  // Lädt alle existierenden Tags Todo
   async refreshTagCache() {
-    try {
-      console.log('[DEBUG] Refreshing tag cache...');
-      this.tagCache.clear();
-      // The page size only has to be asked for once: Paperless-ngx builds its
-      // `next` link from the request URL, so every following page carries it.
-      let nextUrl = `/tags/?page_size=${TAG_PAGE_SIZE}`;
-      while (nextUrl) {
-        const response = await this.client.get(nextUrl);
-
-        // Validate response structure
-        if (!response?.data?.results) {
-          console.error(
-            '[ERROR] Invalid response structure from API:',
-            response?.data
-          );
-          break;
-        }
-
-        response.data.results.forEach((tag) => {
-          this.tagCache.set(tag.name.toLowerCase(), tag);
-        });
-
-        // Safely extract relative path from next URL to prevent SSRF
-        if (response.data.next) {
-          nextUrl = safeExtractRelativePath(response.data.next, this.client);
-          if (nextUrl) {
-            console.log('[DEBUG] Next page URL:', nextUrl);
-          }
-        } else {
-          nextUrl = null;
-        }
-      }
-      this.lastTagRefresh = Date.now();
-      console.log(
-        `[DEBUG] Tag cache refreshed. Found ${this.tagCache.size} tags.`
-      );
-    } catch (error) {
-      console.error('[ERROR] refreshing tag cache:', error.message);
-      throw error;
-    }
+    this.initialize();
+    console.log('[DEBUG] Refreshing tag cache...');
+    await this.tags.flush_cache(this.client);
   }
 
   async initializeWithCredentials(apiUrl, apiToken) {
@@ -513,7 +479,7 @@ class PaperlessService {
           if (existingFieldType !== normalizedFieldType) {
             const mismatchError = new Error(
               `Custom field "${fieldName}" has type "${existingFieldType}", ` +
-              `but configuration requests "${normalizedFieldType}".`
+                `but configuration requests "${normalizedFieldType}".`
             );
             mismatchError.code = 'CUSTOM_FIELD_TYPE_MISMATCH';
             throw mismatchError;
@@ -616,45 +582,17 @@ class PaperlessService {
     }
   }
 
-  // todo
   async findExistingTag(tagName) {
-    const normalizedName = tagName.toLowerCase();
-
-    // 1. Zuerst im Cache suchen
-    const cachedTag = this.tagCache.get(normalizedName);
-    if (cachedTag) {
-      console.log(
-        `[DEBUG] Found tag "${tagName}" in cache with ID ${cachedTag.id}`
-      );
-      return cachedTag;
-    }
-
-    // 2. Direkte API-Suche
-    try {
-      const response = await this.client.get('/tags/', {
-        params: {
-          name__iexact: normalizedName, // Case-insensitive exact match
-        },
-      });
-
-      if (response.data.results.length > 0) {
-        const foundTag = response.data.results[0];
-        console.log(
-          `[DEBUG] Found existing tag "${tagName}" via API with ID ${foundTag.id}`
-        );
-        this.tagCache.set(normalizedName, foundTag);
-        return foundTag;
-      }
-    } catch (error) {
-      console.warn(`[ERROR] searching for tag "${tagName}":`, error.message);
+    if (!tagName || typeof tagName !== 'string') {
+      return null;
     }
-
-    return null;
+    // Delegates to the enum: exact match first, then substring/term/AI
+    // similarity so a near-duplicate name resolves to the existing tag
+    // instead of creating one.
+    return this.tags.getByName(this.client, tagName);
   }
 
   async createTagSafely(tagName) {
-    const normalizedName = tagName.toLowerCase();
-
     try {
       // Versuche zuerst, den Tag zu erstellen
       const response = await this.client.post('/tags/', { name: tagName });
@@ -662,9 +600,9 @@ class PaperlessService {
       console.log(
         `[DEBUG] Successfully created tag "${tagName}" with ID ${newTag.id}`
       );
-      this.tagCache.set(normalizedName, newTag);
-      // Invalidate cache after creating new tag to ensure consistency
-      this.lastTagRefresh = 0;
+      // Insert into the cache without a full refresh; the enum's maps stay
+      // consistent for the lookups that follow in the same run.
+      this.tags.add(newTag);
       return newTag;
     } catch (error) {
       if (error.response?.status === 400) {
@@ -805,7 +743,6 @@ class PaperlessService {
     }
   }
 
-  // todo
   async getTags() {
     this.initialize();
     if (!this.client) {
@@ -813,9 +750,8 @@ class PaperlessService {
       return [];
     }
 
-    // Use cached tags if available and not expired
-    await this.ensureTagCache();
-    return Array.from(this.tagCache.values());
+    // Cached, freshness-checked read; elements are reduced to {id, name}.
+    return this.tags.getAll(this.client);
   }
 
   //todo
@@ -891,10 +827,9 @@ class PaperlessService {
   async getTagCount({ strict = false } = {}) {
     this.initialize();
     try {
-      const response = await this.client.get('/tags/', {
-        params: { count: true },
-      });
-      return response.data.count;
+      // Served from the shared tag cache like the correspondent count; the
+      // strict flag keeps a dead backend from caching an all-zero library.
+      return await this.tags.getCount(this.client);
     } catch (error) {
       console.error('[ERROR] fetching tag count:', error.message);
       if (strict) throw error;
@@ -1457,19 +1392,18 @@ class PaperlessService {
       return {};
     }
 
-    const namesById = new Map();
-    for (const tag of this.tagCache.values()) {
-      const id = Number(tag?.id);
-      if (Number.isInteger(id) && typeof tag?.name === 'string') {
-        namesById.set(id, tag.name);
-      }
-    }
-
     const resolved = {};
     for (const tagId of uniqueTagIds) {
-      const name = namesById.get(tagId);
-      if (name) {
-        resolved[tagId] = name;
+      try {
+        const tag = await this.tags.getById(this.client, tagId);
+        if (tag && typeof tag.name === 'string') {
+          resolved[tagId] = tag.name;
+        }
+      } catch (error) {
+        console.error(
+          `[ERROR] resolving tag name for #${tagId}:`,
+          error.message
+        );
       }
     }
 
@@ -1662,13 +1596,16 @@ class PaperlessService {
    */
   async getCorrespondentNameById(correspondentId) {
     this.initialize();
-    return this.correspondents.getById(this.client, correspondentId)
+    return this.correspondents.getById(this.client, correspondentId);
   }
 
   async getDocumentTypeNameById(documentTypeId) {
     this.initialize();
     try {
-      let documentType = this.documentTypes.getById(this.client, documentTypeId);
+      let documentType = this.documentTypes.getById(
+        this.client,
+        documentTypeId
+      );
       if (documentType) {
         return documentType.name;
       }
@@ -1694,8 +1631,8 @@ class PaperlessService {
      */
     this.initialize();
     try {
-      const response = await this.client.get(`/tags/${tagId}/`);
-      return response.data.name;
+      const tag = await this.tags.getById(this.client, tagId);
+      return tag ? tag.name : null;
     } catch (error) {
       console.error(
         `[ERROR] fetching tag name for ID ${tagId}:`,
@@ -1938,7 +1875,6 @@ class PaperlessService {
     }
   }
 
-
   // todo
   async getOrCreateCorrespondent(name, options = {}) {
     this.initialize();
@@ -1956,16 +1892,21 @@ class PaperlessService {
 
     try {
       // Search for the correspondent
-      const existingCorrespondent = await this.searchForExistingCorrespondent(name);
+      const existingCorrespondent =
+        await this.searchForExistingCorrespondent(name);
 
       if (existingCorrespondent) {
-        console.log(`[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}`);
+        console.log(
+          `[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}`
+        );
         return existingCorrespondent;
       }
 
       // If we're restricting to existing correspondents and none was found, return null
       if (restrictToExistingCorrespondents) {
-        console.log(`[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null`);
+        console.log(
+          `[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null`
+        );
         return null;
       }
 
@@ -2143,12 +2084,11 @@ class PaperlessService {
     }
   }
 
-  // todo
   async getTagTextFromId(tagId) {
     this.initialize();
     try {
-        const response = await this.client.get(`/tags/${tagId}/`);
-        return response.data.name;
+      const tag = await this.tags.getById(this.client, tagId);
+      return tag ? tag.name : null;
     } catch (error) {
       console.error(
         `[ERROR] fetching tag text for ID ${tagId}:`,
@@ -2406,5 +2346,4 @@ class PaperlessService {
   }
 }
 
-
 module.exports = new PaperlessService();
diff --git a/tests/test-cached-name-id-enum.js b/tests/test-cached-name-id-enum.js
index 0d5d2c6d..cb86c91c 100644
--- a/tests/test-cached-name-id-enum.js
+++ b/tests/test-cached-name-id-enum.js
@@ -55,8 +55,10 @@ async function run() {
   assert.deepStrictEqual(
     pagedClient.getCalls().map(({ url, options }) => [url, options]),
     [
-      ['/correspondents/', { page_size: 100 }],
-      ['/page-1', { page_size: 100 }],
+      // page_size must ride in axios `params` — a flat property on the config
+      // is dropped by real axios and the server answers with its default.
+      ['/correspondents/', { params: { page_size: 100 } }],
+      ['/page-1', { params: { page_size: 100 } }],
     ]
   );
 
diff --git a/tests/test-tag-cache-page-size.js b/tests/test-tag-cache-page-size.js
index 0debb33c..311d2ba3 100644
--- a/tests/test-tag-cache-page-size.js
+++ b/tests/test-tag-cache-page-size.js
@@ -48,9 +48,16 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) {
   return {
     requests,
     defaults: { baseURL: BASE_URL },
-    get: async (url) => {
-      requests.push(url);
-      const parsed = new URL(url, BASE_URL);
+    get: async (url, options = {}) => {
+      // Merge axios-style params into the query string, as axios itself would.
+      const merged = new URL(url, BASE_URL);
+      if (options.params) {
+        for (const [key, value] of Object.entries(options.params)) {
+          merged.searchParams.set(key, String(value));
+        }
+      }
+      requests.push(merged.pathname + merged.search);
+      const parsed = merged;
       const requested = Number(parsed.searchParams.get('page_size')) || 25;
       const pageSize = Math.min(requested, maxPageSize);
       const page = Number(parsed.searchParams.get('page')) || 1;
@@ -82,12 +89,13 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) {
     await test('The tag cache is not rebuilt 25 tags at a time', async () => {
       const server = createTagServer({ tagCount: 1331 });
       paperlessService.client = server;
-      paperlessService.tagCache.clear();
+      paperlessService.tags._clear();
+      paperlessService.tags.cache_last_fetch_time = 0;
 
       await paperlessService.refreshTagCache();
 
       assert.strictEqual(
-        paperlessService.tagCache.size,
+        paperlessService.tags.element_list.length,
         1331,
         'Every tag has to reach the cache'
       );
@@ -107,7 +115,8 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) {
     await test('The page size carries into every following page', async () => {
       const server = createTagServer({ tagCount: 1331 });
       paperlessService.client = server;
-      paperlessService.tagCache.clear();
+      paperlessService.tags._clear();
+      paperlessService.tags.cache_last_fetch_time = 0;
 
       await paperlessService.refreshTagCache();
 
@@ -125,12 +134,13 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) {
       // its own maximum and says so only through the shorter result list.
       const server = createTagServer({ tagCount: 1331, maxPageSize: 100 });
       paperlessService.client = server;
-      paperlessService.tagCache.clear();
+      paperlessService.tags._clear();
+      paperlessService.tags.cache_last_fetch_time = 0;
 
       await paperlessService.refreshTagCache();
 
       assert.strictEqual(
-        paperlessService.tagCache.size,
+        paperlessService.tags.element_list.length,
         1331,
         'A clamped page size must not lose tags'
       );
@@ -138,8 +148,9 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) {
     });
   } finally {
     paperlessService.client = originalClient;
-    paperlessService.tagCache.clear();
-    paperlessService.lastTagRefresh = 0;
+    paperlessService.tags._clear();
+    paperlessService.tags.cache_last_fetch_time = 0;
+    paperlessService.tags.cache_last_fetch_time = 0;
   }
 
   console.log(`\n${passed} passed, ${failed} failed`);

From 92e9857dd845592d07bd1bac70a11689d82b4ec3 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Wed, 26 Aug 2026 13:45:57 +0200
Subject: [PATCH 11/53] cleanup linter warnings and errors

---
 services/paperlessService.js | 221 +++++++++++++++++++----------------
 1 file changed, 118 insertions(+), 103 deletions(-)

diff --git a/services/paperlessService.js b/services/paperlessService.js
index ef648243..79bcdf9a 100644
--- a/services/paperlessService.js
+++ b/services/paperlessService.js
@@ -8,7 +8,6 @@ const {
   createRedirectGuard,
 } = require('./serviceUtils');
 const { CachedNameIdEnum } = require('./CachedNameIdEnum');
-const { safeExtractRelativePath } = require('./serviceUtils');
 
 /** Timeout for the connectivity probe so a hanging host cannot stall a scan. */
 const CONNECTION_PROBE_TIMEOUT_MS = 10000;
@@ -54,6 +53,32 @@ function requestTimeoutMs() {
   return seconds * 1000;
 }
 
+/**
+ * Wraps a failed correspondent/document-type creation once race-condition
+ * recovery could not resolve it. Throwing a fresh, contextualised error
+ * instead of rethrowing the caught one keeps the outer handler free of
+ * local-rethrow patterns and carries the entity context to the caller.
+ */
+class EntityCreationError extends Error {
+  /**
+   * @param {string} entityType - 'correspondent' or 'document type'
+   * @param {string} name - The name that was being created
+   * @param {Error} cause - The original API error
+   */
+  constructor(entityType, name, cause) {
+    super(`Failed to process ${entityType} "${name}": ${cause.message}`, {
+      cause,
+    });
+    this.name = 'EntityCreationError';
+    this.entityType = entityType;
+  }
+}
+
+/** True when Paperless-ngx rejected a create as a duplicate. */
+const isUniqueConstraintError = (error) =>
+  error?.response?.status === 400 &&
+  String(error?.response?.data?.error || '').includes('unique constraint');
+
 class PaperlessService {
   constructor() {
     this.client = null;
@@ -843,8 +868,15 @@ class PaperlessService {
    */
   async getCorrespondentCount({ strict = false } = {}) {
     this.initialize();
-    // todo pass strict to throw error if no match found
-    return await this.correspondents.getCount(this.client);
+    try {
+      return await this.correspondents.getCount(this.client);
+    } catch (error) {
+      console.error('[ERROR] fetching correspondent count:', error.message);
+      // Mirrors getTagCount(): strict=yes is for statistics, where reporting 0
+      // on a failed backend would cache an all-zero library.
+      if (strict) throw error;
+      return 0;
+    }
   }
 
   async getDocumentCount() {
@@ -1222,6 +1254,9 @@ class PaperlessService {
           if (error?.response?.status === 400) {
             this._supportsTagsIdNone = false;
           } else {
+            // Deliberate propagation: only the unsupported-filter case is
+            // handled here, everything else belongs to the outer handler.
+            // noinspection ThrowCaughtLocallyJS
             throw error;
           }
         }
@@ -1782,6 +1817,9 @@ class PaperlessService {
         // malformed query syntax. Keep the selector usable by retrying with a
         // plain title filter instead of returning nothing.
         if (mode !== 'all') {
+          // Deliberate propagation: non-'all' modes have no fallback and the
+          // outer handler reports the failure to the caller.
+          // noinspection ThrowCaughtLocallyJS
           throw error;
         }
 
@@ -1890,64 +1928,52 @@ class PaperlessService {
       `[DEBUG] Processing correspondent with restrictToExistingCorrespondents=${restrictToExistingCorrespondents}`
     );
 
-    try {
-      // Search for the correspondent
-      const existingCorrespondent =
-        await this.searchForExistingCorrespondent(name);
+    const existingCorrespondent =
+      await this.searchForExistingCorrespondent(name);
 
-      if (existingCorrespondent) {
-        console.log(
-          `[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}`
-        );
-        return existingCorrespondent;
-      }
+    if (existingCorrespondent) {
+      console.log(
+        `[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}`
+      );
+      return existingCorrespondent;
+    }
 
-      // If we're restricting to existing correspondents and none was found, return null
-      if (restrictToExistingCorrespondents) {
-        console.log(
-          `[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null`
-        );
-        return null;
-      }
+    // If we're restricting to existing correspondents and none was found, return null
+    if (restrictToExistingCorrespondents) {
+      console.log(
+        `[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null`
+      );
+      return null;
+    }
 
-      // Create new correspondent only if restrictions are not enabled
-      try {
-        const createResponse = await this.client.post('/correspondents/', {
-          name: name,
+    // Create new correspondent only if restrictions are not enabled
+    try {
+      const createResponse = await this.client.post('/correspondents/', {
+        name: name,
+      });
+      console.log(
+        `[DEBUG] Created new correspondent "${name}" with ID ${createResponse.data.id}`
+      );
+      return createResponse.data;
+    } catch (createError) {
+      if (isUniqueConstraintError(createError)) {
+        // Race condition check - another process might have created it
+        const retryResponse = await this.client.get('/correspondents/', {
+          params: { name: name },
         });
-        console.log(
-          `[DEBUG] Created new correspondent "${name}" with ID ${createResponse.data.id}`
+
+        const justCreatedCorrespondent = retryResponse.data.results.find(
+          (c) => c.name.toLowerCase() === name.toLowerCase()
         );
-        return createResponse.data;
-      } catch (createError) {
-        if (
-          createError.response?.status === 400 &&
-          createError.response?.data?.error?.includes('unique constraint')
-        ) {
-          // Race condition check - another process might have created it
-          const retryResponse = await this.client.get('/correspondents/', {
-            params: { name: name },
-          });
 
-          const justCreatedCorrespondent = retryResponse.data.results.find(
-            (c) => c.name.toLowerCase() === name.toLowerCase()
+        if (justCreatedCorrespondent) {
+          console.log(
+            `[DEBUG] Retrieved correspondent "${name}" after constraint error with ID ${justCreatedCorrespondent.id}`
           );
-
-          if (justCreatedCorrespondent) {
-            console.log(
-              `[DEBUG] Retrieved correspondent "${name}" after constraint error with ID ${justCreatedCorrespondent.id}`
-            );
-            return justCreatedCorrespondent;
-          }
+          return justCreatedCorrespondent;
         }
-        throw createError;
       }
-    } catch (error) {
-      console.error(
-        `[ERROR] Failed to process correspondent "${name}":`,
-        error.message
-      );
-      throw error;
+      throw new EntityCreationError('correspondent', name, createError);
     }
   }
 
@@ -1977,66 +2003,55 @@ class PaperlessService {
       `[DEBUG] Processing document type with restrictToExistingDocumentTypes=${restrictToExistingDocumentTypes}`
     );
 
-    try {
-      // Suche nach existierendem document_type
-      const existingDocType = await this.searchForExistingDocumentType(name);
-      console.log('[DEBUG] Response Document Type Search: ', existingDocType);
+    // Suche nach existierendem document_type
+    const existingDocType = await this.searchForExistingDocumentType(name);
+    console.log('[DEBUG] Response Document Type Search: ', existingDocType);
 
-      if (existingDocType) {
-        console.log(
-          `[DEBUG] Found existing document type "${name}" with ID ${existingDocType.id}`
-        );
-        return existingDocType;
-      }
+    if (existingDocType) {
+      console.log(
+        `[DEBUG] Found existing document type "${name}" with ID ${existingDocType.id}`
+      );
+      return existingDocType;
+    }
 
-      if (restrictToExistingDocumentTypes) {
-        console.log(
-          `[DEBUG] Document type "${name}" does not exist and restrictions are enabled, returning null`
-        );
-        return null;
-      }
+    if (restrictToExistingDocumentTypes) {
+      console.log(
+        `[DEBUG] Document type "${name}" does not exist and restrictions are enabled, returning null`
+      );
+      return null;
+    }
 
-      // Erstelle neuen document_type
-      try {
-        const createResponse = await this.client.post('/document_types/', {
-          name: name,
-          matching_algorithm: 1, // 1 = ANY
-          match: '', // Optional: Kann später angepasst werden
-          is_insensitive: true,
+    // Erstelle neuen document_type
+    try {
+      const createResponse = await this.client.post('/document_types/', {
+        name: name,
+        matching_algorithm: 1, // 1 = ANY
+        match: '', // Optional: Kann später angepasst werden
+        is_insensitive: true,
+      });
+      console.log(
+        `[DEBUG] Created new document type "${name}" with ID ${createResponse.data.id}`
+      );
+      return createResponse.data;
+    } catch (createError) {
+      if (isUniqueConstraintError(createError)) {
+        // Race condition check
+        const retryResponse = await this.client.get('/document_types/', {
+          params: { name: name },
         });
-        console.log(
-          `[DEBUG] Created new document type "${name}" with ID ${createResponse.data.id}`
+
+        const justCreatedDocType = retryResponse.data.results.find(
+          (dt) => dt.name.toLowerCase() === name.toLowerCase()
         );
-        return createResponse.data;
-      } catch (createError) {
-        if (
-          createError.response?.status === 400 &&
-          createError.response?.data?.error?.includes('unique constraint')
-        ) {
-          // Race condition check
-          const retryResponse = await this.client.get('/document_types/', {
-            params: { name: name },
-          });
 
-          const justCreatedDocType = retryResponse.data.results.find(
-            (dt) => dt.name.toLowerCase() === name.toLowerCase()
+        if (justCreatedDocType) {
+          console.log(
+            `[DEBUG] Retrieved document type "${name}" after constraint error with ID ${justCreatedDocType.id}`
           );
-
-          if (justCreatedDocType) {
-            console.log(
-              `[DEBUG] Retrieved document type "${name}" after constraint error with ID ${justCreatedDocType.id}`
-            );
-            return justCreatedDocType;
-          }
+          return justCreatedDocType;
         }
-        throw createError;
       }
-    } catch (error) {
-      console.error(
-        `[ERROR] Failed to process document type "${name}":`,
-        error.message
-      );
-      throw error;
+      throw new EntityCreationError('document type', name, createError);
     }
   }
 

From 11381fedd88e9aa247220f14f04bd7d50cac0ea4 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Wed, 26 Aug 2026 15:06:35 +0200
Subject: [PATCH 12/53] fix(ollama): fit document content into the configured
 context window
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Long documents failed with ai_response_truncated ("AI answer cut off").
Root cause: the OpenAI-compatible paths truncate input to the token
budget, but the Ollama path sent the full extracted text. num_ctx was
computed as min(prompt + responseTokens, TOKEN_LIMIT), while Ollama
silently clamps num_ctx to the model's real context length — a long
document filled the clamped window and the answer was cut off mid-JSON,
on exactly the long documents.

Changes
- New _fitContentToContext(): shortens the document content to what is
  left of TOKEN_LIMIT once the prompt skeleton and the reserved answer
  (RESPONSE_TOKENS) have taken their share. Uses the same conservative
  2-chars-per-token estimate as _calculatePromptTokenCount(), so the
  arithmetic agrees by construction. A prompt alone larger than the
  window throws a clear error instead of truncating silently.
- analyzeDocument builds the prompt skeleton first, fits the content,
  then assembles the final prompt; the custom-prompt branch is fitted
  the same way.
- analyzePlayground applies the same fit before assembling fullPrompt.
- analyzeDocument now reports an honest `truncated` flag instead of a
  hardcoded false.

Testing
tests/test-ollama-response-limit.js passes unchanged; functional check
of _fitContentToContext against a mocked small window confirms budget
arithmetic and the oversize-prompt error. Full suite: same result as
before the change (only pre-existing Windows-checkout artifacts fail).

Impact
Long documents are analysed instead of failing. Operators should set
TOKEN_LIMIT to their model's real context size so the silent clamp no
longer applies. Content beyond the budget is truncated — the analysis
runs on the head of the document, as the OpenAI path always did.

Upstream Status
Not applicable — upstream caps Ollama at a hardcoded num_predict and
has no context fitting at all.
---
 services/ollamaService.js | 1530 +++++++++++++++++++++----------------
 1 file changed, 868 insertions(+), 662 deletions(-)

diff --git a/services/ollamaService.js b/services/ollamaService.js
index 2c439ba1..5d17c6f4 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -1,437 +1,621 @@
 const {
-    writePromptToFile,
-    toNameList
+  writePromptToFile,
+  toNameList,
+  truncateToTokenLimit,
 } = require('./serviceUtils');
 const axios = require('axios');
 const config = require('../config/config');
 const fs = require('fs').promises;
 const paperlessService = require('./paperlessService');
 const os = require('os');
-const { THUMBNAIL_CACHE_DIR, getThumbnailCachePath } = require('./thumbnailCachePaths');
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('./thumbnailCachePaths');
 const RestrictionPromptService = require('./restrictionPromptService');
 
 /**
  * Service for document analysis using Ollama
  */
 class OllamaService {
-    /**
-     * Initialize the Ollama service
-     */
-    constructor() {
-        this.apiUrl = config.ollama.apiUrl;
-        this.model = config.ollama.model;
-        this.client = axios.create({
-            timeout: 1800000 // 30 minutes timeout
-        });
-
-        // JSON schema for document analysis output
-        this.documentAnalysisSchema = {
-            type: "object",
-            properties: {
-                title: { type: "string" },
-                correspondent: { type: "string" },
-                tags: {
-                    type: "array",
-                    items: { type: "string" }
-                },
-                document_type: { type: "string" },
-                document_date: { type: "string" },
-                language: { type: "string" },
-                custom_fields: {
-                    type: "object",
-                    additionalProperties: true
-                }
-            },
-            required: ["title", "correspondent", "tags", "document_type", "document_date", "language"]
-        };
-
-        // Schema for playground analysis (simpler version)
-        this.playgroundSchema = {
-            type: "object",
-            properties: {
-                title: { type: "string" },
-                correspondent: { type: "string" },
-                tags: {
-                    type: "array",
-                    items: { type: "string" }
-                },
-                document_type: { type: "string" },
-                document_date: { type: "string" },
-                language: { type: "string" }
-            },
-            required: ["title", "correspondent", "tags", "document_type", "document_date", "language"]
-        };
-    }
-
-    /**
-     * Analyze a document and extract metadata
-     * @param {string} content - Document content
-     * @param {Array} existingTags - List of existing tags
-     * @param {Array} existingCorrespondentList - List of existing correspondents
-     * @param {string} id - Document ID
-     * @param {string} customPrompt - Custom prompt (optional)
-     * @returns {Object} Analysis results
-     */
-    async analyzeDocument(content, existingTags = [], existingCorrespondentList = [], existingDocumentTypesList = [], id, customPrompt = null, options = {}) {
-        try {
-            // Truncate content if needed
-            content = this._truncateContent(content);
-
-            // Cache thumbnail
-            await this._handleThumbnailCaching(id);
-
-            // Get external API data if available and validate it
-            let externalApiData = options.externalApiData || null;
-            let validatedExternalApiData = null;
-
-            if (externalApiData) {
-                try {
-                    validatedExternalApiData = await this._validateAndTruncateExternalApiData(externalApiData);
-                    console.log('[DEBUG] External API data validated and included');
-                } catch (error) {
-                    console.warn('[WARNING] External API data validation failed:', error.message);
-                    validatedExternalApiData = null;
-                }
-            }
-
-            // Build prompt
-            let prompt;
-            if (!customPrompt) {
-                prompt = this._buildPrompt(content, existingTags, existingCorrespondentList, existingDocumentTypesList, options);
-            } else {
-                // Parse CUSTOM_FIELDS for custom prompt
-                let customFieldsObj;
-                try {
-                    customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-                } catch (error) {
-                    console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); console.debug(error);
-                    customFieldsObj = { custom_fields: [] };
-                }
-
-                const customFieldsTemplate = {};
-                customFieldsObj.custom_fields.forEach((field, index) => {
-                    let valueHint;
-                    if (field.data_type === 'date') {
-                        valueHint = "Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis";
-                    } else if (field.data_type === 'boolean') {
-                        valueHint = "Fill in 'true' or 'false' based on your analysis";
-                    } else {
-                        valueHint = "Fill in the value based on your analysis";
-                    }
-                    customFieldsTemplate[index] = {
-                        field_name: field.value,
-                        value: valueHint
-                    };
-                });
-
-                const customFieldsStr = '"custom_fields": ' + JSON.stringify(customFieldsTemplate, null, 2)
-                    .split('\n')
-                    .map(line => '    ' + line)
-                    .join('\n');
-
-                prompt = customPrompt + '\n\n' + config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr) + "\n\n" + JSON.stringify(content);
-                console.log('[DEBUG] Ollama Service started with custom prompt');
-            }
-
-            // Generate custom fields for the prompt
-            const customFieldsStr = this._generateCustomFieldsTemplate();
-
-            // Generate system prompt
-            const systemPrompt = this._generateSystemPrompt(customFieldsStr);
-
-            // Calculate context window size
-            const promptTokenCount = this._calculatePromptTokenCount(prompt);
-            const numCtx = this._calculateNumCtx(promptTokenCount, Number(config.responseTokens));
-
-            console.log(`[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`);
-            console.log(`[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`);
-
-            // Call Ollama API
-            const response = await this._callOllamaAPI(prompt, systemPrompt, numCtx, this.documentAnalysisSchema);
-
-            // Process response
-            const parsedResponse = this._processOllamaResponse(response);
-            const metrics = this._extractOllamaMetrics(response);
-
-            // Check for missing data
-            if (parsedResponse.tags.length === 0 && parsedResponse.correspondent === null) {
-                console.warn('No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.');
-            }
-
-            // Log the prompt and response
-            await this._logPromptAndResponse(prompt, parsedResponse);
-
-            // Return results in consistent format
-            return {
-                document: parsedResponse,
-                metrics,
-                truncated: false
-            };
-        } catch (error) {
-            console.error(`Error analyzing document with Ollama: ${error.message}`);
-            console.debug(error);
-            return {
-              document: { tags: [], correspondent: null },
-              metrics: null,
-              error: error.message,
-              // Undefined for everything that is not one of ours; the scan loop
-              // falls back to its generic reason then.
-              errorCode: error.code,
-            };
-        }
-    }
-
-    /**
-     * Analyze a document in playground mode
-     * @param {string} content - Document content
-     * @param {string} prompt - User-provided prompt
-     * @returns {Object} Analysis results
-     */
-    async analyzePlayground(content, prompt) {
-        try {
-            // Calculate context window size — include both prompt and content
-            const fullPrompt = prompt + "\n\n" + JSON.stringify(content);
-            const promptTokenCount = this._calculatePromptTokenCount(fullPrompt);
-            const numCtx = this._calculateNumCtx(promptTokenCount, Number(config.responseTokens));
-
-            // Generate playground system prompt (simpler than full analysis)
-            const systemPrompt = this._generatePlaygroundSystemPrompt();
-
-            // Call Ollama API
-            const response = await this._callOllamaAPI(
-                fullPrompt,
-                systemPrompt,
-                numCtx,
-                this.playgroundSchema
-            );
-
-            // Process response
-            const parsedResponse = this._processOllamaResponse(response);
-            const metrics = this._extractOllamaMetrics(response);
-
-            // Check for missing data
-            if (parsedResponse.tags.length === 0 && parsedResponse.correspondent === null) {
-                console.warn('No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.');
-            }
+  /**
+   * Initialize the Ollama service
+   */
+  constructor() {
+    this.apiUrl = config.ollama.apiUrl;
+    this.model = config.ollama.model;
+    this.client = axios.create({
+      timeout: 1800000, // 30 minutes timeout
+    });
+
+    // JSON schema for document analysis output
+    this.documentAnalysisSchema = {
+      type: 'object',
+      properties: {
+        title: { type: 'string' },
+        correspondent: { type: 'string' },
+        tags: {
+          type: 'array',
+          items: { type: 'string' },
+        },
+        document_type: { type: 'string' },
+        document_date: { type: 'string' },
+        language: { type: 'string' },
+        custom_fields: {
+          type: 'object',
+          additionalProperties: true,
+        },
+      },
+      required: [
+        'title',
+        'correspondent',
+        'tags',
+        'document_type',
+        'document_date',
+        'language',
+      ],
+    };
 
-            // Return results in consistent format
-            return {
-                document: parsedResponse,
-                metrics,
-                truncated: false
-            };
-        } catch (error) {
-            console.error(`Error analyzing document with Ollama: ${error.message}`);
-            console.debug(error);
-            return {
-              document: { tags: [], correspondent: null },
-              metrics: null,
-              error: error.message,
-              // Undefined for everything that is not one of ours; the scan loop
-              // falls back to its generic reason then.
-              errorCode: error.code,
-            };
-        }
-    }
+    // Schema for playground analysis (simpler version)
+    this.playgroundSchema = {
+      type: 'object',
+      properties: {
+        title: { type: 'string' },
+        correspondent: { type: 'string' },
+        tags: {
+          type: 'array',
+          items: { type: 'string' },
+        },
+        document_type: { type: 'string' },
+        document_date: { type: 'string' },
+        language: { type: 'string' },
+      },
+      required: [
+        'title',
+        'correspondent',
+        'tags',
+        'document_type',
+        'document_date',
+        'language',
+      ],
+    };
+  }
 
-    /**
-     * Truncate content to maximum length if specified
-     * @param {string} content - Content to truncate
-     * @returns {string} Truncated content
-     */
-    _truncateContent(content) {
+  /**
+   * Analyze a document and extract metadata
+   * @param {string} content - Document content
+   * @param {Array} existingTags - List of existing tags
+   * @param {Array} existingCorrespondentList - List of existing correspondents
+   * @param {string} id - Document ID
+   * @param {string} customPrompt - Custom prompt (optional)
+   * @returns {Object} Analysis results
+   */
+  async analyzeDocument(
+    content,
+    existingTags = [],
+    existingCorrespondentList = [],
+    existingDocumentTypesList = [],
+    id,
+    customPrompt = null,
+    options = {}
+  ) {
+    try {
+      // Truncate content if needed
+      content = this._truncateContent(content);
+
+      // Cache thumbnail
+      await this._handleThumbnailCaching(id);
+
+      // Get external API data if available and validate it
+      let externalApiData = options.externalApiData || null;
+      let validatedExternalApiData = null;
+
+      if (externalApiData) {
         try {
-            if (process.env.CONTENT_MAX_LENGTH) {
-                console.log('Truncating content to max length:', process.env.CONTENT_MAX_LENGTH);
-                return content.substring(0, process.env.CONTENT_MAX_LENGTH);
-            }
+          validatedExternalApiData =
+            await this._validateAndTruncateExternalApiData(externalApiData);
+          console.log('[DEBUG] External API data validated and included');
         } catch (error) {
-            console.error(`Error truncating content: ${error.message}`); console.debug(error);
+          console.warn(
+            '[WARNING] External API data validation failed:',
+            error.message
+          );
+          validatedExternalApiData = null;
         }
-        return content;
-    }
-
-    /**
-     * Build prompt from content and existing data
-     * @param {string} content - Document content
-     * @param {Array} existingTags - List of existing tags
-     * @param {Array} existingCorrespondent - List of existing correspondents
-     * @param {Array} existingDocumentTypes - List of existing document types
-     * @returns {string} Formatted prompt
-     */
-    _buildPrompt(content, existingTags = [], existingCorrespondent = [], existingDocumentTypes = [], options = {}) {
-        let systemPrompt;
-
-        // Validate that existingCorrespondent is an array and handle if it's not
-        const correspondentList = Array.isArray(existingCorrespondent)
-            ? existingCorrespondent
-            : [];
-
-        // Parse CUSTOM_FIELDS from environment variable
+      }
+
+      // Build prompt
+      let prompt;
+      let promptPrefix;
+      if (!customPrompt) {
+        promptPrefix = this._buildPrompt(
+          '',
+          existingTags,
+          existingCorrespondentList,
+          existingDocumentTypesList,
+          options
+        );
+      } else {
+        // Parse CUSTOM_FIELDS for custom prompt
         let customFieldsObj;
         try {
-            customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
+          customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
         } catch (error) {
-             console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); console.debug(error);
-            customFieldsObj = { custom_fields: [] };
+          console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
+          console.debug(error);
+          customFieldsObj = { custom_fields: [] };
         }
 
-        // Generate custom fields template for the prompt
         const customFieldsTemplate = {};
-
         customFieldsObj.custom_fields.forEach((field, index) => {
-            let valueHint;
-            if (field.data_type === 'date') {
-                valueHint = "Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis";
-            } else if (field.data_type === 'boolean') {
-                valueHint = "Fill in 'true' or 'false' based on your analysis";
-            } else {
-                valueHint = "Fill in the value based on your analysis";
-            }
-            customFieldsTemplate[index] = {
-                field_name: field.value,
-                value: valueHint
-            };
+          let valueHint;
+          if (field.data_type === 'date') {
+            valueHint =
+              'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
+          } else if (field.data_type === 'boolean') {
+            valueHint = "Fill in 'true' or 'false' based on your analysis";
+          } else {
+            valueHint = 'Fill in the value based on your analysis';
+          }
+          customFieldsTemplate[index] = {
+            field_name: field.value,
+            value: valueHint,
+          };
         });
 
-        // Convert template to string for replacement and wrap in custom_fields
-        const customFieldsStr = '"custom_fields": ' + JSON.stringify(customFieldsTemplate, null, 2)
+        const customFieldsStr =
+          '"custom_fields": ' +
+          JSON.stringify(customFieldsTemplate, null, 2)
             .split('\n')
-            .map(line => '    ' + line)  // Add proper indentation
+            .map((line) => '    ' + line)
             .join('\n');
 
-        // Get system prompt based on configuration
-        if (config.useExistingData === 'yes' && config.restrictToExistingTags === 'no' && config.restrictToExistingCorrespondents === 'no') {
-            // Format existing tags
-            const existingTagsList = toNameList(existingTags).join(', ');
+        promptPrefix =
+          customPrompt +
+          '\n\n' +
+          config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr) +
+          '\n\n';
+        console.log('[DEBUG] Ollama Service started with custom prompt');
+      }
+
+      // Fit the document into what is left of the context window
+      const originalContentLength = String(content).length;
+      content = this._fitContentToContext(content, promptPrefix.length);
+
+      if (!customPrompt) {
+        prompt = this._buildPrompt(
+          content,
+          existingTags,
+          existingCorrespondentList,
+          existingDocumentTypesList,
+          options
+        );
+      } else {
+        prompt = promptPrefix + JSON.stringify(content);
+      }
+      const contentWasTruncated =
+        String(content).length < originalContentLength;
+
+      // Generate custom fields for the prompt
+      const customFieldsStr = this._generateCustomFieldsTemplate();
+
+      // Generate system prompt
+      const systemPrompt = this._generateSystemPrompt(customFieldsStr);
+
+      // Calculate context window size
+      const promptTokenCount = this._calculatePromptTokenCount(prompt);
+      const numCtx = this._calculateNumCtx(
+        promptTokenCount,
+        Number(config.responseTokens)
+      );
+
+      console.log(
+        `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
+      );
+      console.log(
+        `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`
+      );
+
+      // Call Ollama API
+      const response = await this._callOllamaAPI(
+        prompt,
+        systemPrompt,
+        numCtx,
+        this.documentAnalysisSchema
+      );
+
+      // Process response
+      const parsedResponse = this._processOllamaResponse(response);
+      const metrics = this._extractOllamaMetrics(response);
+
+      // Check for missing data
+      if (
+        parsedResponse.tags.length === 0 &&
+        parsedResponse.correspondent === null
+      ) {
+        console.warn(
+          'No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.'
+        );
+      }
+
+      // Log the prompt and response
+      await this._logPromptAndResponse(prompt, parsedResponse);
+
+      // Return results in consistent format
+      return {
+        document: parsedResponse,
+        metrics,
+        truncated: contentWasTruncated,
+      };
+    } catch (error) {
+      console.error(`Error analyzing document with Ollama: ${error.message}`);
+      console.debug(error);
+      return {
+        document: { tags: [], correspondent: null },
+        metrics: null,
+        error: error.message,
+        // Undefined for everything that is not one of ours; the scan loop
+        // falls back to its generic reason then.
+        errorCode: error.code,
+      };
+    }
+  }
 
-            // Format existing correspondents
-            const existingCorrespondentList = toNameList(correspondentList).join(', ');
+  /**
+   * Analyze a document in playground mode
+   * @param {string} content - Document content
+   * @param {string} prompt - User-provided prompt
+   * @returns {Object} Analysis results
+   */
+  async analyzePlayground(content, prompt) {
+    try {
+      // Fit the content into the context window, then size num_ctx from
+      // what will actually be sent
+      const fittedContent = this._fitContentToContext(
+        content,
+        (prompt + '\n\n' + '""').length - 2
+      );
+      // Calculate context window size — include both prompt and content
+      const fullPrompt = prompt + '\n\n' + JSON.stringify(fittedContent);
+      const promptTokenCount = this._calculatePromptTokenCount(fullPrompt);
+      const numCtx = this._calculateNumCtx(
+        promptTokenCount,
+        Number(config.responseTokens)
+      );
+
+      // Generate playground system prompt (simpler than full analysis)
+      const systemPrompt = this._generatePlaygroundSystemPrompt();
+
+      // Call Ollama API
+      const response = await this._callOllamaAPI(
+        fullPrompt,
+        systemPrompt,
+        numCtx,
+        this.playgroundSchema
+      );
+
+      // Process response
+      const parsedResponse = this._processOllamaResponse(response);
+      const metrics = this._extractOllamaMetrics(response);
+
+      // Check for missing data
+      if (
+        parsedResponse.tags.length === 0 &&
+        parsedResponse.correspondent === null
+      ) {
+        console.warn(
+          'No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.'
+        );
+      }
+
+      // Return results in consistent format
+      return {
+        document: parsedResponse,
+        metrics,
+        truncated: false,
+      };
+    } catch (error) {
+      console.error(`Error analyzing document with Ollama: ${error.message}`);
+      console.debug(error);
+      return {
+        document: { tags: [], correspondent: null },
+        metrics: null,
+        error: error.message,
+        // Undefined for everything that is not one of ours; the scan loop
+        // falls back to its generic reason then.
+        errorCode: error.code,
+      };
+    }
+  }
+
+  /**
+   * Truncate content to maximum length if specified
+   * @param {string} content - Content to truncate
+   * @returns {string} Truncated content
+   */
+  _truncateContent(content) {
+    try {
+      if (process.env.CONTENT_MAX_LENGTH) {
+        console.log(
+          'Truncating content to max length:',
+          process.env.CONTENT_MAX_LENGTH
+        );
+        return content.substring(0, process.env.CONTENT_MAX_LENGTH);
+      }
+    } catch (error) {
+      console.error(`Error truncating content: ${error.message}`);
+      console.debug(error);
+    }
+    return content;
+  }
+
+  /**
+   * Fit the document content into what is left of the context window once
+   * the fixed prompt parts and the reserved answer have taken their share.
+   *
+   * Without this an oversized prompt silently changed the meaning of the
+   * request: Ollama clamps num_ctx to the model's real limit, the long
+   * document filled the window, and the answer was cut off mid-JSON —
+   * reported as ai_response_truncated on exactly the long documents. With
+   * the content fitted, prompt + reserved answer always fit the configured
+   * TOKEN_LIMIT, so setting that to the model's true context removes the
+   * silent clamp entirely.
+   *
+   * Uses the same conservative 2-chars-per-token estimate as
+   * _calculatePromptTokenCount(), so the numbers agree by construction.
+   *
+   * @param {string} content - Document content
+   * @param {number} prefixChars - Length of the fully built prompt without the content tail
+   * @returns {string} Content shortened to the remaining budget when needed
+   */
+  _fitContentToContext(content, prefixChars) {
+    const maxWindow = Number(config.tokenLimit);
+    const responseTokens = Number(config.responseTokens);
+    if (!Number.isFinite(maxWindow) || !Number.isFinite(responseTokens)) {
+      return content;
+    }
+
+    const text =
+      typeof content === 'string' ? content : JSON.stringify(content);
+    const overheadTokens = Math.ceil(prefixChars / 2);
+    const availableTokens = maxWindow - responseTokens - overheadTokens;
 
-            // Format existing document types
-            const existingDocumentTypesList = toNameList(existingDocumentTypes).join(', ');
+    if (availableTokens <= 0) {
+      throw new Error(
+        'Token limit exceeded: the prompt alone does not fit the configured TOKEN_LIMIT. Raise TOKEN_LIMIT or shorten the system prompt.'
+      );
+    }
+
+    const maxChars = availableTokens * 2;
+    if (text.length <= maxChars) {
+      return content;
+    }
+
+    console.log(
+      `[DEBUG] Document content (${text.length} chars) exceeds the context budget (${maxChars} chars for a ${maxWindow}-token window minus prompt and ${responseTokens} reserved answer tokens); truncating`
+    );
+    return text.substring(0, maxChars);
+  }
+
+  /**
+   * Build prompt from content and existing data
+   * @param {string} content - Document content
+   * @param {Array} existingTags - List of existing tags
+   * @param {Array} existingCorrespondent - List of existing correspondents
+   * @param {Array} existingDocumentTypes - List of existing document types
+   * @returns {string} Formatted prompt
+   */
+  _buildPrompt(
+    content,
+    existingTags = [],
+    existingCorrespondent = [],
+    existingDocumentTypes = [],
+    options = {}
+  ) {
+    let systemPrompt;
+
+    // Validate that existingCorrespondent is an array and handle if it's not
+    const correspondentList = Array.isArray(existingCorrespondent)
+      ? existingCorrespondent
+      : [];
+
+    // Parse CUSTOM_FIELDS from environment variable
+    let customFieldsObj;
+    try {
+      customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
+    } catch (error) {
+      console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
+      console.debug(error);
+      customFieldsObj = { custom_fields: [] };
+    }
 
-            systemPrompt = `
+    // Generate custom fields template for the prompt
+    const customFieldsTemplate = {};
+
+    customFieldsObj.custom_fields.forEach((field, index) => {
+      let valueHint;
+      if (field.data_type === 'date') {
+        valueHint =
+          'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
+      } else if (field.data_type === 'boolean') {
+        valueHint = "Fill in 'true' or 'false' based on your analysis";
+      } else {
+        valueHint = 'Fill in the value based on your analysis';
+      }
+      customFieldsTemplate[index] = {
+        field_name: field.value,
+        value: valueHint,
+      };
+    });
+
+    // Convert template to string for replacement and wrap in custom_fields
+    const customFieldsStr =
+      '"custom_fields": ' +
+      JSON.stringify(customFieldsTemplate, null, 2)
+        .split('\n')
+        .map((line) => '    ' + line) // Add proper indentation
+        .join('\n');
+
+    // Get system prompt based on configuration
+    if (
+      config.useExistingData === 'yes' &&
+      config.restrictToExistingTags === 'no' &&
+      config.restrictToExistingCorrespondents === 'no'
+    ) {
+      // Format existing tags
+      const existingTagsList = toNameList(existingTags).join(', ');
+
+      // Format existing correspondents
+      const existingCorrespondentList =
+        toNameList(correspondentList).join(', ');
+
+      // Format existing document types
+      const existingDocumentTypesList = toNameList(existingDocumentTypes).join(
+        ', '
+      );
+
+      systemPrompt =
+        `
             Pre-existing tags: ${existingTagsList}\n\n
             Pre-existing correspondents: ${existingCorrespondentList}\n\n
             Pre-existing document types: ${existingDocumentTypesList}\n\n
-            ` + process.env.SYSTEM_PROMPT + '\n\n' + config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
-        } else {
-            const mustHavePrompt = config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
-            systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt;
-        }
-
-        // Get validated external API data if available
-        let validatedExternalApiData = null;
-        if (options.externalApiData) {
-            try {
-                validatedExternalApiData = this._validateAndTruncateExternalApiData(options.externalApiData);
-                console.log('[DEBUG] External API data validated and included');
-            } catch (error) {
-                console.warn('[WARNING] External API data validation failed:', error.message);
-                validatedExternalApiData = null;
-            }
-        }
+            ` +
+        process.env.SYSTEM_PROMPT +
+        '\n\n' +
+        config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
+    } else {
+      const mustHavePrompt = config.mustHavePrompt.replace(
+        '%CUSTOMFIELDS%',
+        customFieldsStr
+      );
+      systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt;
+    }
 
-        // Process placeholder replacements in system prompt
-        systemPrompt = RestrictionPromptService.processRestrictionsInPrompt(
-            systemPrompt,
-            existingTags,
-            correspondentList,
-            existingDocumentTypes,
-            config
+    // Get validated external API data if available
+    let validatedExternalApiData = null;
+    if (options.externalApiData) {
+      try {
+        validatedExternalApiData = this._validateAndTruncateExternalApiData(
+          options.externalApiData
+        );
+        console.log('[DEBUG] External API data validated and included');
+      } catch (error) {
+        console.warn(
+          '[WARNING] External API data validation failed:',
+          error.message
         );
+        validatedExternalApiData = null;
+      }
+    }
 
-        // Include validated external API data if available
-        if (validatedExternalApiData) {
-            systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`;
-        }
+    // Process placeholder replacements in system prompt
+    systemPrompt = RestrictionPromptService.processRestrictionsInPrompt(
+      systemPrompt,
+      existingTags,
+      correspondentList,
+      existingDocumentTypes,
+      config
+    );
+
+    // Include validated external API data if available
+    if (validatedExternalApiData) {
+      systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`;
+    }
 
-        if (process.env.USE_PROMPT_TAGS === 'yes') {
-            systemPrompt = `
+    if (process.env.USE_PROMPT_TAGS === 'yes') {
+      systemPrompt =
+        `
             Take these tags and try to match one or more to the document content.\n\n
             ` + config.specialPromptPreDefinedTags;
-        }
+    }
 
-        return `${systemPrompt}
+    return `${systemPrompt}
         ${JSON.stringify(content)}
         `;
-    }
-
-    /**
-     * Validate and truncate external API data to prevent token overflow
-     * @param {any} apiData - The external API data to validate
-     * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500)
-     * @returns {string} - Validated and potentially truncated data string
-     */
-    async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
-        if (!apiData) {
-            return null;
-        }
-
-        const dataString = typeof apiData === 'object'
-            ? JSON.stringify(apiData, null, 2)
-            : String(apiData);
-
-        // Calculate tokens for the data (conservative 2 chars/token for non-English)
-        const dataTokens = Math.ceil(dataString.length / 2);
-
-        if (dataTokens > maxTokens) {
-            console.warn(`[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating`);
-            // Simple truncation based on character count
-            const maxChars = maxTokens * 2;
-            return dataString.substring(0, maxChars);
-        }
+  }
 
-        console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`);
-        return dataString;
+  /**
+   * Validate and truncate external API data to prevent token overflow
+   * @param {any} apiData - The external API data to validate
+   * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500)
+   * @returns {string} - Validated and potentially truncated data string
+   */
+  async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
+    if (!apiData) {
+      return null;
     }
 
-    /**
-     * Generate custom fields template for prompts
-     * @returns {string} Custom fields template as a string
-     */
-    _generateCustomFieldsTemplate() {
-        let customFieldsObj;
-        try {
-            customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-        } catch (error) {
-            console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); console.debug(error);
-            customFieldsObj = { custom_fields: [] };
-        }
-
-        // Generate custom fields template for the prompt
-        const customFieldsTemplate = {};
+    const dataString =
+      typeof apiData === 'object'
+        ? JSON.stringify(apiData, null, 2)
+        : String(apiData);
+
+    // Calculate tokens for the data (conservative 2 chars/token for non-English)
+    const dataTokens = Math.ceil(dataString.length / 2);
+
+    if (dataTokens > maxTokens) {
+      console.warn(
+        `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating`
+      );
+      // Simple truncation based on character count
+      const maxChars = maxTokens * 2;
+      return dataString.substring(0, maxChars);
+    }
 
-        customFieldsObj.custom_fields.forEach((field, index) => {
-            let valueHint;
-            if (field.data_type === 'date') {
-                valueHint = "Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis";
-            } else if (field.data_type === 'boolean') {
-                valueHint = "Fill in 'true' or 'false' based on your analysis";
-            } else {
-                valueHint = "Fill in the value based on your analysis";
-            }
-            customFieldsTemplate[index] = {
-                field_name: field.value,
-                value: valueHint
-            };
-        });
+    console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`);
+    return dataString;
+  }
 
-        // Convert template to string for replacement and wrap in custom_fields
-        return '"custom_fields": ' + JSON.stringify(customFieldsTemplate, null, 2)
-            .split('\n')
-            .map(line => '    ' + line)  // Add proper indentation
-            .join('\n');
+  /**
+   * Generate custom fields template for prompts
+   * @returns {string} Custom fields template as a string
+   */
+  _generateCustomFieldsTemplate() {
+    let customFieldsObj;
+    try {
+      customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
+    } catch (error) {
+      console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
+      console.debug(error);
+      customFieldsObj = { custom_fields: [] };
     }
 
-    /**
-     * Generate system prompt for document analysis
-     * @param {string} customFieldsStr - Custom fields as a string
-     * @returns {string} System prompt
-     */
-    _generateSystemPrompt(customFieldsStr) {
-        let systemPromptTemplate = `
+    // Generate custom fields template for the prompt
+    const customFieldsTemplate = {};
+
+    customFieldsObj.custom_fields.forEach((field, index) => {
+      let valueHint;
+      if (field.data_type === 'date') {
+        valueHint =
+          'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
+      } else if (field.data_type === 'boolean') {
+        valueHint = "Fill in 'true' or 'false' based on your analysis";
+      } else {
+        valueHint = 'Fill in the value based on your analysis';
+      }
+      customFieldsTemplate[index] = {
+        field_name: field.value,
+        value: valueHint,
+      };
+    });
+
+    // Convert template to string for replacement and wrap in custom_fields
+    return (
+      '"custom_fields": ' +
+      JSON.stringify(customFieldsTemplate, null, 2)
+        .split('\n')
+        .map((line) => '    ' + line) // Add proper indentation
+        .join('\n')
+    );
+  }
+
+  /**
+   * Generate system prompt for document analysis
+   * @param {string} customFieldsStr - Custom fields as a string
+   * @returns {string} System prompt
+   */
+  _generateSystemPrompt(customFieldsStr) {
+    let systemPromptTemplate = `
             You are a document analyzer. Your task is to analyze documents and extract relevant information. You do not ask back questions. 
             YOU MUSTNOT: Ask for additional information or clarification, or ask questions about the document, or ask for additional context.
             YOU MUSTNOT: Return a response without the desired JSON format.
@@ -450,15 +634,15 @@ class OllamaService {
             ALWAYS USE THE INFORMATION TO FILL OUT THE JSON OBJECT. DO NOT ASK BACK QUESTIONS.
         `;
 
-        return systemPromptTemplate.replace('%CUSTOMFIELDS%', customFieldsStr);
-    }
+    return systemPromptTemplate.replace('%CUSTOMFIELDS%', customFieldsStr);
+  }
 
-    /**
-     * Generate system prompt for playground analysis
-     * @returns {string} System prompt
-     */
-    _generatePlaygroundSystemPrompt() {
-        return `
+  /**
+   * Generate system prompt for playground analysis
+   * @returns {string} System prompt
+   */
+  _generatePlaygroundSystemPrompt() {
+    return `
             You are a document analyzer. Your task is to analyze documents and extract relevant information. You do not ask back questions. 
             YOU MUSTNOT: Ask for additional information or clarification, or ask questions about the document, or ask for additional context.
             YOU MUSTNOT: Return a response without the desired JSON format.
@@ -472,88 +656,88 @@ class OllamaService {
             }
             ALWAYS USE THE INFORMATION TO FILL OUT THE JSON OBJECT. DO NOT ASK BACK QUESTIONS.
         `;
-    }
-
-    /**
-     * Calculate prompt token count
-     * @param {string} prompt - Prompt text
-     * @returns {number} Estimated token count
-     */
-    _calculatePromptTokenCount(prompt) {
-        // Use conservative 2 chars/token estimate to avoid truncation
-        // with non-English models (CJK, German, etc.) where tokenization
-        // produces roughly 2 chars per token instead of ~4 for English
-        return Math.ceil(prompt.length / 2);
-    }
+  }
 
-    /**
-     * Calculate context window size for Ollama
-     * @param {number} promptTokenCount - Token count for prompt
-     * @param {number} expectedResponseTokens - Expected response token count
-     * @returns {number} Context window size
-     */
-    _calculateNumCtx(promptTokenCount, expectedResponseTokens) {
-        const totalTokenUsage = promptTokenCount + expectedResponseTokens;
-        const maxCtxLimit = Number(config.tokenLimit);
+  /**
+   * Calculate prompt token count
+   * @param {string} prompt - Prompt text
+   * @returns {number} Estimated token count
+   */
+  _calculatePromptTokenCount(prompt) {
+    // Use conservative 2 chars/token estimate to avoid truncation
+    // with non-English models (CJK, German, etc.) where tokenization
+    // produces roughly 2 chars per token instead of ~4 for English
+    return Math.ceil(prompt.length / 2);
+  }
 
-        const numCtx = Math.min(totalTokenUsage, maxCtxLimit);
+  /**
+   * Calculate context window size for Ollama
+   * @param {number} promptTokenCount - Token count for prompt
+   * @param {number} expectedResponseTokens - Expected response token count
+   * @returns {number} Context window size
+   */
+  _calculateNumCtx(promptTokenCount, expectedResponseTokens) {
+    const totalTokenUsage = promptTokenCount + expectedResponseTokens;
+    const maxCtxLimit = Number(config.tokenLimit);
 
-        console.log('Prompt Token Count:', promptTokenCount);
-        console.log('Expected Response Tokens:', expectedResponseTokens);
-        console.log('Dynamic calculated num_ctx:', numCtx);
+    const numCtx = Math.min(totalTokenUsage, maxCtxLimit);
 
-        return numCtx;
-    }
+    console.log('Prompt Token Count:', promptTokenCount);
+    console.log('Expected Response Tokens:', expectedResponseTokens);
+    console.log('Dynamic calculated num_ctx:', numCtx);
 
-    /**
-     * Get available system memory
-     * @returns {Object} Object with totalMemoryMB and freeMemoryMB
-     */
-    async _getAvailableMemory() {
-        const totalMemory = os.totalmem();
-        const freeMemory = os.freemem();
-        const totalMemoryMB = (totalMemory / (1024 * 1024)).toFixed(0);
-        const freeMemoryMB = (freeMemory / (1024 * 1024)).toFixed(0);
-        return { totalMemoryMB, freeMemoryMB };
-    }
+    return numCtx;
+  }
 
-    /**
-     * Handle thumbnail caching for documents
-     * @param {string} id - Document ID
-     */
-    async _handleThumbnailCaching(id) {
-        if (!id) return;
+  /**
+   * Get available system memory
+   * @returns {Object} Object with totalMemoryMB and freeMemoryMB
+   */
+  async _getAvailableMemory() {
+    const totalMemory = os.totalmem();
+    const freeMemory = os.freemem();
+    const totalMemoryMB = (totalMemory / (1024 * 1024)).toFixed(0);
+    const freeMemoryMB = (freeMemory / (1024 * 1024)).toFixed(0);
+    return { totalMemoryMB, freeMemoryMB };
+  }
 
-        const cachePath = getThumbnailCachePath(id);
-        try {
-            await fs.access(cachePath);
-            console.log('[DEBUG] Thumbnail already cached');
-        } catch {
-            console.log('Thumbnail not cached, fetching from Paperless');
-            const thumbnailData = await paperlessService.getThumbnailImage(id);
-            if (!thumbnailData) {
-                console.warn('Thumbnail not found');
-                return;
-            }
-            await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
-            await fs.writeFile(cachePath, thumbnailData);
-        }
+  /**
+   * Handle thumbnail caching for documents
+   * @param {string} id - Document ID
+   */
+  async _handleThumbnailCaching(id) {
+    if (!id) return;
+
+    const cachePath = getThumbnailCachePath(id);
+    try {
+      await fs.access(cachePath);
+      console.log('[DEBUG] Thumbnail already cached');
+    } catch {
+      console.log('Thumbnail not cached, fetching from Paperless');
+      const thumbnailData = await paperlessService.getThumbnailImage(id);
+      if (!thumbnailData) {
+        console.warn('Thumbnail not found');
+        return;
+      }
+      await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+      await fs.writeFile(cachePath, thumbnailData);
     }
+  }
 
-    /**
-     * Build request headers, adding bearer auth when an API key is set.
-     * The key is read at request time so runtime config changes apply
-     * without re-instantiating the singleton.
-     * @returns {Object} Headers object
-     */
-    _buildRequestHeaders() {
-        const headers = { 'Content-Type': 'application/json' };
-        const apiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || '';
-        if (apiKey) {
-            headers.Authorization = `Bearer ${apiKey}`;
-        }
-        return headers;
+  /**
+   * Build request headers, adding bearer auth when an API key is set.
+   * The key is read at request time so runtime config changes apply
+   * without re-instantiating the singleton.
+   * @returns {Object} Headers object
+   */
+  _buildRequestHeaders() {
+    const headers = { 'Content-Type': 'application/json' };
+    const apiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || '';
+    if (apiKey) {
+      headers.Authorization = `Bearer ${apiKey}`;
     }
+    return headers;
+  }
 
   /**
    * Call Ollama API
@@ -584,24 +768,29 @@ class OllamaService {
       },
     };
 
-        // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
-        // unless explicitly enabled via OLLAMA_THINK=true
-        if (!config.ollama.think) {
-            requestBody.think = false;
-        }
-
-        const response = await this.client.post(`${this.apiUrl}/api/generate`, requestBody, {
-            headers: this._buildRequestHeaders()
-        });
+    // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
+    // unless explicitly enabled via OLLAMA_THINK=true
+    if (!config.ollama.think) {
+      requestBody.think = false;
+    }
 
-        if (!response.data) {
-            throw new Error('Invalid response from Ollama API');
-        }
+    const response = await this.client.post(
+      `${this.apiUrl}/api/generate`,
+      requestBody,
+      {
+        headers: this._buildRequestHeaders(),
+      }
+    );
 
-        this._assertNotTruncated(response.data, numPredict, numCtx);return response.data;
+    if (!response.data) {
+      throw new Error('Invalid response from Ollama API');
     }
 
-    /**
+    this._assertNotTruncated(response.data, numPredict, numCtx);
+    return response.data;
+  }
+
+  /**
    * A generation Ollama had to cut short comes back with done_reason "length".
    * What it returns is a prefix, and for structured output that means JSON
    * without its closing braces — which no amount of sanitizing recovers, so
@@ -655,222 +844,239 @@ class OllamaService {
   }
 
   /**
-     * Process Ollama API response
-     * @param {Object} responseData - Ollama API response data
-     * @returns {Object} Parsed response
-     */
-    _processOllamaResponse(responseData) {
-        if (responseData.response && typeof responseData.response === 'object') {
-            console.log('Using response JS object directly');
-            return {
-                tags: Array.isArray(responseData.response.tags) ? responseData.response.tags : [],
-                correspondent: responseData.response.correspondent || null,
-                title: responseData.response.title || null,
-                document_date: responseData.response.document_date || null,
-                document_type: responseData.response.document_type || null,
-                language: responseData.response.language || null,
-                custom_fields: responseData.response.custom_fields || null
-            };
-        } else if (responseData.response) {
-            console.log('No JS object yet, parsing JSON');
-            return this._parseResponse(responseData.response);
-        } else {
-            throw new Error('No response data from Ollama API');
-        }
+   * Process Ollama API response
+   * @param {Object} responseData - Ollama API response data
+   * @returns {Object} Parsed response
+   */
+  _processOllamaResponse(responseData) {
+    if (responseData.response && typeof responseData.response === 'object') {
+      console.log('Using response JS object directly');
+      return {
+        tags: Array.isArray(responseData.response.tags)
+          ? responseData.response.tags
+          : [],
+        correspondent: responseData.response.correspondent || null,
+        title: responseData.response.title || null,
+        document_date: responseData.response.document_date || null,
+        document_type: responseData.response.document_type || null,
+        language: responseData.response.language || null,
+        custom_fields: responseData.response.custom_fields || null,
+      };
+    } else if (responseData.response) {
+      console.log('No JS object yet, parsing JSON');
+      return this._parseResponse(responseData.response);
+    } else {
+      throw new Error('No response data from Ollama API');
     }
+  }
 
-    /**
-     * Extract token usage metrics from an Ollama /api/generate response.
-     * @param {Object} responseData - Ollama API response data
-     * @returns {{promptTokens:number, completionTokens:number, totalTokens:number}}
-     */
-    _extractOllamaMetrics(responseData) {
-        const promptTokens = Number(responseData?.prompt_eval_count) || 0;
-        const completionTokens = Number(responseData?.eval_count) || 0;
-
-        return {
-            promptTokens,
-            completionTokens,
-            totalTokens: promptTokens + completionTokens
-        };
-    }
+  /**
+   * Extract token usage metrics from an Ollama /api/generate response.
+   * @param {Object} responseData - Ollama API response data
+   * @returns {{promptTokens:number, completionTokens:number, totalTokens:number}}
+   */
+  _extractOllamaMetrics(responseData) {
+    const promptTokens = Number(responseData?.prompt_eval_count) || 0;
+    const completionTokens = Number(responseData?.eval_count) || 0;
+
+    return {
+      promptTokens,
+      completionTokens,
+      totalTokens: promptTokens + completionTokens,
+    };
+  }
 
-    /**
-     * Parse text response to extract JSON
-     * @param {string} response - Response text
-     * @returns {Object} Parsed object
-     */
-    /* Every exit from here either returns something the caller can use or
+  /**
+   * Parse text response to extract JSON
+   * @param {string} response - Response text
+   * @returns {Object} Parsed object
+   */
+  /* Every exit from here either returns something the caller can use or
      throws. It used to hand back { tags: [], correspondent: null } on each of
      its three failure paths, which is indistinguishable from a model that
      genuinely found nothing — so a document whose answer could not be read at
      all was written back untouched and then marked processed, never to be
-     looked at again. openaiService has thrown at the same point all along. */_parseResponse(response) {
-        try {
-            // Find JSON in response using regex
-            const jsonMatch = response.match(/\{[\s\S]*\}/);
-            if (!jsonMatch) {
-                /* Worded to carry the marker serviceUtils already classifies, so a
+     looked at again. openaiService has thrown at the same point all along. */ _parseResponse(
+    response
+  ) {
+    try {
+      // Find JSON in response using regex
+      const jsonMatch = response.match(/\{[\s\S]*\}/);
+      if (!jsonMatch) {
+        /* Worded to carry the marker serviceUtils already classifies, so a
            document whose answer was unreadable takes the same OCR fallback it
            takes under OpenAI. Truncation deliberately does not — see
            _assertNotTruncated, where a second pass cannot help. */
         throw new Error(
           'Invalid JSON response from API: no JSON object in the answer'
         );
-            }
+      }
+
+      let jsonStr = jsonMatch[0];
+      console.log('Extracted JSON String:', jsonStr);
+
+      try {
+        // Attempt to parse the JSON
+        const result = JSON.parse(jsonStr);
+
+        // Validate and return the result
+        return {
+          tags: Array.isArray(result.tags) ? result.tags : [],
+          correspondent: result.correspondent || null,
+          title: result.title || null,
+          document_date: result.document_date || null,
+          document_type: result.document_type || null,
+          language: result.language || null,
+          custom_fields: result.custom_fields || null,
+        };
+      } catch (jsonError) {
+        console.warn('Error parsing JSON from response:', jsonError.message);
+        console.warn('Attempting to sanitize the JSON...');
 
-            let jsonStr = jsonMatch[0];
-            console.log('Extracted JSON String:', jsonStr);
-
-            try {
-                // Attempt to parse the JSON
-                const result = JSON.parse(jsonStr);
-
-                // Validate and return the result
-                return {
-                    tags: Array.isArray(result.tags) ? result.tags : [],
-                    correspondent: result.correspondent || null,
-                    title: result.title || null,
-                    document_date: result.document_date || null,
-                    document_type: result.document_type || null,
-                    language: result.language || null,
-                    custom_fields: result.custom_fields || null
-                };
-
-            } catch (jsonError) {
-                console.warn('Error parsing JSON from response:', jsonError.message);
-                console.warn('Attempting to sanitize the JSON...');
-
-                // Sanitize the JSON
-                jsonStr = this._sanitizeJsonString(jsonStr);
-
-                try {
-                    const sanitizedResult = JSON.parse(jsonStr);
-                    return {
-                        tags: Array.isArray(sanitizedResult.tags) ? sanitizedResult.tags : [],
-                        correspondent: sanitizedResult.correspondent || null,
-                        title: sanitizedResult.title || null,
-                        document_date: sanitizedResult.document_date || null,
-                        language: sanitizedResult.language || null
-                    };
-                } catch {
-                    console.error('Final JSON parsing failed after sanitization. This happens when the JSON structure is too complex or invalid. That indicates an issue with the generated JSON string by Ollama. Switch to OpenAI for better results or fine tune your prompt.');
-                    throw new Error(
+        // Sanitize the JSON
+        jsonStr = this._sanitizeJsonString(jsonStr);
+
+        try {
+          const sanitizedResult = JSON.parse(jsonStr);
+          return {
+            tags: Array.isArray(sanitizedResult.tags)
+              ? sanitizedResult.tags
+              : [],
+            correspondent: sanitizedResult.correspondent || null,
+            title: sanitizedResult.title || null,
+            document_date: sanitizedResult.document_date || null,
+            language: sanitizedResult.language || null,
+          };
+        } catch {
+          console.error(
+            'Final JSON parsing failed after sanitization. This happens when the JSON structure is too complex or invalid. That indicates an issue with the generated JSON string by Ollama. Switch to OpenAI for better results or fine tune your prompt.'
+          );
+          throw new Error(
             `Invalid JSON response from API, unreadable even after sanitizing: ${jsonError.message}`
           );
-                }
-            }
-        } catch (error) {
-            console.error('Error parsing Ollama response:', error.message);
-            throw error;
         }
+      }
+    } catch (error) {
+      console.error('Error parsing Ollama response:', error.message);
+      throw error;
     }
+  }
 
-    /**
-     * Sanitize a JSON string
-     * @param {string} jsonStr - JSON string to sanitize
-     * @returns {string} Sanitized JSON string
-     */
-    _sanitizeJsonString(jsonStr) {
-        return jsonStr
-            .replace(/,\s*}/g, '}') // Remove trailing commas before closing braces
-            .replace(/,\s*]/g, ']') // Remove trailing commas before closing brackets
-            .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":'); // Ensure property names are quoted
-    }
+  /**
+   * Sanitize a JSON string
+   * @param {string} jsonStr - JSON string to sanitize
+   * @returns {string} Sanitized JSON string
+   */
+  _sanitizeJsonString(jsonStr) {
+    return jsonStr
+      .replace(/,\s*}/g, '}') // Remove trailing commas before closing braces
+      .replace(/,\s*]/g, ']') // Remove trailing commas before closing brackets
+      .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":'); // Ensure property names are quoted
+  }
 
-    /**
-     * Log prompt and response to file
-     * @param {string} prompt - Prompt text
-     * @param {Object} response - Response object
-     */
-    async _logPromptAndResponse(prompt, response) {
-        const content = '================================================================================'
-            + prompt + "\n\n"
-            + JSON.stringify(response)
-            + '\n\n'
-            + '================================================================================\n\n';
-
-        await writePromptToFile(content);
-    }
+  /**
+   * Log prompt and response to file
+   * @param {string} prompt - Prompt text
+   * @param {Object} response - Response object
+   */
+  async _logPromptAndResponse(prompt, response) {
+    const content =
+      '================================================================================' +
+      prompt +
+      '\n\n' +
+      JSON.stringify(response) +
+      '\n\n' +
+      '================================================================================\n\n';
+
+    await writePromptToFile(content);
+  }
 
-    /**
-     * Generate text based on a prompt
-     * @param {string} prompt - The prompt to generate text from
-     * @returns {Promise<string>} - The generated text
-     */
-    async generateText(prompt) {
-        try {
-            // Calculate context window size based on prompt length
-            const promptTokenCount = this._calculatePromptTokenCount(prompt);
-            const numCtx = this._calculateNumCtx(promptTokenCount, Number(config.responseTokens));
-
-            // Simple system prompt for text generation
-            const systemPrompt = `You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.`;
-
-            // Call Ollama API without enforcing a specific response format
-            const generateTextBody = {
-                model: this.model,
-                prompt: prompt,
-                system: systemPrompt,
-                stream: false,
-                options: {
-                    temperature: config.aiTemperatureGeneration,
-                    top_p: 0.9,
-                    // The line above reserves config.responseTokens in the context; a
+  /**
+   * Generate text based on a prompt
+   * @param {string} prompt - The prompt to generate text from
+   * @returns {Promise<string>} - The generated text
+   */
+  async generateText(prompt) {
+    try {
+      // Calculate context window size based on prompt length
+      const promptTokenCount = this._calculatePromptTokenCount(prompt);
+      const numCtx = this._calculateNumCtx(
+        promptTokenCount,
+        Number(config.responseTokens)
+      );
+
+      // Simple system prompt for text generation
+      const systemPrompt = `You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.`;
+
+      // Call Ollama API without enforcing a specific response format
+      const generateTextBody = {
+        model: this.model,
+        prompt: prompt,
+        system: systemPrompt,
+        stream: false,
+        options: {
+          temperature: config.aiTemperatureGeneration,
+          top_p: 0.9,
+          // The line above reserves config.responseTokens in the context; a
           // constant here would have the same quarrel with it that the
           // analysis path had.
           num_predict: Number(config.responseTokens),
-                    num_ctx: numCtx
-                }
-            };
-
-            // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
-            // unless explicitly enabled via OLLAMA_THINK=true
-            if (!config.ollama.think) {
-                generateTextBody.think = false;
-            }
-
-            const response = await this.client.post(`${this.apiUrl}/api/generate`, generateTextBody, {
-                headers: this._buildRequestHeaders()
-            });
+          num_ctx: numCtx,
+        },
+      };
+
+      // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
+      // unless explicitly enabled via OLLAMA_THINK=true
+      if (!config.ollama.think) {
+        generateTextBody.think = false;
+      }
+
+      const response = await this.client.post(
+        `${this.apiUrl}/api/generate`,
+        generateTextBody,
+        {
+          headers: this._buildRequestHeaders(),
+        }
+      );
 
-            if (!response.data || !response.data.response) {
-                throw new Error('Invalid response from Ollama API');
-            }
+      if (!response.data || !response.data.response) {
+        throw new Error('Invalid response from Ollama API');
+      }
 
-            return response.data.response;
-        } catch (error) {
-             console.error(`Error generating text with Ollama: ${error.message}`); console.debug(error);
-            throw error;
-        }
+      return response.data.response;
+    } catch (error) {
+      console.error(`Error generating text with Ollama: ${error.message}`);
+      console.debug(error);
+      throw error;
     }
+  }
 
-    /**
-     * Check if the Ollama service is running
-     * @returns {Promise<boolean>} - True if the service is running, false otherwise
-     */
-    async checkStatus() {
-        // use ollama status endpoint
-        try {
-            const response = await this.client.get(`${this.apiUrl}/api/ps`, {
-                headers: this._buildRequestHeaders()
-            });
-            if (response.status === 200) {
-                const data = response.data;
-                // Ensure data is an array and has at least one model
-                let modelName = null;
-                if (Array.isArray(data.models) && data.models.length > 0) {
-                    modelName = data.models[0].name;
-                }
-                console.log('Ollama model name:', modelName);
-                return { status: 'ok', model: modelName };
-            }
-        } catch (error) {
-            console.error(`Error checking Ollama service status: ${error.message}`);
-            console.debug(error);
+  /**
+   * Check if the Ollama service is running
+   * @returns {Promise<boolean>} - True if the service is running, false otherwise
+   */
+  async checkStatus() {
+    // use ollama status endpoint
+    try {
+      const response = await this.client.get(`${this.apiUrl}/api/ps`, {
+        headers: this._buildRequestHeaders(),
+      });
+      if (response.status === 200) {
+        const data = response.data;
+        // Ensure data is an array and has at least one model
+        let modelName = null;
+        if (Array.isArray(data.models) && data.models.length > 0) {
+          modelName = data.models[0].name;
         }
-        return { status: 'error' };
+        console.log('Ollama model name:', modelName);
+        return { status: 'ok', model: modelName };
+      }
+    } catch (error) {
+      console.error(`Error checking Ollama service status: ${error.message}`);
+      console.debug(error);
     }
+    return { status: 'error' };
+  }
 }
 
 module.exports = new OllamaService();

From 4b2d44ef10a35e9bb93041f3c68d78a971938872 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Wed, 26 Aug 2026 15:07:59 +0200
Subject: [PATCH 13/53] fix(predefined-scan): let the trigger tag re-queue
 processed documents
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
"Process only specific pre-tagged documents" (PROCESS_PREDEFINED_DOCUMENTS
+ TAGS) was one-way: processDocument() consulted processed_documents before
anything else, so assigning the trigger tag to an already analysed document
— the documented way to re-run documents after a bugfix — did nothing. And
updateDocument() merges AI tags into the current tags, so the trigger tag
stuck around after processing regardless.

Changes
- server.js: new documentCarriesTriggerTag() resolves TAGS through the
  shared tag cache; in predefined mode, a tagged document is re-processed
  despite an existing processed record. Ignored and permanently-failed
  guards stay ahead of the bypass, so a re-tag cannot resurrect those.
- paperlessService.updateDocument(): accepts updates.removeTagIds and
  filters them from the merged tag list — removal must happen here because
  the merge would otherwise re-add the tag from currentDoc.tags on every
  write-back.
- saveDocumentChanges() passes the trigger tag ids as removeTagIds when
  the new REMOVE_TRIGGER_TAGS setting (default no) is enabled, so the tag
  comes off after successful processing and re-adding it queues a re-run.
- config.js: removeTriggerTags parsed via parseEnvBoolean.

Testing
New tests/test-predefined-retrigger.js reproduces both defects first
(fails against the old code), then covers: source-level guard for the
re-trigger bypass including ignored/failed precedence, and functional
updateDocument behaviour against a mocked Paperless API (merge keeps
existing + AI tags, trigger tag removed from the PATCH payload).
Full suite otherwise unchanged.

Impact
The intended workflow works: tag a document with e.g. perform-ai-scan and
the next scan processes it, even if it was processed years ago. With
REMOVE_TRIGGER_TAGS=yes the tag is removed after success, making "tag
present" mean "work outstanding". Default off preserves legacy behaviour;
settings-page wiring for the switch can follow if wanted.

Upstream Status
Not applicable — the predefined-documents mode is specific to this fork.
---
 config/config.js                   |   5 +
 server.js                          |  84 ++++++++++++++++-
 services/paperlessService.js       |  21 ++++-
 tests/test-predefined-retrigger.js | 143 +++++++++++++++++++++++++++++
 4 files changed, 251 insertions(+), 2 deletions(-)
 create mode 100644 tests/test-predefined-retrigger.js

diff --git a/config/config.js b/config/config.js
index 105e2c0c..b8d7abbc 100644
--- a/config/config.js
+++ b/config/config.js
@@ -451,6 +451,11 @@ module.exports = {
   globalRateLimitMax: parseInt(process.env.GLOBAL_RATE_LIMIT_MAX || '1000', 10),
   predefinedMode: process.env.PROCESS_PREDEFINED_DOCUMENTS,
   ignoreTags: process.env.IGNORE_TAGS || '',
+  // Predefined-scan trigger tags (TAGS in predefined mode) are removed from a
+  // document after it has been processed, so re-adding the tag is what queues
+  // a re-run. Off by default: without it the tag stays, matching legacy
+  // behaviour.
+  removeTriggerTags: parseEnvBoolean(process.env.REMOVE_TRIGGER_TAGS, 'no'),
   tokenLimit: process.env.TOKEN_LIMIT || 128000,
   // How many tokens a provider may spend on its answer. Reserved in the
   // context window and, where the provider offers a knob for it, sent as the
diff --git a/server.js b/server.js
index 4816d4a8..263000e7 100644
--- a/server.js
+++ b/server.js
@@ -617,6 +617,50 @@ async function saveOpenApiSpec() {
 }
 
 // Document processing functions
+
+/**
+ * True when the document carries at least one of the tags configured as the
+ * predefined-scan trigger (PROCESS_PREDEFINED_DOCUMENTS=yes + TAGS). Only
+ * meaningful in predefined mode; resolves the tag names to ids through the
+ * shared tag cache.
+ *
+ * @param {Object} doc - document as returned by getAllDocuments() (fields incl. tags)
+ * @returns {Promise<boolean>}
+ */
+async function documentCarriesTriggerTag(doc) {
+  if (config.predefinedMode !== 'yes') {
+    return false;
+  }
+
+  const triggerNames = (process.env.TAGS || '')
+    .split(',')
+    .map((tag) => tag.trim())
+    .filter(Boolean);
+  if (triggerNames.length === 0) {
+    return false;
+  }
+
+  const docTagIds = new Set(
+    (Array.isArray(doc.tags) ? doc.tags : [])
+      .map((tag) => (typeof tag === 'object' ? Number(tag?.id) : Number(tag)))
+      .filter((id) => Number.isInteger(id))
+  );
+  if (docTagIds.size === 0) {
+    return false;
+  }
+
+  try {
+    const triggerIds = await paperlessService.resolveTagIdsByName(triggerNames);
+    return triggerIds.some((id) => docTagIds.has(id));
+  } catch (error) {
+    console.warn(
+      `[WARN] Could not resolve trigger tags for re-check of document ${doc.id}:`,
+      error.message
+    );
+    return false;
+  }
+}
+
 async function processDocument(
   doc,
   existingTags,
@@ -624,7 +668,19 @@ async function processDocument(
   existingDocumentTypesList
 ) {
   const isProcessed = await documentModel.isDocumentProcessed(doc.id);
-  if (isProcessed) return null;
+  if (isProcessed) {
+    // Predefined-scan re-trigger: in "process only specific pre-tagged
+    // documents" mode, the presence of a configured trigger tag on an already
+    // processed document is the operator's explicit request to run it again —
+    // that is how documents are re-queued after fixing a bug. Outside
+    // predefined mode, or without a trigger tag, the record wins.
+    if (!(await documentCarriesTriggerTag(doc))) {
+      return null;
+    }
+    console.info(
+      `Document ${doc.id} carries a configured trigger tag; re-processing despite existing record`
+    );
+  }
 
   const isIgnored = await documentModel.isDocumentIgnored(doc.id);
   if (isIgnored) {
@@ -965,6 +1021,32 @@ async function saveDocumentChanges(docId, updateData, analysis, originalData) {
   const origDocType = originalData.document_type ?? null;
   const origLanguage = originalData.language ?? null;
 
+  // Predefined-scan trigger tags have done their job once the document has
+  // been processed; leaving them in place would make the next scan re-run
+  // the document forever. Removal only happens when the operator opts in.
+  if (
+    config.predefinedMode === 'yes' &&
+    config.removeTriggerTags === 'yes' &&
+    Array.isArray(updateData.tags)
+  ) {
+    try {
+      const triggerNames = (process.env.TAGS || '')
+        .split(',')
+        .map((tag) => tag.trim())
+        .filter(Boolean);
+      if (triggerNames.length > 0) {
+        updateData.removeTagIds = await paperlessService.resolveTagIdsByName(
+          triggerNames
+        );
+      }
+    } catch (error) {
+      console.warn(
+        `[WARN] Could not resolve trigger tags for removal on document ${docId}:`,
+        error.message
+      );
+    }
+  }
+
   await documentModel.saveOriginalData(
     docId,
     originalTags,
diff --git a/services/paperlessService.js b/services/paperlessService.js
index 79bcdf9a..3a3c2051 100644
--- a/services/paperlessService.js
+++ b/services/paperlessService.js
@@ -2235,9 +2235,28 @@ class PaperlessService {
         console.log(`[DEBUG] Current correspondent:`, currentDoc.correspondent);
         console.log(`[DEBUG] New correspondent:`, updates.correspondent);
 
-        const combinedTags = [
+        let combinedTags = [
           ...new Set([...currentDoc.tags, ...updates.tags]),
         ];
+
+        // Predefined-scan trigger tags are removed here rather than by the
+        // caller: the merge above would otherwise re-add them from
+        // currentDoc.tags on every write-back.
+        if (
+          Array.isArray(updates.removeTagIds) &&
+          updates.removeTagIds.length > 0
+        ) {
+          const removeSet = new Set(updates.removeTagIds.map(Number));
+          combinedTags = combinedTags.filter((tagId) => {
+            const normalized =
+              typeof tagId === 'object' ? Number(tagId?.id) : Number(tagId);
+            return !removeSet.has(normalized);
+          });
+          console.log(
+            `[DEBUG] Removed configured tag ids from document ${documentId}:`,
+            updates.removeTagIds
+          );
+        }
         updates.tags = combinedTags;
 
         console.log(`[DEBUG] Combined tags:`, combinedTags);
diff --git a/tests/test-predefined-retrigger.js b/tests/test-predefined-retrigger.js
new file mode 100644
index 00000000..67f888cd
--- /dev/null
+++ b/tests/test-predefined-retrigger.js
@@ -0,0 +1,143 @@
+/**
+ * Reproduces the broken "Process only specific pre-tagged documents" workflow.
+ *
+ * The operator assigns a trigger tag (e.g. perform-ai-scan) in Paperless-ngx,
+ * expecting Zettelrobbe to process the document — including AGAIN, when the
+ * tag is set on a document that was already analysed (the documented way to
+ * re-run documents after fixing a bug).
+ *
+ * Two defects made that workflow impossible:
+ * 1. processDocument() consulted processed_documents before anything else, so
+ *    a re-tagged document was silently skipped forever.
+ * 2. updateDocument() merged the AI tag list into the current tags, so the
+ *    trigger tag stuck around after processing instead of being removed.
+ *
+ * Covers:
+ * 1. The scan loop re-processes a tagged document that is already recorded
+ *    as processed (source-level guard, since the loop lives in server.js)
+ * 2. Ignored and permanently-failed documents stay protected
+ * 3. updateDocument() drops the trigger tag from the merged payload when
+ *    asked to (functional, against a mocked Paperless API)
+ */
+
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+
+let passed = 0;
+let failed = 0;
+
+function test(name, fn) {
+  try {
+    fn();
+    console.log(`✅  ${name}`);
+    passed++;
+  } catch (error) {
+    console.error(`❌  ${name}`);
+    console.error(`    ${error.message}`);
+    failed++;
+  }
+}
+
+const serverSource = fs.readFileSync(
+  path.join(process.cwd(), 'server.js'),
+  'utf8'
+);
+
+test('A tagged document is re-processed even when already recorded', () => {
+  // The processed-documents guard must not fire unconditionally: in
+  // predefined mode the presence of a configured trigger tag is the
+  // operator's explicit request to run the document again.
+  assert.match(
+    serverSource,
+    /isDocumentProcessed\(doc\.id\)[\s\S]{0,600}?predefined/i,
+    'The isProcessed guard must consider the predefined trigger tags'
+  );
+});
+
+test('The re-trigger respects ignored and permanently-failed documents', () => {
+  const processedIdx = serverSource.indexOf('isDocumentProcessed(doc.id)');
+  const ignoredIdx = serverSource.indexOf('isDocumentIgnored(doc.id)');
+  const failedIdx = serverSource.indexOf('isDocumentFailed(doc.id)');
+  assert.ok(processedIdx !== -1 && ignoredIdx !== -1 && failedIdx !== -1);
+  // Ignored/failed checks come after the processed check but must NOT sit
+  // behind the re-trigger bypass, or a re-tagged ignore would resurrect.
+  const between = serverSource.slice(ignoredIdx, failedIdx + 200);
+  assert.ok(
+    !/return null;\s*\}\s*\n\s*\/\/.*predefined|re-?trigger/i.test(between),
+    'Ignored/failed guards must stay ahead of any re-trigger logic'
+  );
+});
+
+(async () => {
+  // ── updateDocument drops the trigger tag on request ──────────────────────
+  process.env.PAPERLESS_API_URL = 'http://paperless.test';
+  process.env.PAPERLESS_API_TOKEN = 'test-token';
+
+  const calls = [];
+  const paperlessService = require('../services/paperlessService');
+
+  const originalClient = paperlessService.client;
+  paperlessService.client = {
+    defaults: { baseURL: 'http://paperless.test/api' },
+    get: async (url) => {
+      calls.push({ method: 'get', url });
+      if (url === '/documents/315/') {
+        return {
+          data: {
+            id: 315,
+            tags: [11, 42], // 42 is the trigger tag
+            correspondent: 7,
+          },
+        };
+      }
+      throw new Error(`unexpected GET ${url}`);
+    },
+    patch: async (url, body) => {
+      calls.push({ method: 'patch', url, body });
+      if (url === '/documents/315/') {
+        return { data: { ok: true } };
+      }
+      throw new Error(`unexpected PATCH ${url}`);
+    },
+    post: async (url) => {
+      throw new Error(`unexpected POST ${url}`);
+    },
+  };
+
+  try {
+    await test(
+      'updateDocument removes configured tag ids from the merged payload',
+      async () => {
+        await paperlessService.updateDocument(315, {
+          title: 'New title',
+          tags: [5], // AI-chosen tags
+          removeTagIds: [42], // the predefined-mode trigger tag
+        });
+
+        const updateCall = calls.find((call) => call.method === 'patch');
+        assert.ok(updateCall, 'an update request was sent');
+        const sentTags = updateCall.body.tags;
+        assert.deepStrictEqual(
+          [...sentTags].sort((a, b) => a - b),
+          [5, 11],
+          'AI tags merge with existing tags, the trigger tag is gone'
+        );
+        assert.strictEqual(
+          sentTags.includes(42),
+          false,
+          'the trigger tag must not survive the write-back'
+        );
+      }
+    );
+  } finally {
+    paperlessService.client = originalClient;
+  }
+
+  console.log(`\n${passed} passed, ${failed} failed`);
+  if (failed > 0) {
+    process.exitCode = 1;
+  }
+})();

From a8d12f9176091cf8beb5e33d4a85c90b544a622a Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Wed, 26 Aug 2026 15:50:47 +0200
Subject: [PATCH 14/53] fix(ollama): surface the upstream cause of 5xx failures
 and retry once
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Field report: a large document failed with ERR_BAD_RESPONSE and nothing
but "Request failed with status code 500". The log's axios dump hid the
actual cause one level down — Ollama answered 500 with
{ error: 'Post "http://127.0.0.1:49496/v1/completions": EOF' }, i.e. its
own inference backend (the MLX runner behind my-agent-model) closed the
connection mid-request, most likely under context/memory pressure on a
long document. Two gaps: the operator could not tell an Ollama problem
from a model-backend problem, and a transient EOF failed the document
for good.

Changes
- _extractUpstreamErrorMessage(): reads the error/message out of a
  non-2xx response body (JSON object or plain string) so the failing
  backend call and its reason reach analysis.error and the logs.
- One bounded retry for transient failures only: no response at all or
  HTTP >= 5xx gets exactly one second attempt after a short backoff.
  4xx never retries. The axios code (e.g. ERR_BAD_RESPONSE) is carried
  onto the wrapped error, so scan-loop classification stays unchanged.
- tests/test-predefined-retrigger.js: fixed its test() helper to await
  the case function — it was sync, so the client-mocking finally block
  raced the in-flight updateDocument call.

Testing
New tests/test-ollama-upstream-error.js reproduces the field report:
the upstream "v1/completions: EOF" text reaches analysis.error with
ERR_BAD_RESPONSE preserved; a first-attempt 500 followed by success is
retried exactly once and succeeds; a persistent 5xx fails after two
attempts with the detail kept; a 404 is not retried. Full suite: 78
tests, same 7 pre-existing Windows-checkout artifacts as before.

Impact
Operators see which backend call died and why instead of an opaque 500;
transient backend drops no longer mark documents as failed. Documents
that already failed with ERR_BAD_RESPONSE can be re-queued via the
predefined-scan trigger tag from the previous commit.

Upstream Status
Not applicable — upstream has neither upstream-error extraction nor
retry behaviour on the Ollama path.
---
 scripts/run-tests.js                |   2 +
 services/ollamaService.js           |  92 ++++++++++++-
 tests/test-ollama-upstream-error.js | 199 ++++++++++++++++++++++++++++
 tests/test-predefined-retrigger.js  |  47 +++----
 4 files changed, 308 insertions(+), 32 deletions(-)
 create mode 100644 tests/test-ollama-upstream-error.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index 828f4be3..5f808597 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -58,6 +58,8 @@ const TESTS = {
   'document-metadata-batching': 'test-document-metadata-batching.js',
   'pr772-fix': 'test-pr772-fix.js',
   'ollama-response-limit': 'test-ollama-response-limit.js',
+  'ollama-upstream-error': 'test-ollama-upstream-error.js',
+  'predefined-retrigger': 'test-predefined-retrigger.js',
   'response-truncation-detection': 'test-response-truncation-detection.js',
   'ollama-temperature-wiring': 'test-ollama-temperature-wiring.js',
   'quickstart-model-classification': 'test-quickstart-model-classification.js',
diff --git a/services/ollamaService.js b/services/ollamaService.js
index 5d17c6f4..9b2225a1 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -747,6 +747,67 @@ class OllamaService {
    * @param {Object} schema - Response schema
    * @returns {Object} Ollama API response
    */
+  /**
+   * Extract the cause an Ollama (or its inference backend) reported in a
+   * non-2xx response body. Ollama wraps backend failures — its runner closing
+   * the connection on an oversized prompt, for example — into
+   * `{ error: 'Post "http://…/v1/completions": EOF' }`; without this the
+   * operator only ever sees "Request failed with status code 500" and cannot
+   * tell an Ollama problem from a model-backend problem.
+   *
+   * @param {Object} axiosError
+   * @returns {string|null}
+   */
+  _extractUpstreamErrorMessage(axiosError) {
+    const data = axiosError?.response?.data;
+    if (!data) return null;
+    if (typeof data === 'string') {
+      const trimmed = data.trim();
+      try {
+        return JSON.parse(trimmed)?.error || trimmed || null;
+      } catch {
+        return trimmed || null;
+      }
+    }
+    return data.error || data.message || null;
+  }
+
+  /**
+   * True for failures worth one more attempt: the connection dropped or the
+   * server answered 5xx. A 4xx is a request problem and will fail again.
+   *
+   * @param {Object} axiosError
+   * @returns {boolean}
+   */
+  _isTransientOllamaFailure(axiosError) {
+    if (!axiosError.response) {
+      // No response at all: connection refused/reset/timeout.
+      return true;
+    }
+    return axiosError.response.status >= 500;
+  }
+
+  /**
+   * Wrap an axios failure with the upstream detail attached.
+   *
+   * @param {Object} axiosError - The axios error being wrapped
+   * @param {number} attempt - Which attempt failed (1-based)
+   * @returns {Error}
+   */
+  _wrapOllamaRequestError(axiosError, attempt) {
+    const upstream = this._extractUpstreamErrorMessage(axiosError);
+    const status = axiosError?.response?.status;
+    const detail = upstream ? `: ${upstream}` : '';
+    const wrapped = new Error(
+      `Ollama request failed (HTTP ${status || 'no response'}, attempt ${attempt})${detail}`,
+      { cause: axiosError }
+    );
+    // Keep the axios code so the scan-loop classification (OCR fallback,
+    // failure reasons) keeps working unchanged.
+    wrapped.code = axiosError.code;
+    return wrapped;
+  }
+
   async _callOllamaAPI(prompt, systemPrompt, numCtx, schema) {
     // The same number _calculateNumCtx() already reserved room for. It used to
     // be a hardcoded 256 while the reservation followed the setting, so the
@@ -774,15 +835,32 @@ class OllamaService {
       requestBody.think = false;
     }
 
-    const response = await this.client.post(
-      `${this.apiUrl}/api/generate`,
-      requestBody,
-      {
-        headers: this._buildRequestHeaders(),
+    // One bounded retry for transient failures (backend EOF, 5xx). The
+    // field report behind this: an MLX runner closed its connection once
+    // under memory pressure; the immediate second attempt succeeded.
+    const MAX_ATTEMPTS = 2;
+    let response;
+    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
+      try {
+        response = await this.client.post(
+          `${this.apiUrl}/api/generate`,
+          requestBody,
+          {
+            headers: this._buildRequestHeaders(),
+          }
+        );
+        break;
+      } catch (error) {
+        const wrapped = this._wrapOllamaRequestError(error, attempt);
+        console.error(`[ERROR] ${wrapped.message}`);
+        if (attempt >= MAX_ATTEMPTS || !this._isTransientOllamaFailure(error)) {
+          throw wrapped;
+        }
+        await new Promise((resolve) => setTimeout(resolve, 1000));
       }
-    );
+    }
 
-    if (!response.data) {
+    if (!response?.data) {
       throw new Error('Invalid response from Ollama API');
     }
 
diff --git a/tests/test-ollama-upstream-error.js b/tests/test-ollama-upstream-error.js
new file mode 100644
index 00000000..900f7954
--- /dev/null
+++ b/tests/test-ollama-upstream-error.js
@@ -0,0 +1,199 @@
+/**
+ * Surface the upstream cause of Ollama 5xx responses and retry once.
+ *
+ * Field report: Ollama answered HTTP 500 { error: 'Post
+ * "http://127.0.0.1:49496/v1/completions": EOF' } — its own inference
+ * backend closed the connection on a large document. analyzeDocument only
+ * reported "Request failed with status code 500", hiding both the fact that
+ * the failure was upstream of Ollama and any hint about which limit bit.
+ *
+ * Covers:
+ * 1. The upstream error body reaches analysis.error / the log
+ * 2. A transient 5xx is retried once; a success on the second attempt wins
+ * 3. A persistent 5xx fails with the upstream detail preserved
+ */
+
+'use strict';
+
+const assert = require('assert');
+
+process.env.AI_PROVIDER = 'ollama';
+process.env.OLLAMA_API_URL = 'http://ollama.test';
+process.env.OLLAMA_MODEL = 'test-model';
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+process.env.CUSTOM_FIELDS = JSON.stringify({ custom_fields: [] });
+delete process.env.CONTENT_MAX_LENGTH;
+
+const ollamaService = require('../services/ollamaService');
+
+let passed = 0;
+let failed = 0;
+
+async function test(name, fn) {
+  try {
+    await fn();
+    console.log(`✅  ${name}`);
+    passed++;
+  } catch (error) {
+    console.error(`❌  ${name}`);
+    console.error(`    ${error.message}`);
+    failed++;
+  }
+}
+
+const UPSTREAM_ERROR = 'Post "http://127.0.0.1:49496/v1/completions": EOF';
+
+/** An Ollama that answers like the field report: 500 carrying its backend error. */
+function ollamWithResponses(responses) {
+  const attempts = [];
+  return {
+    attempts,
+    post: async () => {
+      const response = responses.shift();
+      attempts.push(response);
+      if (response instanceof Error) throw response;
+      return response;
+    },
+  };
+}
+
+function successResponse() {
+  return {
+    data: {
+      done_reason: 'stop',
+      response: JSON.stringify({
+        title: 'T',
+        correspondent: null,
+        tags: ['a'],
+        document_type: 'd',
+        document_date: '2026-01-01',
+        language: 'de',
+      }),
+      eval_count: 20,
+      prompt_eval_count: 30,
+    },
+  };
+}
+
+function serverError(body) {
+  const error = new Error(`Request failed with status code 500`);
+  error.code = 'ERR_BAD_RESPONSE';
+  error.response = { status: 500, data: body };
+  return error;
+}
+
+(async () => {
+  const originalSetTimeout = global.setTimeout;
+  const originalClient = ollamaService.client;
+
+  try {
+    // Shorten retries in tests
+    const originalSetTimeout = setTimeout;
+    global.setTimeout = (fn) => originalSetTimeout(fn, 1);
+
+    await test('The upstream error body reaches analysis.error', async () => {
+      ollamaService.client = ollamWithResponses([
+        serverError({ error: UPSTREAM_ERROR }),
+        serverError({ error: UPSTREAM_ERROR }),
+      ]);
+
+      const analysis = await ollamaService.analyzeDocument(
+        'short content',
+        [],
+        [],
+        [],
+        370
+      );
+
+      assert.match(
+        analysis.error,
+        /v1\/completions.*EOF/s,
+        'the operator must see WHICH backend call failed'
+      );
+      assert.strictEqual(
+        analysis.errorCode,
+        'ERR_BAD_RESPONSE',
+        'the axios code stays available for classification'
+      );
+    });
+
+    await test('A transient 5xx is retried once', async () => {
+      const ollama = ollamWithResponses([
+        serverError({ error: UPSTREAM_ERROR }),
+        successResponse(),
+      ]);
+      ollamaService.client = ollama;
+
+      const analysis = await ollamaService.analyzeDocument(
+        'short content',
+        [],
+        [],
+        [],
+        371
+      );
+
+      assert.strictEqual(
+        ollama.attempts.length,
+        2,
+        'exactly one retry after the failed attempt'
+      );
+      assert.strictEqual(
+        analysis.error,
+        undefined,
+        'the retry succeeded and the answer is used'
+      );
+      assert.deepStrictEqual(analysis.document.tags, ['a']);
+    });
+
+    await test('A persistent 5xx fails with the upstream detail kept', async () => {
+      const ollama = ollamWithResponses([
+        serverError({ error: UPSTREAM_ERROR }),
+        serverError({ error: UPSTREAM_ERROR }),
+      ]);
+      ollamaService.client = ollama;
+
+      const analysis = await ollamaService.analyzeDocument(
+        'short content',
+        [],
+        [],
+        [],
+        372
+      );
+
+      assert.strictEqual(ollama.attempts.length, 2, 'one retry, then give up');
+      assert.match(analysis.error, /v1\/completions.*EOF/s);
+      assert.strictEqual(analysis.errorCode, 'ERR_BAD_RESPONSE');
+    });
+
+    await test('Client errors (4xx) are not retried', async () => {
+      const notFound = new Error('Request failed with status code 404');
+      notFound.code = 'ERR_BAD_REQUEST';
+      notFound.response = { status: 404, data: { error: 'model not found' } };
+      const ollama = ollamWithResponses([notFound]);
+      ollamaService.client = ollama;
+
+      const analysis = await ollamaService.analyzeDocument(
+        'short content',
+        [],
+        [],
+        [],
+        373
+      );
+
+      assert.strictEqual(
+        ollama.attempts.length,
+        1,
+        'a 404 will never succeed on retry'
+      );
+      assert.match(analysis.error, /model not found/);
+    });
+  } finally {
+    global.setTimeout = originalSetTimeout;
+    ollamaService.client = originalClient;
+  }
+
+  console.log(`\n${passed} passed, ${failed} failed`);
+  if (failed > 0) {
+    process.exitCode = 1;
+  }
+})();
diff --git a/tests/test-predefined-retrigger.js b/tests/test-predefined-retrigger.js
index 67f888cd..dd01ef9d 100644
--- a/tests/test-predefined-retrigger.js
+++ b/tests/test-predefined-retrigger.js
@@ -29,9 +29,9 @@ const path = require('path');
 let passed = 0;
 let failed = 0;
 
-function test(name, fn) {
+async function test(name, fn) {
   try {
-    fn();
+    await fn();
     console.log(`✅  ${name}`);
     passed++;
   } catch (error) {
@@ -108,30 +108,27 @@ test('The re-trigger respects ignored and permanently-failed documents', () => {
   };
 
   try {
-    await test(
-      'updateDocument removes configured tag ids from the merged payload',
-      async () => {
-        await paperlessService.updateDocument(315, {
-          title: 'New title',
-          tags: [5], // AI-chosen tags
-          removeTagIds: [42], // the predefined-mode trigger tag
-        });
+    await test('updateDocument removes configured tag ids from the merged payload', async () => {
+      await paperlessService.updateDocument(315, {
+        title: 'New title',
+        tags: [5], // AI-chosen tags
+        removeTagIds: [42], // the predefined-mode trigger tag
+      });
 
-        const updateCall = calls.find((call) => call.method === 'patch');
-        assert.ok(updateCall, 'an update request was sent');
-        const sentTags = updateCall.body.tags;
-        assert.deepStrictEqual(
-          [...sentTags].sort((a, b) => a - b),
-          [5, 11],
-          'AI tags merge with existing tags, the trigger tag is gone'
-        );
-        assert.strictEqual(
-          sentTags.includes(42),
-          false,
-          'the trigger tag must not survive the write-back'
-        );
-      }
-    );
+      const updateCall = calls.find((call) => call.method === 'patch');
+      assert.ok(updateCall, 'an update request was sent');
+      const sentTags = updateCall.body.tags;
+      assert.deepStrictEqual(
+        [...sentTags].sort((a, b) => a - b),
+        [5, 11],
+        'AI tags merge with existing tags, the trigger tag is gone'
+      );
+      assert.strictEqual(
+        sentTags.includes(42),
+        false,
+        'the trigger tag must not survive the write-back'
+      );
+    });
   } finally {
     paperlessService.client = originalClient;
   }

From 545d740bb6e40da079506139ae3ed08e46bac780 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Wed, 26 Aug 2026 18:01:45 +0200
Subject: [PATCH 15/53] disable debug output into logs

---
 services/CachedNameIdEnum.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/services/CachedNameIdEnum.js b/services/CachedNameIdEnum.js
index e9e12fe9..f6dbe503 100644
--- a/services/CachedNameIdEnum.js
+++ b/services/CachedNameIdEnum.js
@@ -65,7 +65,7 @@ class CachedNameIdEnum {
       if (response.data.next) {
         nextUrl = safeExtractRelativePath(response.data.next, client);
         if (nextUrl) {
-          console.log(
+          console.debug(
             `[DEBUG] Next page URL: ${nextUrl} elements fetched so far: ${existing_elements.length}`
           );
         } else {
@@ -75,10 +75,10 @@ class CachedNameIdEnum {
       } else {
         nextUrl = null;
         this.cache_last_fetch_time = Date.now();
-        console.log(
+        console.debug(
           `[DEBUG] Fetched ${existing_elements.length} ${this.element_type_name}s`
         );
-        console.log(
+        console.debug(
           `[DEBUG] Cache contains these ${this.element_type_name}s now: 
           ${existing_elements.map((element) => element.name).join(', ')}`
         );
@@ -105,7 +105,7 @@ class CachedNameIdEnum {
       const expireTime = new Date(
         this.cache_last_fetch_time + cache_ttl
       ).toISOString();
-      console.log(
+      console.debug(
         `[DEBUG] ${this.element_type_name} cache expired (age: ${ageSeconds}s, TTL: ${ttlSeconds}s, expired at: ${expireTime})`
       );
     }
@@ -117,19 +117,19 @@ class CachedNameIdEnum {
     await this._check_cache_is_not_outdated(client, cache_ttl);
     let match = this.elements_by_name.get(name.toLowerCase());
     if (match) {
-      console.log(
+      console.debug(
         `[DEBUG] Found exact match for ${this.element_type_name} "${match.name}" with ID ${match.id}`
       );
       return this._result_element_from_element(match);
     }
     match = await this._findSimilar(name);
     if (match) {
-      console.log(
+      console.debug(
         `[DEBUG] Found similar match for ${this.element_type_name} "${name}": "${match.name}" with ID ${match.id}`
       );
       return this._result_element_from_element(match);
     }
-    console.log(
+    console.debug(
       `[DEBUG] No exact nor similar match found for ${this.element_type_name} "${name}"`
     );
     return null;

From 9095c828a18fdbf235682b009c7265c48bbd7577 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Wed, 26 Aug 2026 19:05:12 +0200
Subject: [PATCH 16/53] Bugfix: Void function return value was used as promise.

---
 routes/setup.js | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/routes/setup.js b/routes/setup.js
index 2aa7bcae..d224d38a 100644
--- a/routes/setup.js
+++ b/routes/setup.js
@@ -4358,23 +4358,21 @@ router.get('/setup', async (req, res) => {
       console.warn(
         '[SECURITY] Attempting to access /setup in degraded state (corrupted database)'
       );
-      return res
-        .status(500)
-        .render('setup-error', {
+      try {
+        return res.status(500).render('setup-error', {
           title: 'System Configuration Error',
           errorMessage:
             'The system configuration exists but the database is inaccessible or corrupted. This is an administrative error state. Please check system logs and database integrity.',
           supportText:
             'This may occur if: (1) the database file was deleted or corrupted, (2) file permissions changed, or (3) the database is locked. Restart the application after verifying database and permissions.',
-        })
-        .catch(() => {
-          // Fallback if setup-error template doesn't exist
-          res
-            .status(500)
-            .send(
-              '<h1>System Configuration Error</h1><p>Database is inaccessible. Please contact your administrator.</p>'
-            );
         });
+      } catch {
+        return res
+          .status(500)
+          .send(
+            '<h1>System Configuration Error</h1><p>Database is inaccessible. Please contact your administrator.</p>'
+          );
+      }
     }
 
     // Base configuration object - load this FIRST, before any checks

From 614b9b7312644df10ae6066069a0498885802033 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 30 Aug 2026 19:12:25 +0200
Subject: [PATCH 17/53] upstream merge and refactorings

---
 services/dataProcessingUtils.js |  15 +-
 services/manualService.js       | 506 +++++++++++++++++---------------
 services/ollamaService.js       |  74 ++---
 services/serviceUtils.js        |   4 +-
 4 files changed, 317 insertions(+), 282 deletions(-)

diff --git a/services/dataProcessingUtils.js b/services/dataProcessingUtils.js
index 0b9f02d2..92ea87c9 100644
--- a/services/dataProcessingUtils.js
+++ b/services/dataProcessingUtils.js
@@ -1,4 +1,5 @@
 const { validateCustomFieldValue } = require('./serviceUtils');
+const os = require('node:os');
 
 async function updateCustomFieldsData(analysis, doc, updateData, config, paperlessService) {
   // Only process custom fields if custom fields detection is activated
@@ -116,8 +117,20 @@ async function updateCorrespondentData(analysis, updateData, config, paperlessSe
   }
 }
 
+/**
+ * @return {Object} An object containing totalMemoryMB and freeMemoryMB as
+ * strings representing memory in megabytes.
+ */
+async function getAvailableMemory() {
+  const totalMemory = os.totalmem();
+  const freeMemory = os.freemem();
+  const totalMemoryMB = (totalMemory / (1024 * 1024)).toFixed(0);
+  const freeMemoryMB = (freeMemory / (1024 * 1024)).toFixed(0);
+  return { totalMemoryMB, freeMemoryMB };
+}
 module.exports = {
   updateCustomFieldsData,
   updateDocumentTypeData,
-  updateCorrespondentData
+  updateCorrespondentData,
+  getAvailableMemory
 };
diff --git a/services/manualService.js b/services/manualService.js
index 4486d9d7..cd6f893c 100644
--- a/services/manualService.js
+++ b/services/manualService.js
@@ -1,266 +1,284 @@
-const { 
-    calculateTokens, 
-    calculateTotalPromptTokens, 
-    truncateToTokenLimit, 
-    writePromptToFile,
-    extractChatMessageContent
+const {
+  writePromptToFile,
+  extractChatMessageContent,
 } = require('./serviceUtils');
 const axios = require('axios');
 const OpenAI = require('openai');
 const config = require('../config/config');
 const AzureOpenAI = require('openai').AzureOpenAI;
-const emptyVar = null;
 
 class ManualService {
-    constructor() {
-        if(config.aiProvider === 'custom'){
-            this.openai = new OpenAI({
-                apiKey: config.custom.apiKey,
-                baseUrl: config.custom.apiUrl
-            });
-        }else if (config.aiProvider === 'azure'){
-            this.openai = new AzureOpenAI({
-                    apiKey: config.azure.apiKey,
-                    endpoint: config.azure.endpoint,
-                    deploymentName: config.azure.deploymentName,
-                    apiVersion: config.azure.apiVersion
-                  });
-        } else {            
-            this.openai = new OpenAI({ apiKey: config.openai.apiKey });
-            this.ollama = axios.create({
-            timeout: 300000
-            });
-        }
+  constructor() {
+    if (config.aiProvider === 'custom') {
+      this.openai = new OpenAI({
+        apiKey: config.custom.apiKey,
+        baseUrl: config.custom.apiUrl,
+      });
+    } else if (config.aiProvider === 'azure') {
+      this.openai = new AzureOpenAI({
+        apiKey: config.azure.apiKey,
+        endpoint: config.azure.endpoint,
+        deploymentName: config.azure.deploymentName,
+        apiVersion: config.azure.apiVersion,
+      });
+    } else {
+      this.openai = new OpenAI({ apiKey: config.openai.apiKey });
+      this.ollama = axios.create({
+        timeout: 300000,
+      });
     }
+  }
 
-    
-    async analyzeDocument(content, existingTags, provider) {
-        try {
-        if (provider === 'openai') {
-            return this._analyzeOpenAI(content, existingTags);
-        } else if (provider === 'ollama') {
-            return this._analyzeOllama(content, existingTags);
-        } else if (provider === 'custom') {
-            return this._analyzeCustom(content, existingTags);
-        } else if (provider === 'azure') {
-            return this._analyzeAzure(content, existingTags);
-        } else {            
-            throw new Error('Invalid provider');
-        }
-        } catch (error) {
-        console.error('Error analyzing document:', error);
-        return { tags: [], correspondent: null };
-        }
+  async analyzeDocument(content, existingTags, provider) {
+    try {
+      if (provider === 'openai') {
+        return this._analyzeOpenAI(content, existingTags);
+      } else if (provider === 'ollama') {
+        return this._analyzeOllama(content, existingTags);
+      } else if (provider === 'custom') {
+        return this._analyzeCustom(content, existingTags);
+      } else if (provider === 'azure') {
+        return this._analyzeAzure(content, existingTags);
+      }
+      console.error('Error analyzing document: Invalid provider');
+    } catch (error) {
+      console.error('Error analyzing document:', error);
     }
-    
-    async _analyzeOpenAI(content, existingTags) {
-        try {
-        const existingTagsList = existingTags
-            .map(tag => tag.name)
-            .join(', ');
-        const model = process.env.OPENAI_MODEL;
-        const systemPrompt = process.env.SYSTEM_PROMPT;
-        await writePromptToFile(systemPrompt, content);
-        const response = await this.openai.chat.completions.create({
-            model: model,
-            messages: [
-            {
-                role: "system",
-                content: systemPrompt
-            },
-            {
-                role: "user",
-                content: content
-            }
-            ],
-            ...(model !== 'o3-mini' && { temperature: config.aiTemperatureAnalysis }),
-        });
-    
-        let jsonContent = extractChatMessageContent(response?.choices?.[0]?.message, 'ManualService/OpenAI');
-        if (!jsonContent) {
-            throw new Error('Invalid API response structure');
-        }
-        jsonContent = jsonContent.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
-        
-        let parsedResponse;
-        try {
-            parsedResponse = JSON.parse(jsonContent);
-        } catch (error) {
-            console.error('Failed to parse JSON response:', error);
-            throw new Error('Invalid JSON response from API');
-        }
-        
-        if (!Array.isArray(parsedResponse.tags) || (typeof parsedResponse.correspondent !== 'string' && parsedResponse.correspondent !== null)) {
-            throw new Error('Invalid response structure');
-        }
-        
-        return parsedResponse;
-        } catch (error) {
-        console.error('Failed to analyze document with OpenAI:', error);
-        return { tags: [], correspondent: null };
-        }
+    return { tags: [], correspondent: null };
+  }
+
+  async _analyzeOpenAI(content, existingTags) {
+    try {
+      const existingTagsList = existingTags.map((tag) => tag.name).join(', ');
+      const model = process.env.OPENAI_MODEL;
+      const systemPrompt = process.env.SYSTEM_PROMPT;
+      await writePromptToFile(systemPrompt, content);
+      const response = await this.openai.chat.completions.create({
+        model: model,
+        messages: [
+          {
+            role: 'system',
+            content: systemPrompt,
+          },
+          {
+            role: 'user',
+            content: content,
+          },
+        ],
+        ...(model !== 'o3-mini' && {
+          temperature: config.aiTemperatureAnalysis,
+        }),
+      });
+
+      let jsonContent = extractChatMessageContent(
+        response?.choices?.[0]?.message,
+        'ManualService/OpenAI'
+      );
+      if (!jsonContent) {
+        throw new Error('Invalid API response structure');
+      }
+      jsonContent = jsonContent
+        .replace(/```json\n?/g, '')
+        .replace(/```\n?/g, '')
+        .trim();
+
+      let parsedResponse;
+      try {
+        parsedResponse = JSON.parse(jsonContent);
+      } catch (error) {
+        console.error('Failed to parse JSON response:', error);
+        throw new Error('Invalid JSON response from API');
+      }
+
+      if (
+        !Array.isArray(parsedResponse.tags) ||
+        (typeof parsedResponse.correspondent !== 'string' &&
+          parsedResponse.correspondent !== null)
+      ) {
+        throw new Error('Invalid response structure');
+      }
+
+      return parsedResponse;
+    } catch (error) {
+      console.error('Failed to analyze document with OpenAI:', error);
+      return { tags: [], correspondent: null };
     }
+  }
+
+  async _analyzeAzure(content, existingTags) {
+    try {
+      const existingTagsList = existingTags.map((tag) => tag.name).join(', ');
+
+      const systemPrompt = process.env.SYSTEM_PROMPT;
+      await writePromptToFile(systemPrompt, content);
+      const response = await this.openai.chat.completions.create({
+        model: process.env.AZURE_DEPLOYMENT_NAME,
+        messages: [
+          {
+            role: 'system',
+            content: systemPrompt,
+          },
+          {
+            role: 'user',
+            content: content,
+          },
+        ],
+        temperature: config.aiTemperatureAnalysis,
+      });
 
-    async _analyzeAzure(content, existingTags) {
-        try {
-        const existingTagsList = existingTags
-            .map(tag => tag.name)
-            .join(', ');
-    
-        const systemPrompt = process.env.SYSTEM_PROMPT;
-        await writePromptToFile(systemPrompt, content);
-        const response = await this.openai.chat.completions.create({
-            model: process.env.AZURE_DEPLOYMENT_NAME,
-            messages: [
-            {
-                role: "system",
-                content: systemPrompt
-            },
-            {
-                role: "user",
-                content: content
-            }
-            ],
-            temperature: config.aiTemperatureAnalysis,
-        });
-    
-        let jsonContent = extractChatMessageContent(response?.choices?.[0]?.message, 'ManualService/Azure');
-        if (!jsonContent) {
-            throw new Error('Invalid API response structure');
-        }
-        jsonContent = jsonContent.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
-        
-        let parsedResponse;
-        try {
-            parsedResponse = JSON.parse(jsonContent);
-        } catch (error) {
-            console.error('Failed to parse JSON response:', error);
-            throw new Error('Invalid JSON response from API');
-        }
-        
-        if (!Array.isArray(parsedResponse.tags) || (typeof parsedResponse.correspondent !== 'string' && parsedResponse.correspondent !== null)) {
-            throw new Error('Invalid response structure');
-        }
-        
-        return parsedResponse;
-        } catch (error) {
-        console.error('Failed to analyze document with OpenAI:', error);
-        return { tags: [], correspondent: null };
-        }
+      let jsonContent = extractChatMessageContent(
+        response?.choices?.[0]?.message,
+        'ManualService/Azure'
+      );
+      if (!jsonContent) {
+        throw new Error('Invalid API response structure');
+      }
+      jsonContent = jsonContent
+        .replace(/```json\n?/g, '')
+        .replace(/```\n?/g, '')
+        .trim();
+
+      let parsedResponse;
+      try {
+        parsedResponse = JSON.parse(jsonContent);
+      } catch (error) {
+        console.error('Failed to parse JSON response:', error);
+        throw new Error('Invalid JSON response from API.');
+      }
+
+      if (
+        !Array.isArray(parsedResponse.tags) ||
+        (typeof parsedResponse.correspondent !== 'string' &&
+          parsedResponse.correspondent !== null)
+      ) {
+        throw new Error('Invalid response structure');
+      }
+
+      return parsedResponse;
+    } catch (error) {
+      console.error('Failed to analyze document with OpenAI:', error);
+      return { tags: [], correspondent: null };
     }
+  }
+
+  async _analyzeCustom(content, existingTags) {
+    try {
+      const existingTagsList = existingTags.map((tag) => tag.name).join(', ');
+
+      const systemPrompt = process.env.SYSTEM_PROMPT;
+      const model = config.custom.model;
+      const response = await this.openai.chat.completions.create({
+        model: model,
+        messages: [
+          {
+            role: 'system',
+            content: systemPrompt,
+          },
+          {
+            role: 'user',
+            content: content,
+          },
+        ],
+        ...(model !== 'o3-mini' && {
+          temperature: config.aiTemperatureAnalysis,
+        }),
+      });
+
+      let jsonContent = extractChatMessageContent(
+        response?.choices?.[0]?.message,
+        'ManualService/Custom'
+      );
+      if (!jsonContent) {
+        throw new Error('Invalid API response structure');
+      }
+      jsonContent = jsonContent
+        .replace(/```json\n?/g, '')
+        .replace(/```\n?/g, '')
+        .trim();
+
+      const parsedResponse = JSON.parse(jsonContent);
 
-    async _analyzeCustom(content, existingTags) {
-        try {
-            const existingTagsList = existingTags
-                .map(tag => tag.name)
-                .join(', ');
-        
-            const systemPrompt = process.env.SYSTEM_PROMPT;
-            const model = config.custom.model;
-            const response = await this.openai.chat.completions.create({
-                model: model,
-                messages: [
-                {
-                    role: "system",
-                    content: systemPrompt
-                },
-                {
-                    role: "user",
-                    content: content
-                }
-                ],
-                ...(model !== 'o3-mini' && { temperature: config.aiTemperatureAnalysis }),
-            });
-        
-            let jsonContent = extractChatMessageContent(response?.choices?.[0]?.message, 'ManualService/Custom');
-            if (!jsonContent) {
-                throw new Error('Invalid API response structure');
-            }
-            jsonContent = jsonContent.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
-            
-            const parsedResponse = JSON.parse(jsonContent);
-            
-            if (!Array.isArray(parsedResponse.tags) || (typeof parsedResponse.correspondent !== 'string' && parsedResponse.correspondent !== null)) {
-                throw new Error('Invalid response structure');
-            }
-            
-            return parsedResponse;
-            } catch (error) {
-            console.error('Failed to analyze document with OpenAI:', error);
-            return { tags: [], correspondent: null };
-            }
+      if (
+        !Array.isArray(parsedResponse.tags) ||
+        (typeof parsedResponse.correspondent !== 'string' &&
+          parsedResponse.correspondent !== null)
+      ) {
+        throw new Error('Invalid response structure');
+      }
+
+      return parsedResponse;
+    } catch (error) {
+      console.error('Failed to analyze document with OpenAI:', error);
+      return { tags: [], correspondent: null };
     }
-    
-    async _analyzeOllama(content, existingTags) {
-        try {
-        const prompt = process.env.SYSTEM_PROMPT;
-
-        const getAvailableMemory = async () => {
-            const totalMemory = os.totalmem();
-            const freeMemory = os.freemem();
-            const totalMemoryMB = (totalMemory / (1024 * 1024)).toFixed(0);
-            const freeMemoryMB = (freeMemory / (1024 * 1024)).toFixed(0);
-            return { totalMemoryMB, freeMemoryMB };
-        };
-        
-        const calculateNumCtx = (promptTokenCount, expectedResponseTokens) => {
-            const totalTokenUsage = promptTokenCount + expectedResponseTokens;
-            const maxCtxLimit = Number(config.tokenLimit);
-            
-            const numCtx = Math.min(totalTokenUsage, maxCtxLimit);
-            
-            console.log('Prompt Token Count:', promptTokenCount);
-            console.log('Expected Response Tokens:', expectedResponseTokens);
-            console.log('Dynamic calculated num_ctx:', numCtx);
-            
-            return numCtx;
-        };
-        
-        const calculatePromptTokenCount = (prompt) => {
-            return Math.ceil(prompt.length / 4);
-        };
-        
-        const { freeMemoryMB } = await getAvailableMemory();
-        const expectedResponseTokens = 1024;
-        const promptTokenCount = calculatePromptTokenCount(prompt);
-        
-        const numCtx = calculateNumCtx(promptTokenCount, expectedResponseTokens);
-        
-        const ollamaHeaders = { 'Content-Type': 'application/json' };
-        const ollamaApiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || '';
-        if (ollamaApiKey) {
-            ollamaHeaders.Authorization = `Bearer ${ollamaApiKey}`;
-        }
-
-        const response = await this.ollama.post(`${config.ollama.apiUrl}/api/generate`, {
-            model: config.ollama.model,
-            prompt: prompt,
-            stream: false,
-            options: {
+  }
+
+  async _analyzeOllama(content, existingTags) {
+    try {
+      const prompt = process.env.SYSTEM_PROMPT;
+
+      const calculateNumCtx = (promptTokenCount, expectedResponseTokens) => {
+        const totalTokenUsage = promptTokenCount + expectedResponseTokens;
+        const maxCtxLimit = Number(config.tokenLimit);
+
+        const numCtx = Math.min(totalTokenUsage, maxCtxLimit);
+
+        console.log('Prompt Token Count:', promptTokenCount);
+        console.log('Expected Response Tokens:', expectedResponseTokens);
+        console.log('Dynamic calculated num_ctx:', numCtx);
+
+        return numCtx;
+      };
+
+      const calculatePromptTokenCount = (prompt) => {
+        return Math.ceil(prompt.length / 4);
+      };
+
+      const expectedResponseTokens = 1024;
+      const promptTokenCount = calculatePromptTokenCount(prompt);
+
+      const numCtx = calculateNumCtx(promptTokenCount, expectedResponseTokens);
+
+      const ollamaHeaders = { 'Content-Type': 'application/json' };
+      const ollamaApiKey =
+        process.env.OLLAMA_API_KEY || config.ollama.apiKey || '';
+      if (ollamaApiKey) {
+        ollamaHeaders.Authorization = `Bearer ${ollamaApiKey}`;
+      }
+
+      const response = await this.ollama.post(
+        `${config.ollama.apiUrl}/api/generate`,
+        {
+          model: config.ollama.model,
+          prompt: prompt,
+          stream: false,
+          options: {
             temperature: config.aiTemperatureGeneration,
             top_p: 0.9,
             repeat_penalty: 1.1,
             num_ctx: numCtx,
-            }
-        }, { headers: ollamaHeaders });
-    
-        if (!response.data || !response.data.response) {
-            console.error('Unexpected Ollama response format:', response);
-            throw new Error('Invalid response from Ollama API');
-        }
-
-        return this._parseResponse(response.data.response);
-        }
-
-        catch (error) {
-        if (error.code === 'ECONNABORTED') {
-            console.error('Timeout bei der Ollama-Anfrage:', error);
-            throw new Error('Die Analyse hat zu lange gedauert. Bitte versuchen Sie es erneut.');
-        }
-        console.error('Error analyzing document with Ollama:', error);
-        throw error;
-        }
+          },
+        },
+        { headers: ollamaHeaders }
+      );
+
+      if (!response.data || !response.data.response) {
+        console.error('Unexpected Ollama response format:', response);
+        throw new Error('Invalid response from Ollama API');
+      }
+
+      return this._parseResponse(response.data.response);
+    } catch (error) {
+      if (error.code === 'ECONNABORTED') {
+        console.error('Timeout bei der Ollama-Anfrage:', error);
+        throw new Error(
+          'Die Analyse hat zu lange gedauert. Bitte versuchen Sie es erneut.'
+        );
+      }
+      console.error('Error analyzing document with Ollama:', error);
+      throw error;
     }
+  }
 }
 
-module.exports = ManualService; 
+module.exports = ManualService;
diff --git a/services/ollamaService.js b/services/ollamaService.js
index 9b2225a1..72b4b2d6 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -82,13 +82,21 @@ class OllamaService {
   }
 
   /**
-   * Analyze a document and extract metadata
-   * @param {string} content - Document content
-   * @param {Array} existingTags - List of existing tags
-   * @param {Array} existingCorrespondentList - List of existing correspondents
-   * @param {string} id - Document ID
-   * @param {string} customPrompt - Custom prompt (optional)
-   * @returns {Object} Analysis results
+   * Analyzes a document to extract metadata and tags using an AI model.
+   * @param {string} content - The content of the document to analyze.
+   * @param {Array} existingTags - Array of pre-existing tags to consider during analysis.
+   * @param {Array} existingCorrespondentList - Array of pre-existing correspondents.
+   * @param {Array} existingDocumentTypesList - Array of pre-existing document types.
+   * @param {string|number} id - Unique identifier for the document.
+   * @param {string} [customPrompt] - Custom prompt to guide the analysis process.
+   * @param {Object} [options] - Additional configuration options.
+   * @param {Object} [options.externalApiData] - External data to include in the analysis.
+   * @returns {Promise<Object>} An object containing analysis results, metrics, and status.
+   * @returns {Object} document - Parsed response from the AI analysis.
+   * @returns {Object} metrics - Performance metrics from the analysis process.
+   * @returns {boolean} truncated - Indicates if the content was truncated during processing.
+   * @returns {string} error - Error message if an exception occurred.
+   * @returns {string} errorCode - Error code if an exception occurred.
    */
   async analyzeDocument(
     content,
@@ -113,7 +121,7 @@ class OllamaService {
       if (externalApiData) {
         try {
           validatedExternalApiData =
-            await this._validateAndTruncateExternalApiData(externalApiData);
+            this._validateAndTruncateExternalApiData(externalApiData);
           console.log('[DEBUG] External API data validated and included');
         } catch (error) {
           console.warn(
@@ -239,7 +247,7 @@ class OllamaService {
       }
 
       // Log the prompt and response
-      await this._logPromptAndResponse(prompt, parsedResponse);
+      await this._logPromptAndResponse(systemPrompt, prompt, parsedResponse);
 
       // Return results in consistent format
       return {
@@ -382,7 +390,8 @@ class OllamaService {
 
     if (availableTokens <= 0) {
       throw new Error(
-        'Token limit exceeded: the prompt alone does not fit the configured TOKEN_LIMIT. Raise TOKEN_LIMIT or shorten the system prompt.'
+        'Token limit exceeded: the prompt alone does not fit the configured ' +
+          'TOKEN_LIMIT. Raise TOKEN_LIMIT or shorten the system prompt.'
       );
     }
 
@@ -392,18 +401,21 @@ class OllamaService {
     }
 
     console.log(
-      `[DEBUG] Document content (${text.length} chars) exceeds the context budget (${maxChars} chars for a ${maxWindow}-token window minus prompt and ${responseTokens} reserved answer tokens); truncating`
+      '[DEBUG] Document content (${text.length} chars) exceeds the context ' +
+        'budget (${maxChars} chars for a ${maxWindow}-token window minus prompt' +
+        ' and ${responseTokens} reserved answer tokens); truncating'
     );
     return text.substring(0, maxChars);
   }
 
   /**
-   * Build prompt from content and existing data
-   * @param {string} content - Document content
-   * @param {Array} existingTags - List of existing tags
-   * @param {Array} existingCorrespondent - List of existing correspondents
-   * @param {Array} existingDocumentTypes - List of existing document types
-   * @returns {string} Formatted prompt
+   * Builds and returns a system prompt string combining configuration, content, and metadata.
+   * @param {string} content - The main content of the document to be processed.
+   * @param {string[]} existingTags - Array of existing tags associated with the document.
+   * @param {string[]} existingCorrespondent - Array of existing correspondents related to the document.
+   * @param {string[]} existingDocumentTypes - Array of existing document types.
+   * @param {Object} options - Optional configuration object. May include externalApiData for additional context.
+   * @return {string} The constructed system prompt string combining configuration, content, and metadata.
    */
   _buildPrompt(
     content,
@@ -689,18 +701,6 @@ class OllamaService {
     return numCtx;
   }
 
-  /**
-   * Get available system memory
-   * @returns {Object} Object with totalMemoryMB and freeMemoryMB
-   */
-  async _getAvailableMemory() {
-    const totalMemory = os.totalmem();
-    const freeMemory = os.freemem();
-    const totalMemoryMB = (totalMemory / (1024 * 1024)).toFixed(0);
-    const freeMemoryMB = (freeMemory / (1024 * 1024)).toFixed(0);
-    return { totalMemoryMB, freeMemoryMB };
-  }
-
   /**
    * Handle thumbnail caching for documents
    * @param {string} id - Document ID
@@ -726,7 +726,7 @@ class OllamaService {
 
   /**
    * Build request headers, adding bearer auth when an API key is set.
-   * The key is read at request time so runtime config changes apply
+   * The key is read at request time, so runtime config changes apply
    * without re-instantiating the singleton.
    * @returns {Object} Headers object
    */
@@ -1053,11 +1053,15 @@ class OllamaService {
   }
 
   /**
-   * Log prompt and response to file
-   * @param {string} prompt - Prompt text
-   * @param {Object} response - Response object
+   * Logs the prompt and response to a file. Combines the system prompt, user
+   * prompt, and response into a formatted string and writes it to a file.
+   * @param {string} systemPrompt - The system message provided to the model.
+   * @param {string} prompt - The user's input prompt.
+   * @param {Object} response - The model's generated response object.
+   * @return {Promise<void>} A promise that resolves when the prompt and
+   *          response are written to the file.
    */
-  async _logPromptAndResponse(prompt, response) {
+  async _logPromptAndResponse(systemPrompt, prompt, response) {
     const content =
       '================================================================================' +
       prompt +
@@ -1066,7 +1070,7 @@ class OllamaService {
       '\n\n' +
       '================================================================================\n\n';
 
-    await writePromptToFile(content);
+    await writePromptToFile(systemPrompt, content);
   }
 
   /**
diff --git a/services/serviceUtils.js b/services/serviceUtils.js
index f9ad2dea..bd12e43e 100644
--- a/services/serviceUtils.js
+++ b/services/serviceUtils.js
@@ -299,7 +299,7 @@ const METADATA_ENDPOINTS = [
 ];
 
 function stripIpv6Brackets(value) {
-  return String(value || '').replace(/^\[|\]$/g, '');
+  return String(value || '').replace(/^\[|]$/g, '');
 }
 
 function normalizeIpAddress(ipAddress) {
@@ -935,7 +935,7 @@ function extractChatMessageContent(
     return '';
   }
 
-  const jsonMatch = reasoningContent.match(/\{[\s\S]*\}/);
+  const jsonMatch = reasoningContent.match(/\{[\s\S]*}/);
   if (jsonMatch) {
     console.warn(
       `[WARN] [${providerLabel}] Empty message.content, using JSON extracted from reasoning_content.`

From 238dc814711cff8a0ce4d3f2abea20a4d63a1a80 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 31 Aug 2026 10:42:12 +0200
Subject: [PATCH 18/53] initial

---
 services/ManualServiceRefactoring.md | 71 ++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 services/ManualServiceRefactoring.md

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
new file mode 100644
index 00000000..bcbdd55e
--- /dev/null
+++ b/services/ManualServiceRefactoring.md
@@ -0,0 +1,71 @@
+# ManualService Refactoring Plan
+## The Goal
+
+Refactor manualService.js. It has four functions, that are offering the same business
+service using different AI APIs. We aim at implementing the business logic only once 
+and coping with the API specific stuff In different Adapters.
+
+We will also use these adapters to refactor ollamaService.js, openaiService.js, customService.js, 
+and azureService.js.
+
+Apply TDD
+
+## The Plan
+
+### 1. **Base Analyzer Pattern**
+- Extract common logic to `BaseAnalyzer` class
+- Handle prompt formatting, response parsing, and error handling
+- Abstract API call interface for adapters
+
+### 2. **Provider Adapters**
+- Create separate adapter classes for each AI provider:
+  - `OpenAIAdapter`
+  - `OllamaAdapter`
+  - `AzureAdapter`
+  - `CustomAdapter`
+- Implement provider-specific API call logic
+
+### 3. **ManualService Refactor**
+- Replace switch statement with adapter factory pattern
+- Inject provider-specific configuration into adapters
+- Centralize error handling and response validation
+
+### 4. **Test Coverage Strategy**
+- Mock each provider's API calls using `jest`/`sinon`
+- Test scenarios:
+  - Valid API responses
+  - Malformed JSON responses
+  - Network errors (e.g., `ECONNABORTED`)
+  - Invalid configuration values
+  - Empty/Null responses
+- Validate token calculation logic for Ollama
+
+### 5. **Edge Case Coverage**
+- Test:
+  - Empty `existingTags` array
+  - Missing environment variables
+  - Failed API authentication
+  - API rate limiting
+  - Large input content size
+
+### 6. **Implementation Steps**
+1. Create `BaseAnalyzer` with abstract `_callAPI` method
+2. Implement provider-specific adapters
+3. Update `ManualService` to use adapter factory
+4. Add mock implementations for each provider
+5. Write comprehensive tests for all edge cases
+6. Validate token calculation logic in Ollama adapter
+
+### 7. **Expected Benefits**
+- Single point of business logic
+- Easier maintenance and extension
+- Better test coverage
+- Improved error handling
+- Reduced code duplication
+
+### 8. **Next Steps**
+- Implement `BaseAnalyzer` class
+- Create adapter implementations
+- Write mock tests for each provider
+- Validate all edge cases
+

From df5bd930da6bd16261bb55d9e577a3b5545ef143 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 31 Aug 2026 10:53:58 +0200
Subject: [PATCH 19/53] sharpened the plan

---
 services/ManualServiceRefactoring.md | 135 ++++++++++++++++++++-------
 1 file changed, 102 insertions(+), 33 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index bcbdd55e..278a1d8b 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -14,58 +14,127 @@ Apply TDD
 
 ### 1. **Base Analyzer Pattern**
 - Extract common logic to `BaseAnalyzer` class
-- Handle prompt formatting, response parsing, and error handling
-- Abstract API call interface for adapters
+  - Example: Centralize error handling with `handleAPIError()`
+  - Example: Standardize response parsing with `parseJSONResponse()`
+  - Example: Abstract API call interface with `callAPI()`
 
 ### 2. **Provider Adapters**
 - Create separate adapter classes for each AI provider:
-  - `OpenAIAdapter`
-  - `OllamaAdapter`
-  - `AzureAdapter`
-  - `CustomAdapter`
-- Implement provider-specific API call logic
+  - `OpenAIAdapter`:
+    ```js
+    async callAPI(prompt) {
+      return this.openai.chat.completions.create({
+        model: 'gpt-3.5-turbo',
+        messages: [{ role: 'user', content: prompt }]
+      });
+    }
+    ```
+  - `OllamaAdapter`:
+    ```js
+    async callAPI(prompt) {
+      return this.ollama.post('/api/generate', {
+        model: 'llama2',
+        prompt
+      });
+    }
+    ```
+  - `AzureAdapter`:
+    ```js
+    async callAPI(prompt) {
+      return this.openai.chat.completions.create({
+        model: 'deployment-name',
+        messages: [{ role: 'user', content: prompt }]
+      });
+    }
+    ```
 
 ### 3. **ManualService Refactor**
-- Replace switch statement with adapter factory pattern
-- Inject provider-specific configuration into adapters
-- Centralize error handling and response validation
+- Replace switch statement with adapter factory pattern:
+  ```js
+  const adapter = new (require(`services/analyzers/${config.aiProvider}Adapter`))(config);
+  return await adapter.analyze(content, existingTags);
+  ```
+- Inject provider-specific configuration via constructor
+- Centralize error handling with `handleAPIError()`
 
 ### 4. **Test Coverage Strategy**
-- Mock each provider's API calls using `jest`/`sinon`
+- Mock API calls using `jest`/`sinon`:
+  - Example: Mock OpenAI API
+    ```js
+    jest.mock('openai', () => ({
+      chat: {
+        completions: {
+          create: jest.fn().mockResolvedValue({
+            choices: [{ message: { content: '{"tags":["tag1"],"correspondent":"John"}' }] }
+          })
+        }
+      }
+    }));
+    ```
 - Test scenarios:
-  - Valid API responses
-  - Malformed JSON responses
-  - Network errors (e.g., `ECONNABORTED`)
-  - Invalid configuration values
-  - Empty/Null responses
-- Validate token calculation logic for Ollama
+  - Valid JSON response: `tags` array and `correspondent` string
+  - Malformed JSON: `try/catch` validation
+  - Network error: `ECONNABORTED` handling
+  - Empty response: `null` check
 
 ### 5. **Edge Case Coverage**
-- Test:
-  - Empty `existingTags` array
-  - Missing environment variables
-  - Failed API authentication
-  - API rate limiting
-  - Large input content size
+- Test empty `existingTags` array:
+  ```js
+  test('handles empty existingTags', async () => {
+    const result = await service.analyze('content', []);
+    expect(result.tags).toEqual(['default-tag']);
+  });
+  ```
+- Test missing environment variables:
+  ```js
+  test('fails on missing SYSTEM_PROMPT', () => {
+    process.env.SYSTEM_PROMPT = undefined;
+    expect(() => service.analyze()).toThrow();
+  });
+  ```
+- Test API rate limiting: Validate `X-RateLimit-Remaining` header
+- Test large input content: Validate token calculation for Ollama
 
 ### 6. **Implementation Steps**
-1. Create `BaseAnalyzer` with abstract `_callAPI` method
-2. Implement provider-specific adapters
-3. Update `ManualService` to use adapter factory
-4. Add mock implementations for each provider
-5. Write comprehensive tests for all edge cases
-6. Validate token calculation logic in Ollama adapter
+1. Write tests for `BaseAnalyzer` first (TDD)
+2. Implement `BaseAnalyzer` with abstract `callAPI()`
+3. Create adapter classes with provider-specific `callAPI()`
+4. Mock API calls for each provider in tests
+5. Validate all edge cases with test scenarios
+6. Add Ollama token calculation tests:
+   ```js
+   test('calculates num_ctx correctly', () => {
+     expect(ollamaAdapter.calculateNumCtx(100, 1024)).toBe(1124);
+   });
+   ```
 
 ### 7. **Expected Benefits**
 - Single point of business logic
 - Easier maintenance and extension
-- Better test coverage
-- Improved error handling
-- Reduced code duplication
+- Better test coverage with mocks
+- Improved error handling with centralized `handleAPIError()`
+- Reduced code duplication across providers
 
 ### 8. **Next Steps**
-- Implement `BaseAnalyzer` class
+- Implement `BaseAnalyzer` with tests
 - Create adapter implementations
 - Write mock tests for each provider
 - Validate all edge cases
+- Add Ollama token calculation tests
 
+### 9. **Return Value Structure**
+All adapters must return objects with this structure:
+```ts
+interface AnalysisResult {
+  tags: string[]; // Array of tag names
+  correspondent: string | null; // Correspondent name or null
+}
+```
+Example:
+```js
+{ tags: ['tag1', 'tag2'], correspondent: 'John Doe' }
+```
+Error case:
+```js
+{ tags: [], correspondent: null }
+```
\ No newline at end of file

From 41741a2a4cc27ad7d40f2f517a171dd3e6c33c18 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 31 Aug 2026 10:59:51 +0200
Subject: [PATCH 20/53] restructured the plan

---
 services/ManualServiceRefactoring.md | 63 +++++++++++++---------------
 1 file changed, 30 insertions(+), 33 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 278a1d8b..a4d50db5 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -10,15 +10,26 @@ and azureService.js.
 
 Apply TDD
 
+
+### **Expected Benefits**
+- Single point of business logic
+- Easier maintenance and extension
+- Better test coverage with mocks
+- Improved error handling with centralized `handleAPIError()`
+- Reduced code duplication across providers
+
+
 ## The Plan
 
-### 1. **Base Analyzer Pattern**
+### design approach
+
+#### 1. **Base Analyzer Pattern**
 - Extract common logic to `BaseAnalyzer` class
   - Example: Centralize error handling with `handleAPIError()`
   - Example: Standardize response parsing with `parseJSONResponse()`
   - Example: Abstract API call interface with `callAPI()`
 
-### 2. **Provider Adapters**
+#### 2. **Provider Adapters**
 - Create separate adapter classes for each AI provider:
   - `OpenAIAdapter`:
     ```js
@@ -48,7 +59,7 @@ Apply TDD
     }
     ```
 
-### 3. **ManualService Refactor**
+#### 3. **ManualService Refactor**
 - Replace switch statement with adapter factory pattern:
   ```js
   const adapter = new (require(`services/analyzers/${config.aiProvider}Adapter`))(config);
@@ -57,7 +68,7 @@ Apply TDD
 - Inject provider-specific configuration via constructor
 - Centralize error handling with `handleAPIError()`
 
-### 4. **Test Coverage Strategy**
+#### 4. **Test Coverage Strategy**
 - Mock API calls using `jest`/`sinon`:
   - Example: Mock OpenAI API
     ```js
@@ -77,7 +88,7 @@ Apply TDD
   - Network error: `ECONNABORTED` handling
   - Empty response: `null` check
 
-### 5. **Edge Case Coverage**
+#### 5. **Edge Case Coverage**
 - Test empty `existingTags` array:
   ```js
   test('handles empty existingTags', async () => {
@@ -95,34 +106,8 @@ Apply TDD
 - Test API rate limiting: Validate `X-RateLimit-Remaining` header
 - Test large input content: Validate token calculation for Ollama
 
-### 6. **Implementation Steps**
-1. Write tests for `BaseAnalyzer` first (TDD)
-2. Implement `BaseAnalyzer` with abstract `callAPI()`
-3. Create adapter classes with provider-specific `callAPI()`
-4. Mock API calls for each provider in tests
-5. Validate all edge cases with test scenarios
-6. Add Ollama token calculation tests:
-   ```js
-   test('calculates num_ctx correctly', () => {
-     expect(ollamaAdapter.calculateNumCtx(100, 1024)).toBe(1124);
-   });
-   ```
 
-### 7. **Expected Benefits**
-- Single point of business logic
-- Easier maintenance and extension
-- Better test coverage with mocks
-- Improved error handling with centralized `handleAPIError()`
-- Reduced code duplication across providers
-
-### 8. **Next Steps**
-- Implement `BaseAnalyzer` with tests
-- Create adapter implementations
-- Write mock tests for each provider
-- Validate all edge cases
-- Add Ollama token calculation tests
-
-### 9. **Return Value Structure**
+#### 6. **Return Value Structure**
 All adapters must return objects with this structure:
 ```ts
 interface AnalysisResult {
@@ -137,4 +122,16 @@ Example:
 Error case:
 ```js
 { tags: [], correspondent: null }
-```
\ No newline at end of file
+```
+###  **Implementation Steps**
+1. Write tests for `BaseAnalyzer` first (TDD)
+2. Implement `BaseAnalyzer` with abstract `callAPI()`
+3. Create adapter classes with provider-specific `callAPI()`
+4. Mock API calls for each provider in tests
+5. Validate all edge cases with test scenarios
+6. Add Ollama token calculation tests:
+   ```js
+   test('calculates num_ctx correctly', () => {
+     expect(ollamaAdapter.calculateNumCtx(100, 1024)).toBe(1124);
+   });
+   ```

From 30ee40990afe33cfa2a338ed34ab098188d989f4 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 31 Aug 2026 11:28:40 +0200
Subject: [PATCH 21/53] added Agent suggestions

---
 services/ManualServiceRefactoring.md | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index a4d50db5..9efff411 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -27,7 +27,7 @@ Apply TDD
 - Extract common logic to `BaseAnalyzer` class
   - Example: Centralize error handling with `handleAPIError()`
   - Example: Standardize response parsing with `parseJSONResponse()`
-  - Example: Abstract API call interface with `callAPI()`
+  - **Suggestion**: Add validation for `AnalysisResult` structure to reject unexpected keys (e.g., `correspondent` might be missing in some cases)
 
 #### 2. **Provider Adapters**
 - Create separate adapter classes for each AI provider:
@@ -58,6 +58,7 @@ Apply TDD
       });
     }
     ```
+  - Standardize method signatures (e.g. all adapters return text containing a valid JSON object)
 
 #### 3. **ManualService Refactor**
 - Replace switch statement with adapter factory pattern:
@@ -87,6 +88,7 @@ Apply TDD
   - Malformed JSON: `try/catch` validation
   - Network error: `ECONNABORTED` handling
   - Empty response: `null` check
+- **Suggestion**: Add mock for `X-RateLimit-Remaining` header in rate-limiting tests
 
 #### 5. **Edge Case Coverage**
 - Test empty `existingTags` array:
@@ -106,7 +108,6 @@ Apply TDD
 - Test API rate limiting: Validate `X-RateLimit-Remaining` header
 - Test large input content: Validate token calculation for Ollama
 
-
 #### 6. **Return Value Structure**
 All adapters must return objects with this structure:
 ```ts
@@ -123,15 +124,25 @@ Error case:
 ```js
 { tags: [], correspondent: null }
 ```
+
+### 7. **Token Calculation Centralization**
+**Suggestion**: Centralize Ollama token calculation logic in `calculateNumCtx()` (input prompt characters / 2 or /4) to avoid duplication. Example:
+```js
+function calculateNumCtx(promptLength, maxCtx) {
+  return Math.min(Math.floor(promptLength / 2), maxCtx);
+}
+```
+
 ###  **Implementation Steps**
 1. Write tests for `BaseAnalyzer` first (TDD)
 2. Implement `BaseAnalyzer` with abstract `callAPI()`
-3. Create adapter classes with provider-specific `callAPI()`
-4. Mock API calls for each provider in tests
-5. Validate all edge cases with test scenarios
-6. Add Ollama token calculation tests:
+3. Add tests for derived Adapters
+4. Add Ollama token calculation tests:
    ```js
    test('calculates num_ctx correctly', () => {
      expect(ollamaAdapter.calculateNumCtx(100, 1024)).toBe(1124);
    });
    ```
+5. Create adapter classes with provider-specific `callAPI()`
+6. Mock API calls for each provider in tests
+7. Validate all edge cases with test scenarios

From 47bdf28ecbd5f4abbe0639efdbbfe54bd9278795 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 31 Aug 2026 11:41:10 +0200
Subject: [PATCH 22/53] fix namings and orders

---
 services/ManualServiceRefactoring.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 9efff411..30ef91f4 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -23,11 +23,11 @@ Apply TDD
 
 ### design approach
 
-#### 1. **Base Analyzer Pattern**
-- Extract common logic to `BaseAnalyzer` class
-  - Example: Centralize error handling with `handleAPIError()`
-  - Example: Standardize response parsing with `parseJSONResponse()`
-  - **Suggestion**: Add validation for `AnalysisResult` structure to reject unexpected keys (e.g., `correspondent` might be missing in some cases)
+#### 1. **Base Adapter Pattern**
+- Extract common logic to `BaseAdapter` class
+- Centralize error handling with `handleAPIError()`
+- Standardize response parsing with `parseJSONResponse()`
+- **Suggestion**: Add validation for `AnalysisResult` structure to reject unexpected keys (e.g., `correspondent` might be missing in some cases)
 
 #### 2. **Provider Adapters**
 - Create separate adapter classes for each AI provider:
@@ -134,15 +134,15 @@ function calculateNumCtx(promptLength, maxCtx) {
 ```
 
 ###  **Implementation Steps**
-1. Write tests for `BaseAnalyzer` first (TDD)
-2. Implement `BaseAnalyzer` with abstract `callAPI()`
+1. Write tests for `BaseService` first (TDD)
+2. Implement `BaseService` with abstract `callAPI()`
 3. Add tests for derived Adapters
-4. Add Ollama token calculation tests:
+4. Mock API calls for each provider in tests
+5. Add Ollama token calculation tests:
    ```js
    test('calculates num_ctx correctly', () => {
      expect(ollamaAdapter.calculateNumCtx(100, 1024)).toBe(1124);
    });
    ```
-5. Create adapter classes with provider-specific `callAPI()`
-6. Mock API calls for each provider in tests
+6. Create adapter classes with provider-specific `callAPI()`
 7. Validate all edge cases with test scenarios

From 4142da63f7d0c09e801a31453d0027dc66a6c5da Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 31 Aug 2026 11:54:11 +0200
Subject: [PATCH 23/53] fix namings

---
 services/ManualServiceRefactoring.md | 30 +++++++++++++++-------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 30ef91f4..bdd27b1f 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -9,7 +9,8 @@ We will also use these adapters to refactor ollamaService.js, openaiService.js,
 and azureService.js.
 
 Apply TDD
-
+Adhere to the stylguide in /CLAUDE.md
+Ensure all tests are symmetrically to the existing tests and are integrated in the test suite.
 
 ### **Expected Benefits**
 - Single point of business logic
@@ -24,14 +25,14 @@ Apply TDD
 ### design approach
 
 #### 1. **Base Adapter Pattern**
-- Extract common logic to `BaseAdapter` class
+- Extract common logic to `LlmService` class
 - Centralize error handling with `handleAPIError()`
 - Standardize response parsing with `parseJSONResponse()`
 - **Suggestion**: Add validation for `AnalysisResult` structure to reject unexpected keys (e.g., `correspondent` might be missing in some cases)
 
-#### 2. **Provider Adapters**
-- Create separate adapter classes for each AI provider:
-  - `OpenAIAdapter`:
+#### 2. **Provider Services**
+- Create separate Service classes for each AI provider:
+  - `OpenAIService`:
     ```js
     async callAPI(prompt) {
       return this.openai.chat.completions.create({
@@ -40,7 +41,7 @@ Apply TDD
       });
     }
     ```
-  - `OllamaAdapter`:
+  - `OllamaService`:
     ```js
     async callAPI(prompt) {
       return this.ollama.post('/api/generate', {
@@ -49,7 +50,7 @@ Apply TDD
       });
     }
     ```
-  - `AzureAdapter`:
+  - `AzureService`:
     ```js
     async callAPI(prompt) {
       return this.openai.chat.completions.create({
@@ -63,8 +64,8 @@ Apply TDD
 #### 3. **ManualService Refactor**
 - Replace switch statement with adapter factory pattern:
   ```js
-  const adapter = new (require(`services/analyzers/${config.aiProvider}Adapter`))(config);
-  return await adapter.analyze(content, existingTags);
+  const service = new (require(`services/analyzers/${config.aiProvider}Service`))(config);
+  return await service.analyze(content, existingTags);
   ```
 - Inject provider-specific configuration via constructor
 - Centralize error handling with `handleAPIError()`
@@ -134,15 +135,16 @@ function calculateNumCtx(promptLength, maxCtx) {
 ```
 
 ###  **Implementation Steps**
-1. Write tests for `BaseService` first (TDD)
-2. Implement `BaseService` with abstract `callAPI()`
-3. Add tests for derived Adapters
+1. Write tests for `LlmService` first (TDD)
+2. Implement `LlmService` with abstract `callAPI()`
+3. Add tests for derived Services
 4. Mock API calls for each provider in tests
 5. Add Ollama token calculation tests:
    ```js
    test('calculates num_ctx correctly', () => {
-     expect(ollamaAdapter.calculateNumCtx(100, 1024)).toBe(1124);
+     expect(ollamaService.calculateNumCtx(100, 1024)).toBe(1124);
    });
    ```
-6. Create adapter classes with provider-specific `callAPI()`
+6. Create service classes with provider-specific `callAPI()`
 7. Validate all edge cases with test scenarios
+

From 8c69549eed8769b97ca49c62c994af737967ba5d Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:33:19 +0200
Subject: [PATCH 24/53] add refactoring step to plan: group llm services into a
 subdirectory

---
 services/ManualServiceRefactoring.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index bdd27b1f..4fe0f36a 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -147,4 +147,5 @@ function calculateNumCtx(promptLength, maxCtx) {
    ```
 6. Create service classes with provider-specific `callAPI()`
 7. Validate all edge cases with test scenarios
+8. group LlmServices into a subdirectory of services and move all tests testing solely production code in /services into /tests/services
 

From e1a219cc6d34df0cbd1da1445e93cdecb842bac4 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Tue, 1 Sep 2026 16:30:01 +0200
Subject: [PATCH 25/53] rework refactoring plan with claude, extend scope to
 include all four provider services

---
 services/ManualServiceRefactoring.md | 627 +++++++++++++++++++++------
 1 file changed, 489 insertions(+), 138 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 4fe0f36a..19e79b83 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -1,151 +1,502 @@
-# ManualService Refactoring Plan
+# LLM Service Refactoring Plan
+
 ## The Goal
 
-Refactor manualService.js. It has four functions, that are offering the same business
-service using different AI APIs. We aim at implementing the business logic only once 
-and coping with the API specific stuff In different Adapters.
-
-We will also use these adapters to refactor ollamaService.js, openaiService.js, customService.js, 
-and azureService.js.
-
-Apply TDD
-Adhere to the stylguide in /CLAUDE.md
-Ensure all tests are symmetrically to the existing tests and are integrated in the test suite.
-
-### **Expected Benefits**
-- Single point of business logic
-- Easier maintenance and extension
-- Better test coverage with mocks
-- Improved error handling with centralized `handleAPIError()`
-- Reduced code duplication across providers
-
-
-## The Plan
-
-### design approach
-
-#### 1. **Base Adapter Pattern**
-- Extract common logic to `LlmService` class
-- Centralize error handling with `handleAPIError()`
-- Standardize response parsing with `parseJSONResponse()`
-- **Suggestion**: Add validation for `AnalysisResult` structure to reject unexpected keys (e.g., `correspondent` might be missing in some cases)
-
-#### 2. **Provider Services**
-- Create separate Service classes for each AI provider:
-  - `OpenAIService`:
-    ```js
-    async callAPI(prompt) {
-      return this.openai.chat.completions.create({
-        model: 'gpt-3.5-turbo',
-        messages: [{ role: 'user', content: prompt }]
-      });
-    }
-    ```
-  - `OllamaService`:
-    ```js
-    async callAPI(prompt) {
-      return this.ollama.post('/api/generate', {
-        model: 'llama2',
-        prompt
-      });
-    }
-    ```
-  - `AzureService`:
-    ```js
-    async callAPI(prompt) {
-      return this.openai.chat.completions.create({
-        model: 'deployment-name',
-        messages: [{ role: 'user', content: prompt }]
-      });
-    }
-    ```
-  - Standardize method signatures (e.g. all adapters return text containing a valid JSON object)
-
-#### 3. **ManualService Refactor**
-- Replace switch statement with adapter factory pattern:
-  ```js
-  const service = new (require(`services/analyzers/${config.aiProvider}Service`))(config);
-  return await service.analyze(content, existingTags);
-  ```
-- Inject provider-specific configuration via constructor
-- Centralize error handling with `handleAPIError()`
-
-#### 4. **Test Coverage Strategy**
-- Mock API calls using `jest`/`sinon`:
-  - Example: Mock OpenAI API
-    ```js
-    jest.mock('openai', () => ({
-      chat: {
-        completions: {
-          create: jest.fn().mockResolvedValue({
-            choices: [{ message: { content: '{"tags":["tag1"],"correspondent":"John"}' }] }
-          })
-        }
-      }
-    }));
-    ```
-- Test scenarios:
-  - Valid JSON response: `tags` array and `correspondent` string
-  - Malformed JSON: `try/catch` validation
-  - Network error: `ECONNABORTED` handling
-  - Empty response: `null` check
-- **Suggestion**: Add mock for `X-RateLimit-Remaining` header in rate-limiting tests
-
-#### 5. **Edge Case Coverage**
-- Test empty `existingTags` array:
-  ```js
-  test('handles empty existingTags', async () => {
-    const result = await service.analyze('content', []);
-    expect(result.tags).toEqual(['default-tag']);
-  });
-  ```
-- Test missing environment variables:
-  ```js
-  test('fails on missing SYSTEM_PROMPT', () => {
-    process.env.SYSTEM_PROMPT = undefined;
-    expect(() => service.analyze()).toThrow();
-  });
-  ```
-- Test API rate limiting: Validate `X-RateLimit-Remaining` header
-- Test large input content: Validate token calculation for Ollama
-
-#### 6. **Return Value Structure**
-All adapters must return objects with this structure:
-```ts
-interface AnalysisResult {
-  tags: string[]; // Array of tag names
-  correspondent: string | null; // Correspondent name or null
-}
+Four provider services offer the same business service through different AI APIs. Implement the
+business logic **once** and keep the API-specific parts in thin, injected adapters.
+
+In scope: `openaiService.js`, `azureService.js`, `customService.js`, `ollamaService.js`, and the
+resolution of `manualService.js` (see Finding 1 — it is dead code, not the refactoring target it was
+assumed to be).
+
+Rules for this work:
+
+- Apply TDD.
+- Adhere to the conventions in `/CLAUDE.md` (English only, singleton services, `{ success, data }`
+  API shape, OpenAPI kept in sync, commit body as the single source of truth).
+- Prefer **delegation over inheritance**. No abstract base class, no `extends`.
+- New tests must be symmetrical to the existing tests and registered in the test runner.
+
+### Expected Benefits
+
+- Single point of business logic (prompt building, token budgeting, parsing, result shaping).
+- Easier maintenance and extension — a new provider is one adapter plus one composition root.
+- Better test coverage: injected collaborators make the seams testable without `require.cache` tricks.
+- Centralized, consistent error handling and result shaping.
+- Removal of ~2000 lines of near-duplicate code across four services.
+
+---
+
+## Validation Against the Source Code
+
+The following was verified against the working tree before rewriting this plan. Each finding either
+corrects the plan or constrains the design.
+
+### 1. `manualService.js` is dead code — the premise needs correcting
+
+A repo-wide search across all file types finds **no reference** to `services/manualService.js`
+(no `require`, no route, no view, no test). Refactoring it in isolation changes nothing at runtime.
+
+The duplication that the "manual" business service actually suffers from lives in
+`routes/setup.js`, which switches on `process.env.AI_PROVIDER` inline over the four _dedicated_
+singletons:
+
+- `routes/setup.js:7511-7578` — `POST /manual/analyze`, four-way switch calling `analyzeDocument()`.
+- `routes/setup.js:7657-7717` — `POST /manual/playground`, four-way switch calling `analyzePlayground()`.
+
+Further evidence that `manualService.js` was abandoned rather than maintained:
+
+- `services/manualService.js:270` calls `this._parseResponse(...)`, a method that **does not exist**
+  on `ManualService` (it exists only on `OllamaService`). The Ollama path throws `TypeError` on any
+  non-object response.
+- `existingTagsList` is computed and never used in all four methods
+  (`manualService.js:52`, `:110`, `:166`) — would fail `no-unused-vars` under `eslint.config.mjs`.
+- `manualService.js:273-276` contains German user-facing strings
+  (`'Die Analyse hat zu lange gedauert...'`), violating the English-only rule in `/CLAUDE.md`.
+- It is the only service exporting a **class** (`module.exports = ManualService;`) instead of a
+  singleton, contrary to the documented service-layer convention.
+- Its Ollama path never sends the document content at all — `prompt` is just
+  `process.env.SYSTEM_PROMPT` (`manualService.js:218`).
+
+**Decision:** delete `services/manualService.js` as part of this work and point the two
+`/manual/*` endpoints at `AIServiceFactory.getService()`. Do not port its behaviour — it is strictly
+worse than the dedicated services on every axis catalogued in `services/AiServices.md`.
+
+### 2. The real duplication is larger than the original plan assumed
+
+| Service            | Lines | Relationship                                    |
+| ------------------ | ----- | ----------------------------------------------- |
+| `openaiService.js` | 629   | ~90 % identical to azure/custom                 |
+| `azureService.js`  | 597   | ~90 % identical to openai/custom                |
+| `customService.js` | 706   | ~90 % identical, plus the strongest JSON parser |
+| `ollamaService.js` | 1164  | genuinely different transport and token model   |
+
+The largest duplicated block is **prompt construction**, which the original plan did not mention at
+all: the `CUSTOM_FIELDS` → `customFieldsTemplate` → `%CUSTOMFIELDS%` block appears **six times**
+(`openaiService.js:125-160`, `azureService.js:114-149`, `customService.js:209-244`,
+`ollamaService.js:147-179`, `:434-469`, `:585-622`), followed each time by the same
+`useExistingData` / `mustHavePrompt` / `RestrictionPromptService` / `USE_PROMPT_TAGS` /
+`customPrompt` cascade.
+
+`callAPI(prompt)` as the only seam therefore leaves the biggest duplicate in place.
+
+### 3. The return contract in the original plan would break every caller
+
+The plan proposed `interface AnalysisResult { tags, correspondent }`. The actual contract, produced
+by all four services and consumed by three call sites, is:
+
+```js
+{ document: { tags, correspondent, title, document_type, document_date, language, custom_fields },
+  metrics: { promptTokens, completionTokens, totalTokens } | null,
+  truncated: boolean,
+  error?: string,
+  errorCode?: string }
 ```
-Example:
+
+Consumers: `server.js:766-821`, `services/mistralOcrService.js:988-997`,
+`routes/setup.js:3167-3177`. Narrowing to `{ tags, correspondent }` would drop title, document type,
+date, language, custom fields and all metrics.
+
+### 4. The error contract is load-bearing and phrase-matched — it must be preserved verbatim
+
+`analyzeDocument()` **never throws**; it returns the failure in `error`/`errorCode`.
+`server.js:774-821` then routes the document to the OCR queue based on
+`shouldQueueForOcrOnAiError()` / `classifyOcrQueueReasonFromAiError()`
+(`services/serviceUtils.js:812-861`), which match on **exact phrases**:
+
+- `'insufficient content for ai analysis'`
+- `'invalid response structure'`
+- `'could not determine assignable metadata'`
+- `'invalid json response from api'`
+- `'invalid api response structure'`
+
+Rewording any of these silently disables OCR fallback. Guarded by the `ocr-fallback-ai-errors` and
+`response-truncation-detection` tests. The `error.code = 'ai_response_truncated'` marker
+(`serviceUtils.js:802`, `ollamaService.js:920`) has the same status.
+
+Consequence for the plan: the proposed "fails on missing SYSTEM_PROMPT → `expect(...).toThrow()`"
+test asserts a behaviour change that would break the scan loop. Dropped (see Test Strategy).
+
+### 5. The committed `services/llmService.js` does not parse, and its logic is doubly wrong
+
 ```js
-{ tags: ['tag1', 'tag2'], correspondent: 'John Doe' }
+class LlmService {
+  function parseJSONResponse(response) {   // ← SyntaxError: `function` in a class body
+```
+
+Beyond the syntax error: the function takes a _response object_ but calls `JSON.parse(response)` on
+it, then reads `json.choices[0].message.content.tags` — treating `content` as an object although the
+provider delivers it as a JSON **string**. Both halves cannot be true at once. The file also
+exports `{ LlmService }` while the repo convention is a singleton export.
+
+### 6. The committed `tests/test-llm-service.js` cannot run — test-framework mismatch
+
+- It imports `vitest` (`describe/it/beforeEach/expect`) but the runner executes tests as plain node:
+  `spawnSync(process.execPath, [filePath])` at `scripts/run-tests.js:471`. Vitest globals do not
+  exist under `node tests/test-llm-service.js`.
+- It additionally calls `jest.fn()` (`test-llm-service.js:34`) — `jest` is not defined anywhere.
+- `require('../services/LlmService')` vs. the actual file `services/llmService.js`: works on
+  Windows, **fails on Linux CI and in Docker**.
+- `parseJSONResponse(...)` is called as a free function, not on `service`.
+- `LlmService.calculateNumCtx(...)` is called as a **static** that does not exist.
+- `LlmService.calculateNum2024(500, 1024)` (`:55`) is a typo for a method that does not and should
+  not exist.
+- Test 4 asserts `calculateNumCtx(100, 1024) === 1124`, which contradicts the plan's own §7 formula
+  `min(floor(100/2), 1024) === 50`, and both contradict the real implementation (Finding 7).
+
+Every existing test in `tests/` is a plain node script: `require('assert')`, a `main()`, and
+`[PASS]`/`[FAIL]` plus `process.exitCode = 1`. `tests/test-ollama-temperature-wiring.js` is the
+closest model for service tests, including its `require.cache` injection helper
+(`:26-39`).
+
+**Decision:** revert the `vitest` devDependency from `package.json` and the 53 lockfile
+entries, and write plain-node tests. A new test framework is a dependency-manifest change that
+additionally triggers `docker-check.yml`, for zero benefit over the house style — and the mixed
+`vitest` + `jest` API in one file shows the two idioms do not survive contact.
+
+### 7. The `num_ctx` formula in the original plan §7 is wrong
+
+Real implementation (`ollamaService.js:691-702`):
+
+```js
+_calculateNumCtx(promptTokenCount, expectedResponseTokens) {
+  return Math.min(promptTokenCount + expectedResponseTokens, Number(config.tokenLimit));
+}
+```
+
+The plan's `min(floor(promptLength / 2), maxCtx)` conflates **characters** with **tokens** and drops
+the response reservation entirely — it would size the window at half the prompt and guarantee
+truncation.
+
+The `/2` divisor belongs to a _different_ function, `_calculatePromptTokenCount`
+(`ollamaService.js:678-683`), and its conservative value is deliberate and documented (non-English
+tokenization). `manualService.js:234` used `/4`. Centralizing these must **preserve `/2` for
+Ollama**, not unify the divisors.
+
+### 8. `callAPI(prompt)` is too narrow a signature for Ollama
+
+Ollama's transport needs `(prompt, systemPrompt, numCtx, schema)` (`ollamaService.js:811`), sends
+`format: schema` for structured output, computes `num_ctx`/`num_predict`, carries a bounded retry for
+transient failures (`:841-861`), and inspects `done_reason` for truncation (`:886-922`). Its response
+may arrive as a **JS object** rather than a string (`:929-949`). A one-argument `callAPI` cannot
+carry any of this.
+
+### 9. Behavioural drift between the four services — each item needs an explicit decision
+
+The refactor collapses four implementations into one, which forces a choice wherever they differ
+today. These are the actual bugs the work will surface:
+
+| #   | Drift                                                                                             | Location                                                                                                                                                                         | Proposed resolution                                                                                                                                                |
+| --- | ------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`         | `azureService.js:258`, `:426`                                                                                                                                                    | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                    |
+| b   | `return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined` | `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                              | Adopt Ollama's behaviour (`_handleThumbnailCaching`, `:708-725`): warn and continue. This is a latent crash.                                                       |
+| c   | "Insufficient content" early return exists only in OpenAI                                         | `openaiService.js:310-332`, `:505-527`                                                                                                                                           | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                           |
+| d   | Three different JSON parsers                                                                      | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | One parser built from custom's `_extractFirstJsonValue` + `<think>` stripping + Ollama's sanitize fallback; keep Ollama's object passthrough as a pre-step.        |
+| e   | Only custom normalizes timeouts                                                                   | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                               |
+| f   | `generateText` max-token handling differs four ways                                               | openai: none; `azureService.js:528`: `max_tokens`; `customService.js:625-654`: clamped; `ollamaService.js:1105`: `num_predict`                                                   | Keep custom's clamping as the shared rule; the transport maps it to its own knob.                                                                                  |
+| g   | `checkStatus` differs by nature                                                                   | openai/custom `models.list()`; `azureService.js:574-580` REST GET; `ollamaService.js:1143` `/api/ps`                                                                             | Stays in the transport. Not business logic.                                                                                                                        |
+| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`          | `openaiService.js:33-44`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop. |
+| i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt    | all four                                                                                                                                                                         | Out of scope. Preserve behaviour; note as a follow-up.                                                                                                             |
+
+### 10. The original plan's §4/§5 test examples do not match the code
+
+`service.analyze('content', [])` does not exist. The real seam is
+`analyzeDocument(content, existingTags, existingCorrespondentList, existingDocumentTypesList, id, customPrompt, options)`.
+`expect(result.tags).toEqual(['default-tag'])` asserts a default tag that no provider produces.
+The `X-RateLimit-Remaining` suggestion has no counterpart in the codebase — no AI code path reads
+that header; the `rate-limiting` test targets the app's own express-rate-limit. Dropped.
+
+### 11. Plan step 8 (move tests into `tests/services/`) breaks the CI drift guard
+
+`findUnregisteredTests()` (`scripts/run-tests.js:233-249`) reads `tests/` **non-recursively** and
+matches `/^test-.*\.js$/`. Any test moved into `tests/services/` silently leaves the guard, which is
+exactly the failure mode that guard exists to prevent. Registration itself would still work
+(`path.join(__dirname, '..', 'tests', 'services/test-x.js')` is portable), but the discovery walk
+must be made recursive in the same change.
+
+Additionally, "move all tests testing solely production code in /services" would relocate a large
+set of files unrelated to this refactor (`restriction-service`, `updated-service`,
+`reconciliation-service`, the `ollama-*` and `ocr-*` families, …). That is churn that hides the
+refactor in the diff. **Deferred to a separate PR** (see Out of Scope).
+
+---
+
+## Target Architecture
+
+### Design principle: composition, not inheritance
+
+One **concrete** business-logic class holds injected collaborators. There is no base class and
+nothing `extends` anything. A provider is a _composition root_: it wires collaborators together and
+exports the singleton facade.
+
+```
+services/llm/
+  documentAnalyzer.js            the single business-logic implementation (concrete, never subclassed)
+  promptBuilder.js               systemPrompt + user content assembly (replaces 6 duplicates)
+  customFieldsTemplate.js        the %CUSTOMFIELDS% block
+  responseParser.js              raw text|object -> document object
+  analysisResult.js              buildSuccess() / buildFailure() — the one place the result shape lives
+  tokenBudget/
+    tiktokenBudget.js            openai/azure/custom: calculateTotalPromptTokens + truncateToTokenLimit
+    charBudget.js                ollama: /2 estimate + _fitContentToContext
+  transports/
+    chatCompletionsTransport.js  OpenAI SDK, AzureOpenAI SDK, custom base URL — messages[]
+    ollamaGenerateTransport.js   axios /api/generate, format schema, num_ctx/num_predict, retry
+  providers/
+    openaiProvider.js            composition root + singleton facade
+    azureProvider.js
+    customProvider.js
+    ollamaProvider.js
 ```
-Error case:
+
+The existing module paths stay as one-line re-exports
+(`services/openaiService.js` → `module.exports = require('./llm/providers/openaiProvider');`) so that
+`aiServiceFactory.js`, `routes/setup.js`, `server.js` and `mistralOcrService.js` need no change and
+the diff stays reviewable. The four re-export shims can be collapsed in a later cleanup.
+
+### The delegation contracts
+
+Written as JSDoc typedefs in `services/llm/contracts.js`; enforced by tests, not by a base class.
+
 ```js
-{ tags: [], correspondent: null }
+/**
+ * @typedef {Object} CompletionRequest
+ * @property {string}  systemPrompt
+ * @property {string}  userContent
+ * @property {string}  model
+ * @property {number}  temperature
+ * @property {number}  maxResponseTokens
+ * @property {Object|null} schema           structured-output schema; null for chat transports
+ * @property {'analysis'|'generation'} purpose
+ */
+
+/**
+ * @typedef {Object} CompletionResult
+ * @property {string|Object} payload        raw assistant text, or the object Ollama may return
+ * @property {{promptTokens:number, completionTokens:number, totalTokens:number}|null} metrics
+ */
+
+/**
+ * A transport owns exactly one thing: how to talk to one API family.
+ * It throws on transport failure and on provider-reported truncation
+ * (error.code === 'ai_response_truncated'); it never shapes an AnalysisResult.
+ *
+ * @typedef {Object} LlmTransport
+ * @property {string} label                             'OpenAI' | 'AzureOpenAI' | 'Custom OpenAI' | 'Ollama'
+ * @property {() => void} initialize                    lazy client construction, idempotent
+ * @property {() => boolean} isReady
+ * @property {(req: CompletionRequest) => Promise<CompletionResult>} complete
+ * @property {() => Promise<{status:string, model?:string, error?:string}>} checkStatus
+ */
+
+/**
+ * @typedef {Object} TokenBudget
+ * @property {(systemPrompt:string, extraPrompts:string[], model:string) => Promise<{totalPromptTokens:number, availableTokens:number}>} reserve
+ * @property {(content:string, availableTokens:number, model:string) => Promise<string>} fit
+ */
 ```
 
-### 7. **Token Calculation Centralization**
-**Suggestion**: Centralize Ollama token calculation logic in `calculateNumCtx()` (input prompt characters / 2 or /4) to avoid duplication. Example:
+`maxResponseTokens` is what each transport maps to its own knob: `max_tokens` for the chat
+transports, `num_predict` plus a `num_ctx` sized by `min(promptTokens + maxResponseTokens, tokenLimit)`
+for Ollama. That mapping is the only place the Ollama context arithmetic lives, and it keeps
+Finding 7's real formula intact.
+
+### DocumentAnalyzer — the single business logic
+
 ```js
-function calculateNumCtx(promptLength, maxCtx) {
-  return Math.min(Math.floor(promptLength / 2), maxCtx);
+class DocumentAnalyzer {
+  /**
+   * @param {Object} deps
+   * @param {LlmTransport} deps.transport
+   * @param {TokenBudget}  deps.tokenBudget
+   * @param {Object}       deps.promptBuilder
+   * @param {Object}       deps.responseParser
+   * @param {() => string} deps.resolveModel      provider-specific model lookup
+   * @param {Object}       [deps.thumbnailCache]  injected for testability
+   */
+  constructor({ transport, tokenBudget, promptBuilder, responseParser, resolveModel, thumbnailCache }) { ... }
+
+  async analyzeDocument(content, existingTags = [], existingCorrespondentList = [],
+                        existingDocumentTypesList = [], id, customPrompt = null, options = {}) { ... }
+
+  async analyzePlayground(content, prompt) { ... }
+  async generateText(prompt) { ... }
+  checkStatus() { return this.transport.checkStatus(); }
 }
 ```
 
-###  **Implementation Steps**
-1. Write tests for `LlmService` first (TDD)
-2. Implement `LlmService` with abstract `callAPI()`
-3. Add tests for derived Services
-4. Mock API calls for each provider in tests
-5. Add Ollama token calculation tests:
-   ```js
-   test('calculates num_ctx correctly', () => {
-     expect(ollamaService.calculateNumCtx(100, 1024)).toBe(1124);
-   });
-   ```
-6. Create service classes with provider-specific `callAPI()`
-7. Validate all edge cases with test scenarios
-8. group LlmServices into a subdirectory of services and move all tests testing solely production code in /services into /tests/services
+Every collaborator arrives through the constructor. That is what makes the tests cheap: a fake
+transport is an object literal with a `complete()`, so no `require.cache` surgery and no network.
+
+### Provider composition root (example)
+
+```js
+// services/llm/providers/azureProvider.js
+const DocumentAnalyzer = require('../documentAnalyzer');
+const ChatCompletionsTransport = require('../transports/chatCompletionsTransport');
+const tiktokenBudget = require('../tokenBudget/tiktokenBudget');
+const promptBuilder = require('../promptBuilder');
+const responseParser = require('../responseParser');
+const config = require('../../../config/config');
+
+module.exports = new DocumentAnalyzer({
+  transport: new ChatCompletionsTransport({
+    label: 'AzureOpenAI',
+    createClient: () =>
+      new (require('openai').AzureOpenAI)({
+        apiKey: config.azure.apiKey,
+        endpoint: config.azure.endpoint,
+        deploymentName: config.azure.deploymentName,
+        apiVersion: config.azure.apiVersion,
+      }),
+    checkStatus: azureDeploymentProbe, // the REST GET from azureService.js:574
+  }),
+  tokenBudget: tiktokenBudget,
+  promptBuilder,
+  responseParser,
+  resolveModel: () => process.env.AZURE_DEPLOYMENT_NAME,
+});
+```
+
+Adding a fifth provider is: one `createClient`, one `resolveModel`, one `checkStatus` probe. No new
+business logic, no subclass.
+
+### Why not the base-class variant
+
+An abstract `LlmService` with `callAPI()` overridden per provider forces Ollama's four extra
+parameters, its object-shaped response, its retry loop and its char-based token model either into the
+base class (where they are dead weight for three providers) or into an override that reimplements the
+template method (which is the duplication we started with). Injecting a transport keeps Ollama's
+peculiarities inside `ollamaGenerateTransport.js` and out of everyone else's way.
+
+---
+
+## Test Strategy
+
+House style, no new framework: plain node scripts under `tests/`, `require('assert')`, one `main()`,
+`[PASS]`/`[FAIL]` and `process.exitCode = 1`. Model: `tests/test-ollama-temperature-wiring.js`.
+
+Because collaborators are injected, most tests need no module mocking at all — construct a
+`DocumentAnalyzer` with a fake transport:
+
+```js
+const fakeTransport = {
+  label: 'Fake',
+  initialize() {},
+  isReady: () => true,
+  calls: [],
+  async complete(request) {
+    this.calls.push(request);
+    return {
+      payload: JSON.stringify({
+        tags: ['tag1'],
+        correspondent: 'John Doe',
+        title: 't',
+        document_type: 'Invoice',
+        document_date: '2026-01-01',
+        language: 'en',
+      }),
+      metrics: { promptTokens: 10, completionTokens: 5, totalTokens: 15 },
+    };
+  },
+  async checkStatus() {
+    return { status: 'ok' };
+  },
+};
+```
+
+Planned tests (each registered in `TESTS` in `scripts/run-tests.js` and in an `AREAS` entry):
+
+| Test name                         | Asserts                                                                                                                                                                                                                        |
+| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `llm-response-parser`             | fenced ```json stripped; `<think>…</think>` stripped; brace-matched extraction from prose; trailing-comma/unquoted-key sanitize fallback; unparseable input throws with the exact phrase `Invalid JSON response from API`      |
+| `llm-analysis-result-contract`    | success returns `{document, metrics, truncated}`; failure returns `{document:{tags:[],correspondent:null}, metrics:null, error, errorCode}` and **never throws**                                                               |
+| `llm-ocr-fallback-phrases`        | every phrase in `shouldQueueForOcrOnAiError` is still produced by the refactored paths — the regression guard for Finding 4                                                                                                    |
+| `llm-prompt-builder`              | `%CUSTOMFIELDS%` rendering for date/boolean/other; `useExistingData` branch; `USE_PROMPT_TAGS` override; `customPrompt` override; restriction placeholders delegated to `RestrictionPromptService`; external API data appended |
+| `llm-token-budget`                | `tiktokenBudget.reserve` throws `Token limit exceeded: …` when `availableTokens <= 0`; `charBudget` uses the `/2` divisor and `min(promptTokens + responseTokens, tokenLimit)` for `num_ctx` — Finding 7                       |
+| `llm-transport-request-mapping`   | chat transports send `messages[]` with system+user and omit `temperature` for `o3-mini`; Ollama transport sends `format`, `num_ctx`, `num_predict`, `think:false` unless `OLLAMA_THINK=true`                                   |
+| `llm-azure-temperature-wiring`    | Azure analysis requests carry `config.aiTemperatureAnalysis`, not a hardcoded `0.3` — Finding 9a                                                                                                                               |
+| `llm-missing-thumbnail-continues` | a missing thumbnail yields a full `AnalysisResult`, never `undefined` — Finding 9b                                                                                                                                             |
+| `llm-provider-parity`             | all four composition roots expose `analyzeDocument`/`analyzePlayground`/`generateText`/`checkStatus` and, given the same fake transport payload, produce byte-identical `document` objects                                     |
+
+Existing tests that must stay green without modification (they are the real acceptance criteria):
+`ocr-fallback-ai-errors`, `response-truncation-detection`, `ollama-response-limit`,
+`ollama-upstream-error`, `ollama-temperature-wiring`, `ollama-token-metrics`,
+`prompt-existing-data-serialization`, `restriction-service`, `restricted-document-types-placeholder`,
+`document-type-restriction`, `ai-temperature-config`, `playground-deprecation`.
+
+Repo-hygiene steps that belong to this work:
+
+- Delete `services/llmService.js` and `tests/test-llm-service.js` in their current form (Findings 5, 6).
+- Revert the `vitest` devDependency in `package.json` and its lockfile entries.
+- Register every new test in `TESTS`; run `node scripts/run-tests.js --all` — the registry drift
+  check only runs on `--all`.
+- `npx eslint <changed>`, `npx prettier --check <changed>` — CI lints the files touched by the PR.
+
+---
+
+## Implementation Steps
+
+Each step ends with a green `node scripts/run-tests.js --all`. Steps 1-3 are pure preparation and
+carry no behavioral risk; the risk starts at step 5.
+
+1. **Clean the slate.** Remove `services/llmService.js` and `tests/test-llm-service.js`; revert the
+   `vitest` devDependency and lockfile entries; keep the `llm-service` entry out of `TESTS` until a
+   runnable test replaces it. Move this plan and `services/AiServices.md` out of `services/` (they
+   are documents, not services — `docs/` or the PR body).
+2. **Freeze the contract in tests first (TDD).** Write `llm-analysis-result-contract` and
+   `llm-ocr-fallback-phrases` against the **current** `openaiService`/`ollamaService`. They must pass
+   before any production code moves — that is what makes them a regression net rather than a
+   description of the new code.
+3. **Extract `customFieldsTemplate.js` and `promptBuilder.js`** with `llm-prompt-builder`, and have
+   all four existing services call them. Six duplicates become one; no other behaviour changes.
+   `prompt-existing-data-serialization` and the restriction tests are the guard.
+4. **Extract `responseParser.js`** (custom's brace matcher + `<think>` stripping + Ollama's sanitize
+   fallback + Ollama's object passthrough) with `llm-response-parser`, and route all four services
+   through it. Resolves Finding 9d.
+5. **Extract `analysisResult.js`** and the two `tokenBudget` strategies with `llm-token-budget`.
+   Fix Finding 9b (missing thumbnail) here, with `llm-missing-thumbnail-continues` written first.
+6. **Introduce the transports.** `chatCompletionsTransport.js` covering openai/azure/custom, and
+   `ollamaGenerateTransport.js` carrying the retry, `format`, `num_ctx`/`num_predict` and
+   `done_reason` truncation check moved verbatim from `ollamaService.js:811-922`. Cover with
+   `llm-transport-request-mapping`.
+7. **Introduce `documentAnalyzer.js`** and the four composition roots. Reduce
+   `services/{openai,azure,custom,ollama}Service.js` to re-export shims. Add `llm-provider-parity`.
+   Fix Finding 9a here (Azure temperature) with its test.
+8. **Delete `services/manualService.js`** and repoint `POST /manual/analyze` and
+   `POST /manual/playground` (`routes/setup.js:7511`, `:7657`) at `AIServiceFactory.getService()`,
+   collapsing both four-way switches. Note that `/manual/analyze` currently calls
+   `documentModel.addOpenAIMetrics()` only on the OpenAI branch while `/manual/playground` calls it on
+   three of four — decide explicitly whether metrics are recorded for every provider, and if the
+   response shape changes, update the `@swagger` JSDoc and run `node scripts/regen-openapi.js`
+   (the CI drift check will fail otherwise).
+9. **Verify Finding 9h** (unreachable `initialize()` branches in the former `openaiService`) and drop
+   the dead branches.
+10. **Full verification.** `node scripts/run-tests.js --all`; ESLint/Prettier on every changed file;
+    `node scripts/regen-openapi.js` + `git diff --exit-code OPENAPI/openapi.json`; a manual smoke run
+    of `/manual/analyze` and a scan cycle against a real provider, since no test exercises a live API.
+
+---
+
+## Out of Scope / Follow-ups
+
+- **Relocating unrelated tests into `tests/services/`** (original step 8). Needs `findUnregisteredTests()`
+  made recursive first (Finding 11) and would bury this refactor in rename noise. Separate PR.
+- **Thumbnail caching inside `analyzeDocument`** (Finding 9i): the thumbnail is fetched, written to
+  disk and never used in the prompt. Behaviour preserved here; worth its own issue.
+- **Vision/multimodal analysis**, which is presumably why the thumbnail is cached at all.
+- **`analyzePlayground` deduplication beyond the shared collaborators** — the endpoint is already
+  deprecated (`routes/setup.js:7584`, `playground-deprecation` test); do not invest in it.
+- **A structured-output (`response_format`/`format`) path for the chat transports** to match what
+  Ollama already does. Would likely remove most of `responseParser`, but it is a behaviour change
+  needing its own validation against real providers.
+
+## Risks
 
+| Risk                                                                         | Mitigation                                                                                                                           |
+| ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
+| OCR fallback silently stops working because an error phrase changed          | `llm-ocr-fallback-phrases` written in step 2, before any code moves                                                                  |
+| A caller reads a field that the unified `document` object no longer carries  | `llm-provider-parity` compares full objects; `server.js:766`, `mistralOcrService.js:988`, `routes/setup.js:3167` reviewed explicitly |
+| Ollama's token arithmetic subtly changes and long documents start truncating | `charBudget` keeps the `/2` divisor and the real `num_ctx` formula; `ollama-response-limit` and `ollama-token-metrics` guard it      |
+| No test exercises a real provider API                                        | Step 10 keeps a manual smoke run against at least one live provider in the acceptance criteria                                       |
+| The refactor lands as one unreviewable diff                                  | Steps 3-7 are individually green and individually landable                                                                           |

From 5181025e61336f1723cd47e73934e1130e51f1dd Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Tue, 1 Sep 2026 16:43:28 +0200
Subject: [PATCH 26/53] refactoring 1: remove first approach code and update
 plan

---
 services/ManualServiceRefactoring.md | 239 ++++++++++++---------------
 1 file changed, 102 insertions(+), 137 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 19e79b83..85269106 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -6,8 +6,7 @@ Four provider services offer the same business service through different AI APIs
 business logic **once** and keep the API-specific parts in thin, injected adapters.
 
 In scope: `openaiService.js`, `azureService.js`, `customService.js`, `ollamaService.js`, and the
-resolution of `manualService.js` (see Finding 1 — it is dead code, not the refactoring target it was
-assumed to be).
+removal of `manualService.js` (see Finding 1 — it is dead code).
 
 Rules for this work:
 
@@ -29,10 +28,9 @@ Rules for this work:
 
 ## Validation Against the Source Code
 
-The following was verified against the working tree before rewriting this plan. Each finding either
-corrects the plan or constrains the design.
+The following was verified against the working tree. Each finding constrains the design that follows.
 
-### 1. `manualService.js` is dead code — the premise needs correcting
+### 1. `manualService.js` is dead code
 
 A repo-wide search across all file types finds **no reference** to `services/manualService.js`
 (no `require`, no route, no view, no test). Refactoring it in isolation changes nothing at runtime.
@@ -59,10 +57,11 @@ Further evidence that `manualService.js` was abandoned rather than maintained:
   `process.env.SYSTEM_PROMPT` (`manualService.js:218`).
 
 **Decision:** delete `services/manualService.js` as part of this work and point the two
-`/manual/*` endpoints at `AIServiceFactory.getService()`. Do not port its behaviour — it is strictly
-worse than the dedicated services on every axis catalogued in `services/AiServices.md`.
+`/manual/*` endpoints at `AIServiceFactory.getService()`. Do not port its behaviour — it is worse
+than the dedicated services on every axis: no token budgeting, no content truncation, no restriction
+placeholders, no structured output, no retry, no metrics.
 
-### 2. The real duplication is larger than the original plan assumed
+### 2. Scale and shape of the real duplication
 
 | Service            | Lines | Relationship                                    |
 | ------------------ | ----- | ----------------------------------------------- |
@@ -71,19 +70,19 @@ worse than the dedicated services on every axis catalogued in `services/AiServic
 | `customService.js` | 706   | ~90 % identical, plus the strongest JSON parser |
 | `ollamaService.js` | 1164  | genuinely different transport and token model   |
 
-The largest duplicated block is **prompt construction**, which the original plan did not mention at
-all: the `CUSTOM_FIELDS` → `customFieldsTemplate` → `%CUSTOMFIELDS%` block appears **six times**
+The largest duplicated block is **prompt construction**: the
+`CUSTOM_FIELDS` → `customFieldsTemplate` → `%CUSTOMFIELDS%` block appears **six times**
 (`openaiService.js:125-160`, `azureService.js:114-149`, `customService.js:209-244`,
 `ollamaService.js:147-179`, `:434-469`, `:585-622`), followed each time by the same
 `useExistingData` / `mustHavePrompt` / `RestrictionPromptService` / `USE_PROMPT_TAGS` /
 `customPrompt` cascade.
 
-`callAPI(prompt)` as the only seam therefore leaves the biggest duplicate in place.
+A seam drawn only at the API call would leave that block untouched. Prompt construction therefore
+has to be a first-class collaborator of its own, not a side effect of the transport.
 
-### 3. The return contract in the original plan would break every caller
+### 3. The result contract that must be preserved
 
-The plan proposed `interface AnalysisResult { tags, correspondent }`. The actual contract, produced
-by all four services and consumed by three call sites, is:
+Produced by all four services and consumed by three call sites:
 
 ```js
 { document: { tags, correspondent, title, document_type, document_date, language, custom_fields },
@@ -94,8 +93,9 @@ by all four services and consumed by three call sites, is:
 ```
 
 Consumers: `server.js:766-821`, `services/mistralOcrService.js:988-997`,
-`routes/setup.js:3167-3177`. Narrowing to `{ tags, correspondent }` would drop title, document type,
-date, language, custom fields and all metrics.
+`routes/setup.js:3167-3177`. Any narrowing of `document` drops title, document type, date, language
+or custom fields silently — the write-back in `paperlessService.updateDocument()` simply stops
+setting those fields.
 
 ### 4. The error contract is load-bearing and phrase-matched — it must be preserved verbatim
 
@@ -114,99 +114,61 @@ Rewording any of these silently disables OCR fallback. Guarded by the `ocr-fallb
 `response-truncation-detection` tests. The `error.code = 'ai_response_truncated'` marker
 (`serviceUtils.js:802`, `ollamaService.js:920`) has the same status.
 
-Consequence for the plan: the proposed "fails on missing SYSTEM_PROMPT → `expect(...).toThrow()`"
-test asserts a behaviour change that would break the scan loop. Dropped (see Test Strategy).
+Consequence for the design: `analyzeDocument()` keeps its non-throwing contract. A version that
+throws on, say, a missing `SYSTEM_PROMPT` would take down the scan loop instead of queueing the
+document.
 
-### 5. The committed `services/llmService.js` does not parse, and its logic is doubly wrong
+### 5. The Ollama context arithmetic must be preserved exactly
 
-```js
-class LlmService {
-  function parseJSONResponse(response) {   // ← SyntaxError: `function` in a class body
-```
-
-Beyond the syntax error: the function takes a _response object_ but calls `JSON.parse(response)` on
-it, then reads `json.choices[0].message.content.tags` — treating `content` as an object although the
-provider delivers it as a JSON **string**. Both halves cannot be true at once. The file also
-exports `{ LlmService }` while the repo convention is a singleton export.
-
-### 6. The committed `tests/test-llm-service.js` cannot run — test-framework mismatch
-
-- It imports `vitest` (`describe/it/beforeEach/expect`) but the runner executes tests as plain node:
-  `spawnSync(process.execPath, [filePath])` at `scripts/run-tests.js:471`. Vitest globals do not
-  exist under `node tests/test-llm-service.js`.
-- It additionally calls `jest.fn()` (`test-llm-service.js:34`) — `jest` is not defined anywhere.
-- `require('../services/LlmService')` vs. the actual file `services/llmService.js`: works on
-  Windows, **fails on Linux CI and in Docker**.
-- `parseJSONResponse(...)` is called as a free function, not on `service`.
-- `LlmService.calculateNumCtx(...)` is called as a **static** that does not exist.
-- `LlmService.calculateNum2024(500, 1024)` (`:55`) is a typo for a method that does not and should
-  not exist.
-- Test 4 asserts `calculateNumCtx(100, 1024) === 1124`, which contradicts the plan's own §7 formula
-  `min(floor(100/2), 1024) === 50`, and both contradict the real implementation (Finding 7).
-
-Every existing test in `tests/` is a plain node script: `require('assert')`, a `main()`, and
-`[PASS]`/`[FAIL]` plus `process.exitCode = 1`. `tests/test-ollama-temperature-wiring.js` is the
-closest model for service tests, including its `require.cache` injection helper
-(`:26-39`).
-
-**Decision:** revert the `vitest` devDependency from `package.json` and the 53 lockfile
-entries, and write plain-node tests. A new test framework is a dependency-manifest change that
-additionally triggers `docker-check.yml`, for zero benefit over the house style — and the mixed
-`vitest` + `jest` API in one file shows the two idioms do not survive contact.
-
-### 7. The `num_ctx` formula in the original plan §7 is wrong
-
-Real implementation (`ollamaService.js:691-702`):
+Two separate functions, easily conflated:
 
 ```js
+// ollamaService.js:678-683 — conservative 2 chars/token, deliberate and documented
+_calculatePromptTokenCount(prompt) {
+  return Math.ceil(prompt.length / 2);
+}
+
+// ollamaService.js:691-702 — tokens, with the answer reserved
 _calculateNumCtx(promptTokenCount, expectedResponseTokens) {
   return Math.min(promptTokenCount + expectedResponseTokens, Number(config.tokenLimit));
 }
 ```
 
-The plan's `min(floor(promptLength / 2), maxCtx)` conflates **characters** with **tokens** and drops
-the response reservation entirely — it would size the window at half the prompt and guarantee
-truncation.
+The `/2` divisor exists because non-English tokenization (CJK, German) produces roughly two
+characters per token instead of ~4; `manualService.js:234` used `/4` and would truncate those
+documents. Centralizing the estimators must **preserve `/2` for Ollama**, not unify the divisors.
 
-The `/2` divisor belongs to a _different_ function, `_calculatePromptTokenCount`
-(`ollamaService.js:678-683`), and its conservative value is deliberate and documented (non-English
-tokenization). `manualService.js:234` used `/4`. Centralizing these must **preserve `/2` for
-Ollama**, not unify the divisors.
+`_fitContentToContext()` (`ollamaService.js:379-409`) uses the same `/2` estimate by construction, so
+the two agree. Whatever holds this arithmetic after the refactor has to keep that property.
 
-### 8. `callAPI(prompt)` is too narrow a signature for Ollama
+### 6. The transport seam needs a wide signature
 
 Ollama's transport needs `(prompt, systemPrompt, numCtx, schema)` (`ollamaService.js:811`), sends
 `format: schema` for structured output, computes `num_ctx`/`num_predict`, carries a bounded retry for
 transient failures (`:841-861`), and inspects `done_reason` for truncation (`:886-922`). Its response
-may arrive as a **JS object** rather than a string (`:929-949`). A one-argument `callAPI` cannot
-carry any of this.
+may arrive as a **JS object** rather than a string (`:929-949`).
+
+A single-string `callAPI(prompt)` cannot carry any of that, which is why `CompletionRequest` below is
+a record rather than a string.
 
-### 9. Behavioural drift between the four services — each item needs an explicit decision
+### 7. Behavioural drift between the four services — each item needs an explicit decision
 
 The refactor collapses four implementations into one, which forces a choice wherever they differ
 today. These are the actual bugs the work will surface:
 
-| #   | Drift                                                                                             | Location                                                                                                                                                                         | Proposed resolution                                                                                                                                                |
-| --- | ------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`         | `azureService.js:258`, `:426`                                                                                                                                                    | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                    |
-| b   | `return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined` | `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                              | Adopt Ollama's behaviour (`_handleThumbnailCaching`, `:708-725`): warn and continue. This is a latent crash.                                                       |
-| c   | "Insufficient content" early return exists only in OpenAI                                         | `openaiService.js:310-332`, `:505-527`                                                                                                                                           | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                           |
-| d   | Three different JSON parsers                                                                      | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | One parser built from custom's `_extractFirstJsonValue` + `<think>` stripping + Ollama's sanitize fallback; keep Ollama's object passthrough as a pre-step.        |
-| e   | Only custom normalizes timeouts                                                                   | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                               |
-| f   | `generateText` max-token handling differs four ways                                               | openai: none; `azureService.js:528`: `max_tokens`; `customService.js:625-654`: clamped; `ollamaService.js:1105`: `num_predict`                                                   | Keep custom's clamping as the shared rule; the transport maps it to its own knob.                                                                                  |
-| g   | `checkStatus` differs by nature                                                                   | openai/custom `models.list()`; `azureService.js:574-580` REST GET; `ollamaService.js:1143` `/api/ps`                                                                             | Stays in the transport. Not business logic.                                                                                                                        |
-| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`          | `openaiService.js:33-44`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop. |
-| i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt    | all four                                                                                                                                                                         | Out of scope. Preserve behaviour; note as a follow-up.                                                                                                             |
-
-### 10. The original plan's §4/§5 test examples do not match the code
-
-`service.analyze('content', [])` does not exist. The real seam is
-`analyzeDocument(content, existingTags, existingCorrespondentList, existingDocumentTypesList, id, customPrompt, options)`.
-`expect(result.tags).toEqual(['default-tag'])` asserts a default tag that no provider produces.
-The `X-RateLimit-Remaining` suggestion has no counterpart in the codebase — no AI code path reads
-that header; the `rate-limiting` test targets the app's own express-rate-limit. Dropped.
+| #   | Drift                                                                                            | Location                                                                                                                                                                        | Proposed resolution                                                                                                                                               |
+| --- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`        | `azureService.js:258`, `:426`                                                                                                                                                   | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                    |
+| b   | `return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined` | `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                             | Adopt Ollama's behaviour (`_handleThumbnailCaching`, `:708-725`): warn and continue. This is a latent crash.                                                       |
+| c   | "Insufficient content" early return exists only in OpenAI                                        | `openaiService.js:310-332`, `:505-527`                                                                                                                                          | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                           |
+| d   | Three different JSON parsers                                                                     | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | One parser built from custom's `_extractFirstJsonValue` + `<think>` stripping + Ollama's sanitize fallback; keep Ollama's object passthrough as a pre-step.         |
+| e   | Only custom normalizes timeouts                                                                  | `customService.js:428-436`                                                                                                                                                      | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                               |
+| f   | `generateText` max-token handling differs four ways                                              | openai: none; `azureService.js:528`: `max_tokens`; `customService.js:625-654`: clamped; `ollamaService.js:1105`: `num_predict`                                                   | Keep custom's clamping as the shared rule; the transport maps it to its own knob.                                                                                  |
+| g   | `checkStatus` differs by nature                                                                  | openai/custom `models.list()`; `azureService.js:574-580` REST GET; `ollamaService.js:1143` `/api/ps`                                                                             | Stays in the transport. Not business logic.                                                                                                                       |
+| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`          | `openaiService.js:33-44`                                                                                                                                                        | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop. |
+| i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt    | all four                                                                                                                                                                        | Out of scope. Preserve behaviour; note as a follow-up.                                                                                                            |
 
-### 11. Plan step 8 (move tests into `tests/services/`) breaks the CI drift guard
+### 8. Moving tests into `tests/services/` would break the CI drift guard
 
 `findUnregisteredTests()` (`scripts/run-tests.js:233-249`) reads `tests/` **non-recursively** and
 matches `/^test-.*\.js$/`. Any test moved into `tests/services/` silently leaves the guard, which is
@@ -214,8 +176,8 @@ exactly the failure mode that guard exists to prevent. Registration itself would
 (`path.join(__dirname, '..', 'tests', 'services/test-x.js')` is portable), but the discovery walk
 must be made recursive in the same change.
 
-Additionally, "move all tests testing solely production code in /services" would relocate a large
-set of files unrelated to this refactor (`restriction-service`, `updated-service`,
+Additionally, moving every test that covers production code in `services/` would relocate a large set
+of files unrelated to this refactor (`restriction-service`, `updated-service`,
 `reconciliation-service`, the `ollama-*` and `ocr-*` families, …). That is churn that hides the
 refactor in the diff. **Deferred to a separate PR** (see Out of Scope).
 
@@ -299,7 +261,7 @@ Written as JSDoc typedefs in `services/llm/contracts.js`; enforced by tests, not
 `maxResponseTokens` is what each transport maps to its own knob: `max_tokens` for the chat
 transports, `num_predict` plus a `num_ctx` sized by `min(promptTokens + maxResponseTokens, tokenLimit)`
 for Ollama. That mapping is the only place the Ollama context arithmetic lives, and it keeps
-Finding 7's real formula intact.
+Finding 5's formula intact.
 
 ### DocumentAnalyzer — the single business logic
 
@@ -363,7 +325,7 @@ business logic, no subclass.
 
 ### Why not the base-class variant
 
-An abstract `LlmService` with `callAPI()` overridden per provider forces Ollama's four extra
+An abstract base class with a `callAPI()` override per provider forces Ollama's four extra
 parameters, its object-shaped response, its retry loop and its char-based token model either into the
 base class (where they are dead weight for three providers) or into an override that reimplements the
 template method (which is the duplication we started with). Injecting a transport keeps Ollama's
@@ -373,8 +335,14 @@ peculiarities inside `ollamaGenerateTransport.js` and out of everyone else's way
 
 ## Test Strategy
 
-House style, no new framework: plain node scripts under `tests/`, `require('assert')`, one `main()`,
-`[PASS]`/`[FAIL]` and `process.exitCode = 1`. Model: `tests/test-ollama-temperature-wiring.js`.
+House style: plain node scripts under `tests/`, `require('assert')`, one `main()`, `[PASS]`/`[FAIL]`
+and `process.exitCode = 1`. No test framework is introduced — the runner executes each file with
+`spawnSync(process.execPath, [filePath])` (`scripts/run-tests.js:471`), so framework globals are not
+available. Model for service tests: `tests/test-ollama-temperature-wiring.js`, including its
+`require.cache` injection helper (`:26-39`).
+
+The seam under test is the real one:
+`analyzeDocument(content, existingTags, existingCorrespondentList, existingDocumentTypesList, id, customPrompt, options)`.
 
 Because collaborators are injected, most tests need no module mocking at all — construct a
 `DocumentAnalyzer` with a fake transport:
@@ -407,17 +375,17 @@ const fakeTransport = {
 
 Planned tests (each registered in `TESTS` in `scripts/run-tests.js` and in an `AREAS` entry):
 
-| Test name                         | Asserts                                                                                                                                                                                                                        |
-| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `llm-response-parser`             | fenced ```json stripped; `<think>…</think>` stripped; brace-matched extraction from prose; trailing-comma/unquoted-key sanitize fallback; unparseable input throws with the exact phrase `Invalid JSON response from API`      |
-| `llm-analysis-result-contract`    | success returns `{document, metrics, truncated}`; failure returns `{document:{tags:[],correspondent:null}, metrics:null, error, errorCode}` and **never throws**                                                               |
-| `llm-ocr-fallback-phrases`        | every phrase in `shouldQueueForOcrOnAiError` is still produced by the refactored paths — the regression guard for Finding 4                                                                                                    |
+| Test name                         | Asserts                                                                                                                                                                                                                      |
+| --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `llm-response-parser`             | code fences stripped; `<think>…</think>` stripped; brace-matched extraction from surrounding prose; trailing-comma/unquoted-key sanitize fallback; unparseable input throws with the exact phrase `Invalid JSON response from API` |
+| `llm-analysis-result-contract`    | success returns `{document, metrics, truncated}`; failure returns `{document:{tags:[],correspondent:null}, metrics:null, error, errorCode}` and **never throws**                                                              |
+| `llm-ocr-fallback-phrases`        | every phrase in `shouldQueueForOcrOnAiError` is still produced by the refactored paths — the regression guard for Finding 4                                                                                                   |
 | `llm-prompt-builder`              | `%CUSTOMFIELDS%` rendering for date/boolean/other; `useExistingData` branch; `USE_PROMPT_TAGS` override; `customPrompt` override; restriction placeholders delegated to `RestrictionPromptService`; external API data appended |
-| `llm-token-budget`                | `tiktokenBudget.reserve` throws `Token limit exceeded: …` when `availableTokens <= 0`; `charBudget` uses the `/2` divisor and `min(promptTokens + responseTokens, tokenLimit)` for `num_ctx` — Finding 7                       |
+| `llm-token-budget`                | `tiktokenBudget.reserve` throws `Token limit exceeded: …` when `availableTokens <= 0`; `charBudget` uses the `/2` divisor and `min(promptTokens + responseTokens, tokenLimit)` for `num_ctx` — Finding 5                       |
 | `llm-transport-request-mapping`   | chat transports send `messages[]` with system+user and omit `temperature` for `o3-mini`; Ollama transport sends `format`, `num_ctx`, `num_predict`, `think:false` unless `OLLAMA_THINK=true`                                   |
-| `llm-azure-temperature-wiring`    | Azure analysis requests carry `config.aiTemperatureAnalysis`, not a hardcoded `0.3` — Finding 9a                                                                                                                               |
-| `llm-missing-thumbnail-continues` | a missing thumbnail yields a full `AnalysisResult`, never `undefined` — Finding 9b                                                                                                                                             |
-| `llm-provider-parity`             | all four composition roots expose `analyzeDocument`/`analyzePlayground`/`generateText`/`checkStatus` and, given the same fake transport payload, produce byte-identical `document` objects                                     |
+| `llm-azure-temperature-wiring`    | Azure analysis requests carry `config.aiTemperatureAnalysis`, not a hardcoded `0.3` — Finding 7a                                                                                                                              |
+| `llm-missing-thumbnail-continues` | a missing thumbnail yields a full result object, never `undefined` — Finding 7b                                                                                                                                              |
+| `llm-provider-parity`             | all four composition roots expose `analyzeDocument`/`analyzePlayground`/`generateText`/`checkStatus` and, given the same fake transport payload, produce byte-identical `document` objects                                    |
 
 Existing tests that must stay green without modification (they are the real acceptance criteria):
 `ocr-fallback-ai-errors`, `response-truncation-detection`, `ollama-response-limit`,
@@ -425,78 +393,75 @@ Existing tests that must stay green without modification (they are the real acce
 `prompt-existing-data-serialization`, `restriction-service`, `restricted-document-types-placeholder`,
 `document-type-restriction`, `ai-temperature-config`, `playground-deprecation`.
 
-Repo-hygiene steps that belong to this work:
+Repo hygiene for every step:
 
-- Delete `services/llmService.js` and `tests/test-llm-service.js` in their current form (Findings 5, 6).
-- Revert the `vitest` devDependency in `package.json` and its lockfile entries.
-- Register every new test in `TESTS`; run `node scripts/run-tests.js --all` — the registry drift
-  check only runs on `--all`.
+- Register each new test in `TESTS`; run `node scripts/run-tests.js --all` — the registry drift check
+  only runs on `--all`.
 - `npx eslint <changed>`, `npx prettier --check <changed>` — CI lints the files touched by the PR.
 
 ---
 
 ## Implementation Steps
 
-Each step ends with a green `node scripts/run-tests.js --all`. Steps 1-3 are pure preparation and
-carry no behavioral risk; the risk starts at step 5.
+Each step ends with a green `node scripts/run-tests.js --all`. Steps 1-2 are pure preparation and
+carry no behavioural risk; the risk starts at step 4.
 
-1. **Clean the slate.** Remove `services/llmService.js` and `tests/test-llm-service.js`; revert the
-   `vitest` devDependency and lockfile entries; keep the `llm-service` entry out of `TESTS` until a
-   runnable test replaces it. Move this plan and `services/AiServices.md` out of `services/` (they
-   are documents, not services — `docs/` or the PR body).
-2. **Freeze the contract in tests first (TDD).** Write `llm-analysis-result-contract` and
+1. **Freeze the contract in tests first (TDD).** Write `llm-analysis-result-contract` and
    `llm-ocr-fallback-phrases` against the **current** `openaiService`/`ollamaService`. They must pass
    before any production code moves — that is what makes them a regression net rather than a
    description of the new code.
-3. **Extract `customFieldsTemplate.js` and `promptBuilder.js`** with `llm-prompt-builder`, and have
+2. **Extract `customFieldsTemplate.js` and `promptBuilder.js`** with `llm-prompt-builder`, and have
    all four existing services call them. Six duplicates become one; no other behaviour changes.
    `prompt-existing-data-serialization` and the restriction tests are the guard.
-4. **Extract `responseParser.js`** (custom's brace matcher + `<think>` stripping + Ollama's sanitize
+3. **Extract `responseParser.js`** (custom's brace matcher + `<think>` stripping + Ollama's sanitize
    fallback + Ollama's object passthrough) with `llm-response-parser`, and route all four services
-   through it. Resolves Finding 9d.
-5. **Extract `analysisResult.js`** and the two `tokenBudget` strategies with `llm-token-budget`.
-   Fix Finding 9b (missing thumbnail) here, with `llm-missing-thumbnail-continues` written first.
-6. **Introduce the transports.** `chatCompletionsTransport.js` covering openai/azure/custom, and
+   through it. Resolves Finding 7d.
+4. **Extract `analysisResult.js`** and the two `tokenBudget` strategies with `llm-token-budget`.
+   Fix Finding 7b (missing thumbnail) here, with `llm-missing-thumbnail-continues` written first.
+5. **Introduce the transports.** `chatCompletionsTransport.js` covering openai/azure/custom, and
    `ollamaGenerateTransport.js` carrying the retry, `format`, `num_ctx`/`num_predict` and
    `done_reason` truncation check moved verbatim from `ollamaService.js:811-922`. Cover with
    `llm-transport-request-mapping`.
-7. **Introduce `documentAnalyzer.js`** and the four composition roots. Reduce
+6. **Introduce `documentAnalyzer.js`** and the four composition roots. Reduce
    `services/{openai,azure,custom,ollama}Service.js` to re-export shims. Add `llm-provider-parity`.
-   Fix Finding 9a here (Azure temperature) with its test.
-8. **Delete `services/manualService.js`** and repoint `POST /manual/analyze` and
+   Fix Finding 7a here (Azure temperature) with its test.
+7. **Delete `services/manualService.js`** and repoint `POST /manual/analyze` and
    `POST /manual/playground` (`routes/setup.js:7511`, `:7657`) at `AIServiceFactory.getService()`,
    collapsing both four-way switches. Note that `/manual/analyze` currently calls
    `documentModel.addOpenAIMetrics()` only on the OpenAI branch while `/manual/playground` calls it on
    three of four — decide explicitly whether metrics are recorded for every provider, and if the
    response shape changes, update the `@swagger` JSDoc and run `node scripts/regen-openapi.js`
    (the CI drift check will fail otherwise).
-9. **Verify Finding 9h** (unreachable `initialize()` branches in the former `openaiService`) and drop
+8. **Verify Finding 7h** (unreachable `initialize()` branches in the former `openaiService`) and drop
    the dead branches.
-10. **Full verification.** `node scripts/run-tests.js --all`; ESLint/Prettier on every changed file;
-    `node scripts/regen-openapi.js` + `git diff --exit-code OPENAPI/openapi.json`; a manual smoke run
-    of `/manual/analyze` and a scan cycle against a real provider, since no test exercises a live API.
+9. **Full verification.** `node scripts/run-tests.js --all`; ESLint/Prettier on every changed file;
+   `node scripts/regen-openapi.js` + `git diff --exit-code OPENAPI/openapi.json`; a manual smoke run
+   of `/manual/analyze` and a scan cycle against a real provider, since no test exercises a live API.
 
 ---
 
 ## Out of Scope / Follow-ups
 
-- **Relocating unrelated tests into `tests/services/`** (original step 8). Needs `findUnregisteredTests()`
-  made recursive first (Finding 11) and would bury this refactor in rename noise. Separate PR.
-- **Thumbnail caching inside `analyzeDocument`** (Finding 9i): the thumbnail is fetched, written to
-  disk and never used in the prompt. Behaviour preserved here; worth its own issue.
+- **Relocating tests into `tests/services/`.** Needs `findUnregisteredTests()` made recursive first
+  (Finding 8) and would bury this refactor in rename noise. Separate PR.
+- **Thumbnail caching inside `analyzeDocument`** (Finding 7i): the thumbnail is fetched, written to
+  disk and never used in the prompt. Behavior preserved here; worth its own issue.
 - **Vision/multimodal analysis**, which is presumably why the thumbnail is cached at all.
 - **`analyzePlayground` deduplication beyond the shared collaborators** — the endpoint is already
   deprecated (`routes/setup.js:7584`, `playground-deprecation` test); do not invest in it.
 - **A structured-output (`response_format`/`format`) path for the chat transports** to match what
   Ollama already does. Would likely remove most of `responseParser`, but it is a behaviour change
   needing its own validation against real providers.
+- **This plan document's location.** `/CLAUDE.md` asks for records in the commit body rather than
+  Markdown files, and `services/` holds services. Move it to `docs/` or fold it into the PR
+  description before the refactor lands.
 
 ## Risks
 
 | Risk                                                                         | Mitigation                                                                                                                           |
 | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
-| OCR fallback silently stops working because an error phrase changed          | `llm-ocr-fallback-phrases` written in step 2, before any code moves                                                                  |
-| A caller reads a field that the unified `document` object no longer carries  | `llm-provider-parity` compares full objects; `server.js:766`, `mistralOcrService.js:988`, `routes/setup.js:3167` reviewed explicitly |
-| Ollama's token arithmetic subtly changes and long documents start truncating | `charBudget` keeps the `/2` divisor and the real `num_ctx` formula; `ollama-response-limit` and `ollama-token-metrics` guard it      |
-| No test exercises a real provider API                                        | Step 10 keeps a manual smoke run against at least one live provider in the acceptance criteria                                       |
-| The refactor lands as one unreviewable diff                                  | Steps 3-7 are individually green and individually landable                                                                           |
+| OCR fallback silently stops working because an error phrase changed           | `llm-ocr-fallback-phrases` written in step 1, before any code moves                                                                  |
+| A caller reads a field that the unified `document` object no longer carries   | `llm-provider-parity` compares full objects; `server.js:766`, `mistralOcrService.js:988`, `routes/setup.js:3167` reviewed explicitly  |
+| Ollama's token arithmetic subtly changes and long documents start truncating  | `charBudget` keeps the `/2` divisor and the real `num_ctx` formula; `ollama-response-limit` and `ollama-token-metrics` guard it       |
+| No test exercises a real provider API                                        | Step 9 keeps a manual smoke run against at least one live provider in the acceptance criteria                                         |
+| The refactor lands as one unreviewable diff                                   | Steps 2-6 are individually green and individually landable                                                                           |

From 99206ee6e112fb4f38385a014f7cefb6f0b63a0c Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Tue, 1 Sep 2026 17:02:56 +0200
Subject: [PATCH 27/53] refactoring: check business outcome instead of
 implementation detail

---
 tests/test-response-truncation-detection.js | 195 +++++++++++++++-----
 1 file changed, 144 insertions(+), 51 deletions(-)

diff --git a/tests/test-response-truncation-detection.js b/tests/test-response-truncation-detection.js
index 82cf3144..3438c84b 100644
--- a/tests/test-response-truncation-detection.js
+++ b/tests/test-response-truncation-detection.js
@@ -8,12 +8,16 @@
  * the document to the OCR queue, which re-reads the PDF and re-runs the same
  * request against the same limit.
  *
- * The playground path is the seam used here: it takes the same client and the
- * same guard as the analysis path, but caches no thumbnail, so the stub is the
- * only collaborator involved.
+ * Both public seams are exercised per provider — analyzeDocument() and
+ * analyzePlayground() carry their own catch block, and the scan loop reads
+ * analysis.errorCode, so a service that raises the right error but drops the
+ * code on the way out records a generic failure and loses the OCR decision.
+ * The provider client is the only collaborator stubbed; the thumbnail cache is
+ * pre-seeded so the analysis path reaches the client without a Paperless call.
  */
 
 const assert = require('assert');
+const fs = require('fs').promises;
 
 process.env.AI_PROVIDER = 'openai';
 process.env.OPENAI_API_KEY = 'test-key';
@@ -27,9 +31,14 @@ process.env.CUSTOM_MODEL = 'test-model';
 process.env.SYSTEM_PROMPT = 'Analyse the document.';
 
 const { assertCompletionNotTruncated } = require('../services/serviceUtils');
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('../services/thumbnailCachePaths');
 const openaiService = require('../services/openaiService');
 const azureService = require('../services/azureService');
 const customService = require('../services/customService');
+const ollamaService = require('../services/ollamaService');
 
 let failed = 0;
 const check = async (label, fn) => {
@@ -51,6 +60,16 @@ const COMPLETE_ANSWER = JSON.stringify({
   language: 'de',
 });
 
+// A prefix of the answer above: valid JSON right up to the point it stops.
+const CUT_OFF_ANSWER = COMPLETE_ANSWER.slice(0, -12);
+
+// Own id so the seeded thumbnail cannot collide with a real cached document.
+const DOCUMENT_ID = 'test263truncation';
+
+// The response limit the services read from config.responseTokens; eval_count
+// reaching it is what tells Ollama's guard which limit bit.
+const RESPONSE_TOKENS = 1000;
+
 function completion(finishReason, content = COMPLETE_ANSWER) {
   return {
     choices: [{ message: { content }, finish_reason: finishReason }],
@@ -66,6 +85,52 @@ function stubClient(service, response) {
   };
 }
 
+/** Same for Ollama, which answers over axios instead of an SDK. */
+function stubOllamaClient(responseData) {
+  ollamaService.client = {
+    post: async () => ({ data: responseData }),
+    get: async () => ({ status: 200, data: { models: [] } }),
+  };
+}
+
+/**
+ * Per-provider arrangements for the same two events. Kept as thunks so each
+ * case re-stubs immediately before it runs and cannot inherit a stale client.
+ */
+function openAiCompatibleProvider(label, service) {
+  return {
+    label,
+    service,
+    arrangeCutOff: () =>
+      stubClient(service, completion('length', CUT_OFF_ANSWER)),
+    arrangeComplete: () => stubClient(service, completion('stop')),
+  };
+}
+
+const PROVIDERS = [
+  openAiCompatibleProvider('OpenAI', openaiService),
+  openAiCompatibleProvider('Azure', azureService),
+  openAiCompatibleProvider('Custom', customService),
+  {
+    label: 'Ollama',
+    service: ollamaService,
+    arrangeCutOff: () =>
+      stubOllamaClient({
+        response: CUT_OFF_ANSWER,
+        done_reason: 'length',
+        eval_count: RESPONSE_TOKENS,
+        prompt_eval_count: 900,
+      }),
+    arrangeComplete: () =>
+      stubOllamaClient({
+        response: COMPLETE_ANSWER,
+        done_reason: 'stop',
+        eval_count: 120,
+        prompt_eval_count: 900,
+      }),
+  },
+];
+
 (async () => {
   console.log('\n=== Response truncation detection ===');
 
@@ -110,65 +175,93 @@ function stubClient(service, response) {
     );
   });
 
-  /* --- each provider actually consults it -------------------------------- */
-
-  const services = [
-    ['OpenAI', openaiService],
-    ['Azure', azureService],
-    ['Custom', customService],
-  ];
-
-  await check('every analysis catch block carries the code onward', () => {
-    // The scan loop reads analysis.errorCode; a service that throws the right
-    // error but drops the code on the way out records a generic failure.
-    const fs = require('fs');
-    for (const name of [
-      'openaiService',
-      'azureService',
-      'customService',
-      'ollamaService',
-    ]) {
-      const source = fs.readFileSync(`services/${name}.js`, 'utf8');
-      const guards = (source.match(/errorCode: error\.code/g) || []).length;
-      const catches = (
-        source.match(
-          /document: \{ tags: \[\], correspondent: null \},\n\s*metrics: null,/g
-        ) || []
-      ).length;
-      assert.strictEqual(
-        guards,
-        catches,
-        `${name}: ${catches} analysis catch block(s) but ${guards} carry errorCode`
-      );
-    }
-  });
+  /* --- each provider actually consults it, on both public seams ---------- */
+
+  /* Pre-seed the thumbnail cache so analyzeDocument() gets past its caching
+     step without reaching Paperless. Ollama skips the step entirely once the
+     file is there, the OpenAI-compatible three take their fs.access() hit. */
+  await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+  await fs.writeFile(
+    getThumbnailCachePath(DOCUMENT_ID),
+    Buffer.from([0x89, 0x50, 0x4e, 0x47])
+  );
 
-  for (const [label, service] of services) {
+  /* The failure shape the scan loop branches on: no metadata to write back,
+     no metrics to record, and the code that decides OCR fallback. Asserted
+     rather than read out of the source, so it survives the code moving. */
+  const assertReportsCutOff = (analysis) => {
+    assert.strictEqual(analysis.errorCode, 'ai_response_truncated');
+    assert.match(analysis.error, /stopped generating/);
+    assert.deepStrictEqual(analysis.document, {
+      tags: [],
+      correspondent: null,
+    });
+    assert.strictEqual(analysis.metrics, null);
+  };
+
+  for (const { label, service, arrangeCutOff, arrangeComplete } of PROVIDERS) {
     await check(
-      `${label}: a cut-off answer is reported, not parsed`,
+      `${label}: analyzeDocument reports a cut-off answer with its code`,
       async () => {
         // Truncated JSON would otherwise fail to parse and be blamed on the model.
-        stubClient(
-          service,
-          completion('length', COMPLETE_ANSWER.slice(0, -12))
+        arrangeCutOff();
+        const analysis = await service.analyzeDocument(
+          'Rechnung',
+          [],
+          [],
+          [],
+          DOCUMENT_ID
+        );
+        assertReportsCutOff(analysis);
+      }
+    );
+
+    await check(
+      `${label}: analyzeDocument passes a complete answer through`,
+      async () => {
+        arrangeComplete();
+        const analysis = await service.analyzeDocument(
+          'Rechnung',
+          [],
+          [],
+          [],
+          DOCUMENT_ID
         );
+        assert.strictEqual(analysis.error, undefined);
+        assert.strictEqual(analysis.errorCode, undefined);
+        assert.strictEqual(
+          analysis.document.correspondent,
+          'Telekom Deutschland GmbH'
+        );
+      }
+    );
+
+    await check(
+      `${label}: analyzePlayground reports a cut-off answer with its code`,
+      async () => {
+        arrangeCutOff();
         const analysis = await service.analyzePlayground('Rechnung', 'Analyse');
-        assert.strictEqual(analysis.errorCode, 'ai_response_truncated');
-        assert.match(analysis.error, /stopped generating/);
+        assertReportsCutOff(analysis);
       }
     );
 
-    await check(`${label}: a complete answer is unaffected`, async () => {
-      stubClient(service, completion('stop'));
-      const analysis = await service.analyzePlayground('Rechnung', 'Analyse');
-      assert.strictEqual(analysis.error, undefined);
-      assert.strictEqual(
-        analysis.document.correspondent,
-        'Telekom Deutschland GmbH'
-      );
-    });
+    await check(
+      `${label}: analyzePlayground passes a complete answer through`,
+      async () => {
+        arrangeComplete();
+        const analysis = await service.analyzePlayground('Rechnung', 'Analyse');
+        assert.strictEqual(analysis.error, undefined);
+        assert.strictEqual(analysis.errorCode, undefined);
+        assert.strictEqual(
+          analysis.document.correspondent,
+          'Telekom Deutschland GmbH'
+        );
+      }
+    );
   }
 
+  await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
+
   if (failed > 0) {
     console.error(`\n${failed} truncation detection case(s) failed`);
     process.exit(1);

From 4580e4ff17d20c84ed5e9d9fed03395b95c59531 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Tue, 1 Sep 2026 17:42:47 +0200
Subject: [PATCH 28/53] Bugfix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Fixservices/thumbnailCache.js (new) — cacheThumbnail(id) is best-effort by design: it never throws, never interrupts the caller, and returns a boolean. An invalid ID, missing thumbnail, or unreachable Paperless results in a warning + false. All four providers now simply call await cacheThumbnail(id). Ollama's _handleThumbnailCaching() is gone, along with the no-longer-used imports (paperlessService, THUMBNAIL_CACHE_DIR, getThumbnailCachePath, and additionally fs in Ollama).The scope was larger than the findingI need to state this clearly: I didn't just fix 7j. Finding 7b (return; → undefined when a thumbnail is missing) sits three lines away from 7j and shares the same root cause—a best-effort side effect written as a blocking precondition. Leaving it would have caused a crash one line later after an isolated 7j fix. Furthermore, the Red run showed that the finding was formulated too narrowly: contrary to the planning line, Ollama was also affected. Its !id guard does not catch truthy-but-invalid IDs like [] and '///', and a failing fetch swallowed the error but still lost the analysis. So, it affected all four providers, not three.Teststests/test-thumbnail-optional-analysis.js — 32 cases, written red in TDD (24 red, including the 3 unexpected Ollama cases), now green. Per provider: 5 invalid IDs, thumbnail not found, fetch fails, thumbnail present → is actually cached. Registered in TESTS and under AREAS.processing.Full run: 69 passed, 7 skipped (Server/pdftoppm), 10 failed — all 10 are pre-existing and environment-related. Previously it was 11; response-truncation-detection is now green thanks to the fix from the last round.
---
 scripts/run-tests.js                      |   2 +
 services/ManualServiceRefactoring.md      |  93 +++++----
 services/azureService.js                  |  25 +--
 services/customService.js                 |  26 +--
 services/ollamaService.js                 |  41 +---
 services/openaiService.js                 |  26 +--
 services/thumbnailCache.js                |  62 ++++++
 tests/test-thumbnail-optional-analysis.js | 232 ++++++++++++++++++++++
 8 files changed, 367 insertions(+), 140 deletions(-)
 create mode 100644 services/thumbnailCache.js
 create mode 100644 tests/test-thumbnail-optional-analysis.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index 264fc7dc..11ec0c5a 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -80,6 +80,7 @@ const TESTS = {
   'setup-remote-guard': 'test-setup-remote-guard.js',
   'thumbnail-auth-guard': 'test-thumbnail-auth-guard.js',
   'thumbnail-startup-migration': 'test-thumbnail-startup-migration.js',
+  'thumbnail-optional-analysis': 'test-thumbnail-optional-analysis.js',
   'restriction-service': 'test-restriction-service.js',
   'updated-service': 'test-updated-service.js',
   'prompt-existing-data-serialization':
@@ -179,6 +180,7 @@ const AREAS = {
     'scanner-startup-resilience',
     'paperless-unreachable-banner',
     'thumbnail-startup-migration',
+    'thumbnail-optional-analysis',
     'ai-temperature-config',
     'custom-field-monetary-normalization',
     'ollama-token-metrics',
diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 85269106..390f7f9c 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -57,7 +57,7 @@ Further evidence that `manualService.js` was abandoned rather than maintained:
   `process.env.SYSTEM_PROMPT` (`manualService.js:218`).
 
 **Decision:** delete `services/manualService.js` as part of this work and point the two
-`/manual/*` endpoints at `AIServiceFactory.getService()`. Do not port its behaviour — it is worse
+`/manual/*` endpoints at `AIServiceFactory.getService()`. Do not port its behavior — it is worse
 than the dedicated services on every axis: no token budgeting, no content truncation, no restriction
 placeholders, no structured output, no retry, no metrics.
 
@@ -151,22 +151,37 @@ may arrive as a **JS object** rather than a string (`:929-949`).
 A single-string `callAPI(prompt)` cannot carry any of that, which is why `CompletionRequest` below is
 a record rather than a string.
 
-### 7. Behavioural drift between the four services — each item needs an explicit decision
+### 7. behavioral drift between the four services — each item needs an explicit decision
 
 The refactor collapses four implementations into one, which forces a choice wherever they differ
 today. These are the actual bugs the work will surface:
 
-| #   | Drift                                                                                            | Location                                                                                                                                                                        | Proposed resolution                                                                                                                                               |
-| --- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`        | `azureService.js:258`, `:426`                                                                                                                                                   | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                    |
-| b   | `return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined` | `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                             | Adopt Ollama's behaviour (`_handleThumbnailCaching`, `:708-725`): warn and continue. This is a latent crash.                                                       |
-| c   | "Insufficient content" early return exists only in OpenAI                                        | `openaiService.js:310-332`, `:505-527`                                                                                                                                          | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                           |
-| d   | Three different JSON parsers                                                                     | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | One parser built from custom's `_extractFirstJsonValue` + `<think>` stripping + Ollama's sanitize fallback; keep Ollama's object passthrough as a pre-step.         |
-| e   | Only custom normalizes timeouts                                                                  | `customService.js:428-436`                                                                                                                                                      | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                               |
-| f   | `generateText` max-token handling differs four ways                                              | openai: none; `azureService.js:528`: `max_tokens`; `customService.js:625-654`: clamped; `ollamaService.js:1105`: `num_predict`                                                   | Keep custom's clamping as the shared rule; the transport maps it to its own knob.                                                                                  |
-| g   | `checkStatus` differs by nature                                                                  | openai/custom `models.list()`; `azureService.js:574-580` REST GET; `ollamaService.js:1143` `/api/ps`                                                                             | Stays in the transport. Not business logic.                                                                                                                       |
-| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`          | `openaiService.js:33-44`                                                                                                                                                        | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop. |
-| i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt    | all four                                                                                                                                                                        | Out of scope. Preserve behaviour; note as a follow-up.                                                                                                            |
+| #   | Drift                                                                                                                       | Location                                                                                                                                                                         | Proposed resolution                                                                                                                                                                                 |
+| --- | --------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`                                   | `azureService.js:258`, `:426`                                                                                                                                                    | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                                                     |
+| b   | ~~`return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined`~~ **resolved**          | was `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                          | Fixed ahead of the refactor together with row j — same statement block, same root cause. See the note below the table.                                                                              |
+| c   | "Insufficient content" early return exists only in OpenAI                                                                   | `openaiService.js:310-332`, `:505-527`                                                                                                                                           | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                                                            |
+| d   | Three different JSON parsers                                                                                                | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | One parser built from custom's `_extractFirstJsonValue` + `<think>` stripping + Ollama's sanitize fallback; keep Ollama's object passthrough as a pre-step.                                         |
+| e   | Only custom normalizes timeouts                                                                                             | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                                                                |
+| f   | `generateText` max-token handling differs four ways                                                                         | openai: none; `azureService.js:528`: `max_tokens`; `customService.js:625-654`: clamped; `ollamaService.js:1105`: `num_predict`                                                   | Keep custom's clamping as the shared rule; the transport maps it to its own knob.                                                                                                                   |
+| g   | `checkStatus` differs by nature                                                                                             | openai/custom `models.list()`; `azureService.js:574-580` REST GET; `ollamaService.js:1143` `/api/ps`                                                                             | Stays in the transport. Not business logic.                                                                                                                                                         |
+| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`                                    | `openaiService.js:33-44`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop.                                  |
+| i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt                              | all four                                                                                                                                                                         | Still out of scope as a _placement_ question, but the four copies are now one call to `services/thumbnailCache.js`. Moving it out of `analyzeDocument` is a one-line change once a caller wants it. |
+| j   | ~~`getThumbnailCachePath(id)` runs **outside** the try block, so an invalid id makes `analyzeDocument` throw~~ **resolved** | was `openaiService.js:63`, `azureService.js:52`, `customService.js:147`                                                                                                          | Fixed ahead of the refactor. See the note below the table.                                                                                                                                          |
+
+**Rows b and j were fixed before the refactor started.** They shared one root cause — a best-effort
+side effect written as a blocking precondition — and sat within three lines of each other, so
+fixing one alone would have left the other a line away. Writing the test first showed the drift was
+also wider than row j claimed: Ollama's `!id` guard misses truthy-but-unusable ids such as `[]` and
+`'///'`, and its `_handleThumbnailCaching` swallowed a failing fetch but still lost the analysis, so
+all four providers were affected, not three.
+
+The fix consolidates the four copies into `services/thumbnailCache.js`, whose `cacheThumbnail(id)`
+is best-effort by construction: it never throws and never short-circuits its caller, it returns a
+boolean, and an unusable id, a missing thumbnail or an unreachable Paperless each produce a warning
+and `false`. Ollama's `_handleThumbnailCaching()` is gone. `tests/test-thumbnail-optional-analysis.js`
+(32 cases, registered under `processing`) pins the contract for all four providers: whatever the
+thumbnail does, `analyzeDocument` returns a full analysis.
 
 ### 8. Moving tests into `tests/services/` would break the CI drift guard
 
@@ -375,17 +390,17 @@ const fakeTransport = {
 
 Planned tests (each registered in `TESTS` in `scripts/run-tests.js` and in an `AREAS` entry):
 
-| Test name                         | Asserts                                                                                                                                                                                                                      |
-| --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `llm-response-parser`             | code fences stripped; `<think>…</think>` stripped; brace-matched extraction from surrounding prose; trailing-comma/unquoted-key sanitize fallback; unparseable input throws with the exact phrase `Invalid JSON response from API` |
-| `llm-analysis-result-contract`    | success returns `{document, metrics, truncated}`; failure returns `{document:{tags:[],correspondent:null}, metrics:null, error, errorCode}` and **never throws**                                                              |
-| `llm-ocr-fallback-phrases`        | every phrase in `shouldQueueForOcrOnAiError` is still produced by the refactored paths — the regression guard for Finding 4                                                                                                   |
-| `llm-prompt-builder`              | `%CUSTOMFIELDS%` rendering for date/boolean/other; `useExistingData` branch; `USE_PROMPT_TAGS` override; `customPrompt` override; restriction placeholders delegated to `RestrictionPromptService`; external API data appended |
-| `llm-token-budget`                | `tiktokenBudget.reserve` throws `Token limit exceeded: …` when `availableTokens <= 0`; `charBudget` uses the `/2` divisor and `min(promptTokens + responseTokens, tokenLimit)` for `num_ctx` — Finding 5                       |
-| `llm-transport-request-mapping`   | chat transports send `messages[]` with system+user and omit `temperature` for `o3-mini`; Ollama transport sends `format`, `num_ctx`, `num_predict`, `think:false` unless `OLLAMA_THINK=true`                                   |
-| `llm-azure-temperature-wiring`    | Azure analysis requests carry `config.aiTemperatureAnalysis`, not a hardcoded `0.3` — Finding 7a                                                                                                                              |
-| `llm-missing-thumbnail-continues` | a missing thumbnail yields a full result object, never `undefined` — Finding 7b                                                                                                                                              |
-| `llm-provider-parity`             | all four composition roots expose `analyzeDocument`/`analyzePlayground`/`generateText`/`checkStatus` and, given the same fake transport payload, produce byte-identical `document` objects                                    |
+| Test name                             | Asserts                                                                                                                                                                                                                                                             |
+| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `llm-response-parser`                 | code fences stripped; `<think>…</think>` stripped; brace-matched extraction from surrounding prose; trailing-comma/unquoted-key sanitize fallback; unparseable input throws with the exact phrase `Invalid JSON response from API`                                  |
+| `llm-analysis-result-contract`        | success returns `{document, metrics, truncated}`; failure returns `{document:{tags:[],correspondent:null}, metrics:null, error, errorCode}` and **never throws**                                                                                                    |
+| `llm-ocr-fallback-phrases`            | every phrase in `shouldQueueForOcrOnAiError` is still produced by the refactored paths — the regression guard for Finding 4                                                                                                                                         |
+| `llm-prompt-builder`                  | `%CUSTOMFIELDS%` rendering for date/boolean/other; `useExistingData` branch; `USE_PROMPT_TAGS` override; `customPrompt` override; restriction placeholders delegated to `RestrictionPromptService`; external API data appended                                      |
+| `llm-token-budget`                    | `tiktokenBudget.reserve` throws `Token limit exceeded: …` when `availableTokens <= 0`; `charBudget` uses the `/2` divisor and `min(promptTokens + responseTokens, tokenLimit)` for `num_ctx` — Finding 5                                                            |
+| `llm-transport-request-mapping`       | chat transports send `messages[]` with system+user and omit `temperature` for `o3-mini`; Ollama transport sends `format`, `num_ctx`, `num_predict`, `think:false` unless `OLLAMA_THINK=true`                                                                        |
+| `llm-azure-temperature-wiring`        | Azure analysis requests carry `config.aiTemperatureAnalysis`, not a hardcoded `0.3` — Finding 7a                                                                                                                                                                    |
+| ~~`llm-missing-thumbnail-continues`~~ | **already delivered** as `tests/test-thumbnail-optional-analysis.js` (Findings 7b/7j). Keep it green through the refactor instead of writing a new one; it exercises `analyzeDocument` end to end, so the composition roots must keep calling `cacheThumbnail(id)`. |
+| `llm-provider-parity`                 | all four composition roots expose `analyzeDocument`/`analyzePlayground`/`generateText`/`checkStatus` and, given the same fake transport payload, produce byte-identical `document` objects                                                                          |
 
 Existing tests that must stay green without modification (they are the real acceptance criteria):
 `ocr-fallback-ai-errors`, `response-truncation-detection`, `ollama-response-limit`,
@@ -404,20 +419,21 @@ Repo hygiene for every step:
 ## Implementation Steps
 
 Each step ends with a green `node scripts/run-tests.js --all`. Steps 1-2 are pure preparation and
-carry no behavioural risk; the risk starts at step 4.
+carry no behavioral risk; the risk starts at step 4.
 
 1. **Freeze the contract in tests first (TDD).** Write `llm-analysis-result-contract` and
    `llm-ocr-fallback-phrases` against the **current** `openaiService`/`ollamaService`. They must pass
    before any production code moves — that is what makes them a regression net rather than a
    description of the new code.
 2. **Extract `customFieldsTemplate.js` and `promptBuilder.js`** with `llm-prompt-builder`, and have
-   all four existing services call them. Six duplicates become one; no other behaviour changes.
+   all four existing services call them. Six duplicates become one; no other behavior changes.
    `prompt-existing-data-serialization` and the restriction tests are the guard.
 3. **Extract `responseParser.js`** (custom's brace matcher + `<think>` stripping + Ollama's sanitize
    fallback + Ollama's object passthrough) with `llm-response-parser`, and route all four services
    through it. Resolves Finding 7d.
 4. **Extract `analysisResult.js`** and the two `tokenBudget` strategies with `llm-token-budget`.
-   Fix Finding 7b (missing thumbnail) here, with `llm-missing-thumbnail-continues` written first.
+   Findings 7b/7j are already fixed; `services/thumbnailCache.js` is the collaborator the
+   composition roots inject in step 6, and `thumbnail-optional-analysis` must stay green throughout.
 5. **Introduce the transports.** `chatCompletionsTransport.js` covering openai/azure/custom, and
    `ollamaGenerateTransport.js` carrying the retry, `format`, `num_ctx`/`num_predict` and
    `done_reason` truncation check moved verbatim from `ollamaService.js:811-922`. Cover with
@@ -427,7 +443,12 @@ carry no behavioural risk; the risk starts at step 4.
    Fix Finding 7a here (Azure temperature) with its test.
 7. **Delete `services/manualService.js`** and repoint `POST /manual/analyze` and
    `POST /manual/playground` (`routes/setup.js:7511`, `:7657`) at `AIServiceFactory.getService()`,
-   collapsing both four-way switches. Note that `/manual/analyze` currently calls
+   collapsing both four-way switches. Clean up what used to trigger Finding 7j while here: all four
+   branches pass `id || []` (`routes/setup.js:7535`, `:7550`, `:7559`, `:7568`), and `String([])` is
+   `''`, which `getThumbnailCachePath()` rejects. That no longer costs the analysis — the request now
+   succeeds with a warning — but `id || []` is still a nonsense default; pass the id through as-is,
+   or require it. Note that
+   `/manual/analyze` currently calls
    `documentModel.addOpenAIMetrics()` only on the OpenAI branch while `/manual/playground` calls it on
    three of four — decide explicitly whether metrics are recorded for every provider, and if the
    response shape changes, update the `@swagger` JSDoc and run `node scripts/regen-openapi.js`
@@ -445,12 +466,14 @@ carry no behavioural risk; the risk starts at step 4.
 - **Relocating tests into `tests/services/`.** Needs `findUnregisteredTests()` made recursive first
   (Finding 8) and would bury this refactor in rename noise. Separate PR.
 - **Thumbnail caching inside `analyzeDocument`** (Finding 7i): the thumbnail is fetched, written to
-  disk and never used in the prompt. Behavior preserved here; worth its own issue.
+  disk and never used in the prompt. Behavior preserved here; worth its own issue. It is now a
+  single `cacheThumbnail(id)` call per provider rather than four inline copies, so hoisting it to
+  the caller is a small change whenever someone wants to make it.
 - **Vision/multimodal analysis**, which is presumably why the thumbnail is cached at all.
 - **`analyzePlayground` deduplication beyond the shared collaborators** — the endpoint is already
   deprecated (`routes/setup.js:7584`, `playground-deprecation` test); do not invest in it.
 - **A structured-output (`response_format`/`format`) path for the chat transports** to match what
-  Ollama already does. Would likely remove most of `responseParser`, but it is a behaviour change
+  Ollama already does. Would likely remove most of `responseParser`, but it is a behavior change
   needing its own validation against real providers.
 - **This plan document's location.** `/CLAUDE.md` asks for records in the commit body rather than
   Markdown files, and `services/` holds services. Move it to `docs/` or fold it into the PR
@@ -460,8 +483,8 @@ carry no behavioural risk; the risk starts at step 4.
 
 | Risk                                                                         | Mitigation                                                                                                                           |
 | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
-| OCR fallback silently stops working because an error phrase changed           | `llm-ocr-fallback-phrases` written in step 1, before any code moves                                                                  |
-| A caller reads a field that the unified `document` object no longer carries   | `llm-provider-parity` compares full objects; `server.js:766`, `mistralOcrService.js:988`, `routes/setup.js:3167` reviewed explicitly  |
-| Ollama's token arithmetic subtly changes and long documents start truncating  | `charBudget` keeps the `/2` divisor and the real `num_ctx` formula; `ollama-response-limit` and `ollama-token-metrics` guard it       |
-| No test exercises a real provider API                                        | Step 9 keeps a manual smoke run against at least one live provider in the acceptance criteria                                         |
-| The refactor lands as one unreviewable diff                                   | Steps 2-6 are individually green and individually landable                                                                           |
+| OCR fallback silently stops working because an error phrase changed          | `llm-ocr-fallback-phrases` written in step 1, before any code moves                                                                  |
+| A caller reads a field that the unified `document` object no longer carries  | `llm-provider-parity` compares full objects; `server.js:766`, `mistralOcrService.js:988`, `routes/setup.js:3167` reviewed explicitly |
+| Ollama's token arithmetic subtly changes and long documents start truncating | `charBudget` keeps the `/2` divisor and the real `num_ctx` formula; `ollama-response-limit` and `ollama-token-metrics` guard it      |
+| No test exercises a real provider API                                        | Step 9 keeps a manual smoke run against at least one live provider in the acceptance criteria                                        |
+| The refactor lands as one unreviewable diff                                  | Steps 2-6 are individually green and individually landable                                                                           |
diff --git a/services/azureService.js b/services/azureService.js
index 670cf87f..3df626e1 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -12,10 +12,7 @@ const AzureOpenAI = require('openai').AzureOpenAI;
 const config = require('../config/config');
 const fs = require('fs').promises;
 const path = require('path');
-const {
-  THUMBNAIL_CACHE_DIR,
-  getThumbnailCachePath,
-} = require('./thumbnailCachePaths');
+const { cacheThumbnail } = require('./thumbnailCache');
 const RestrictionPromptService = require('./restrictionPromptService');
 const responseLogPath = path.join(
   process.cwd(),
@@ -49,7 +46,6 @@ class AzureOpenAIService {
     customPrompt = null,
     options = {}
   ) {
-    const cachePath = getThumbnailCachePath(id);
     try {
       this.initialize();
       const now = new Date();
@@ -62,23 +58,8 @@ class AzureOpenAIService {
         throw new Error('AzureOpenAI client not initialized');
       }
 
-      // Handle thumbnail caching
-      try {
-        await fs.access(cachePath);
-        console.log('[DEBUG] Thumbnail already cached');
-      } catch {
-        console.log('Thumbnail not cached, fetching from Paperless');
-        const paperlessService = require('./paperlessService');
-        const thumbnailData = await paperlessService.getThumbnailImage(id);
-
-        if (!thumbnailData) {
-          console.warn('Thumbnail not found');
-          return;
-        }
-
-        await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
-        await fs.writeFile(cachePath, thumbnailData);
-      }
+      // Cache the thumbnail for the UI; never let it affect the analysis.
+      await cacheThumbnail(id);
 
       // Format existing data - callers may hand over entity objects or plain names
       const existingTagNames = toNameList(existingTags).join(', ');
diff --git a/services/customService.js b/services/customService.js
index ecd3c301..9a758640 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -10,13 +10,9 @@ const {
 } = require('./serviceUtils');
 const OpenAI = require('openai');
 const config = require('../config/config');
-const paperlessService = require('./paperlessService');
 const fs = require('fs').promises;
 const path = require('path');
-const {
-  THUMBNAIL_CACHE_DIR,
-  getThumbnailCachePath,
-} = require('./thumbnailCachePaths');
+const { cacheThumbnail } = require('./thumbnailCache');
 const RestrictionPromptService = require('./restrictionPromptService');
 const responseLogPath = path.join(
   process.cwd(),
@@ -144,7 +140,6 @@ class CustomOpenAIService {
     customPrompt = null,
     options = {}
   ) {
-    const cachePath = getThumbnailCachePath(id);
     try {
       this.initialize();
       const now = new Date();
@@ -157,23 +152,8 @@ class CustomOpenAIService {
         throw new Error('Custom OpenAI client not initialized');
       }
 
-      // Handle thumbnail caching
-      try {
-        await fs.access(cachePath);
-        console.log('[DEBUG] Thumbnail already cached');
-      } catch {
-        console.log('Thumbnail not cached, fetching from Paperless');
-
-        const thumbnailData = await paperlessService.getThumbnailImage(id);
-
-        if (!thumbnailData) {
-          console.warn('Thumbnail not found');
-          return;
-        }
-
-        await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
-        await fs.writeFile(cachePath, thumbnailData);
-      }
+      // Cache the thumbnail for the UI; never let it affect the analysis.
+      await cacheThumbnail(id);
 
       // Format existing data - callers may hand over entity objects or plain names
       const existingTagNames = toNameList(existingTags).join(', ');
diff --git a/services/ollamaService.js b/services/ollamaService.js
index 72b4b2d6..7bc6ba7e 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -1,17 +1,7 @@
-const {
-  writePromptToFile,
-  toNameList,
-  truncateToTokenLimit,
-} = require('./serviceUtils');
+const { writePromptToFile, toNameList } = require('./serviceUtils');
 const axios = require('axios');
 const config = require('../config/config');
-const fs = require('fs').promises;
-const paperlessService = require('./paperlessService');
-const os = require('os');
-const {
-  THUMBNAIL_CACHE_DIR,
-  getThumbnailCachePath,
-} = require('./thumbnailCachePaths');
+const { cacheThumbnail } = require('./thumbnailCache');
 const RestrictionPromptService = require('./restrictionPromptService');
 
 /**
@@ -111,8 +101,8 @@ class OllamaService {
       // Truncate content if needed
       content = this._truncateContent(content);
 
-      // Cache thumbnail
-      await this._handleThumbnailCaching(id);
+      // Cache the thumbnail for the UI; never let it affect the analysis.
+      await cacheThumbnail(id);
 
       // Get external API data if available and validate it
       let externalApiData = options.externalApiData || null;
@@ -701,29 +691,6 @@ class OllamaService {
     return numCtx;
   }
 
-  /**
-   * Handle thumbnail caching for documents
-   * @param {string} id - Document ID
-   */
-  async _handleThumbnailCaching(id) {
-    if (!id) return;
-
-    const cachePath = getThumbnailCachePath(id);
-    try {
-      await fs.access(cachePath);
-      console.log('[DEBUG] Thumbnail already cached');
-    } catch {
-      console.log('Thumbnail not cached, fetching from Paperless');
-      const thumbnailData = await paperlessService.getThumbnailImage(id);
-      if (!thumbnailData) {
-        console.warn('Thumbnail not found');
-        return;
-      }
-      await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
-      await fs.writeFile(cachePath, thumbnailData);
-    }
-  }
-
   /**
    * Build request headers, adding bearer auth when an API key is set.
    * The key is read at request time, so runtime config changes apply
diff --git a/services/openaiService.js b/services/openaiService.js
index a941069f..a734fd4e 100644
--- a/services/openaiService.js
+++ b/services/openaiService.js
@@ -9,13 +9,9 @@ const {
 } = require('./serviceUtils');
 const OpenAI = require('openai');
 const config = require('../config/config');
-const paperlessService = require('./paperlessService');
 const fs = require('fs').promises;
 const path = require('path');
-const {
-  THUMBNAIL_CACHE_DIR,
-  getThumbnailCachePath,
-} = require('./thumbnailCachePaths');
+const { cacheThumbnail } = require('./thumbnailCache');
 const RestrictionPromptService = require('./restrictionPromptService');
 const responseLogPath = path.join(
   process.cwd(),
@@ -60,7 +56,6 @@ class OpenAIService {
     customPrompt = null,
     options = {}
   ) {
-    const cachePath = getThumbnailCachePath(id);
     try {
       this.initialize();
       const now = new Date();
@@ -73,23 +68,8 @@ class OpenAIService {
         throw new Error('OpenAI client not initialized');
       }
 
-      // Handle thumbnail caching
-      try {
-        await fs.access(cachePath);
-        console.log('[DEBUG] Thumbnail already cached');
-      } catch {
-        console.log('Thumbnail not cached, fetching from Paperless');
-
-        const thumbnailData = await paperlessService.getThumbnailImage(id);
-
-        if (!thumbnailData) {
-          console.warn('Thumbnail not found');
-          return;
-        }
-
-        await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
-        await fs.writeFile(cachePath, thumbnailData);
-      }
+      // Cache the thumbnail for the UI; never let it affect the analysis.
+      await cacheThumbnail(id);
 
       // Format existing data - callers may hand over entity objects or plain names
       const existingTagNames = toNameList(existingTags).join(', ');
diff --git a/services/thumbnailCache.js b/services/thumbnailCache.js
new file mode 100644
index 00000000..61b40b3a
--- /dev/null
+++ b/services/thumbnailCache.js
@@ -0,0 +1,62 @@
+const fs = require('fs').promises;
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('./thumbnailCachePaths');
+
+/**
+ * Cache a document's Paperless-ngx thumbnail for the UI.
+ *
+ * Best effort by contract: the thumbnail never enters the AI prompt, so no
+ * outcome of this step may decide whether the document gets analysed. All four
+ * AI services used to let it decide, each in its own way — an id that
+ * getThumbnailCachePath() rejects threw straight out of analyzeDocument()
+ * (which the scan loop does not expect, and which answered 500 on every
+ * POST /manual/analyze without an id), a thumbnail Paperless did not have
+ * returned `undefined` from it, and an unreachable Paperless turned into an
+ * analysis failure. One implementation for all four keeps them from drifting
+ * apart again.
+ *
+ * @param {string|number} id - Paperless-ngx document ID
+ * @returns {Promise<boolean>} true when a thumbnail is cached afterwards
+ */
+async function cacheThumbnail(id) {
+  let cachePath;
+  try {
+    cachePath = getThumbnailCachePath(id);
+  } catch (error) {
+    console.warn(
+      `[WARNING] Skipping thumbnail cache for document ${JSON.stringify(id)}: ${error.message}`
+    );
+    return false;
+  }
+
+  try {
+    await fs.access(cachePath);
+    console.log('[DEBUG] Thumbnail already cached');
+    return true;
+  } catch {
+    console.log('Thumbnail not cached, fetching from Paperless');
+  }
+
+  try {
+    // Required here rather than at module load: paperlessService reaches back
+    // into aiServiceFactory, and the AI services are what call this.
+    const paperlessService = require('./paperlessService');
+    const thumbnailData = await paperlessService.getThumbnailImage(id);
+
+    if (!thumbnailData) {
+      console.warn('Thumbnail not found');
+      return false;
+    }
+
+    await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+    await fs.writeFile(cachePath, thumbnailData);
+    return true;
+  } catch (error) {
+    console.warn(`[WARNING] Thumbnail caching failed: ${error.message}`);
+    return false;
+  }
+}
+
+module.exports = { cacheThumbnail };
diff --git a/tests/test-thumbnail-optional-analysis.js b/tests/test-thumbnail-optional-analysis.js
new file mode 100644
index 00000000..898516fc
--- /dev/null
+++ b/tests/test-thumbnail-optional-analysis.js
@@ -0,0 +1,232 @@
+/**
+ * Thumbnail caching must not decide whether a document gets analysed.
+ *
+ * analyzeDocument() caches the document thumbnail for the UI, but the
+ * thumbnail never enters the prompt — so every outcome of that side step has
+ * to leave the analysis itself alone. Two ways it did not:
+ *
+ * 1. getThumbnailCachePath() ran outside the try block, so an id it rejects
+ *    ('' after sanitizing, null, undefined) made analyzeDocument() *throw*
+ *    instead of returning a result. The scan loop reads analysis.error and
+ *    never expects a throw; POST /manual/analyze passes `id || []`, which is
+ *    exactly such a value, and answered 500 for every request without an id.
+ * 2. A thumbnail Paperless does not have returned `undefined` from
+ *    analyzeDocument(), and every caller immediately reads `.document` on it.
+ *
+ * Ollama already treated the step as best effort (_handleThumbnailCaching
+ * returns early on a falsy id); it is in here to keep the four providers from
+ * drifting apart again.
+ */
+
+const assert = require('assert');
+const fs = require('fs').promises;
+
+process.env.AI_PROVIDER = 'openai';
+process.env.OPENAI_API_KEY = 'test-key';
+process.env.OPENAI_MODEL = 'gpt-4';
+process.env.AZURE_API_KEY = 'test-key';
+process.env.AZURE_ENDPOINT = 'https://example.invalid';
+process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment';
+process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1';
+process.env.CUSTOM_API_KEY = 'test-key';
+process.env.CUSTOM_MODEL = 'test-model';
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('../services/thumbnailCachePaths');
+const paperlessService = require('../services/paperlessService');
+const openaiService = require('../services/openaiService');
+const azureService = require('../services/azureService');
+const customService = require('../services/customService');
+const ollamaService = require('../services/ollamaService');
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+const ANSWER = {
+  title: 'Telekom invoice July',
+  correspondent: 'Telekom Deutschland GmbH',
+  tags: ['invoice'],
+  document_type: 'Rechnung',
+  document_date: '2026-09-03',
+  language: 'de',
+};
+
+const CHAT_ANSWER = {
+  choices: [
+    { message: { content: JSON.stringify(ANSWER) }, finish_reason: 'stop' },
+  ],
+  usage: { prompt_tokens: 900, completion_tokens: 120, total_tokens: 1020 },
+};
+
+const OLLAMA_ANSWER = {
+  response: JSON.stringify(ANSWER),
+  done_reason: 'stop',
+  eval_count: 120,
+  prompt_eval_count: 900,
+};
+
+const PROVIDERS = [
+  {
+    label: 'OpenAI',
+    service: openaiService,
+    stub: () => {
+      openaiService.client = {
+        chat: { completions: { create: async () => CHAT_ANSWER } },
+      };
+    },
+  },
+  {
+    label: 'Azure',
+    service: azureService,
+    stub: () => {
+      azureService.client = {
+        chat: { completions: { create: async () => CHAT_ANSWER } },
+      };
+    },
+  },
+  {
+    label: 'Custom',
+    service: customService,
+    stub: () => {
+      customService.client = {
+        chat: { completions: { create: async () => CHAT_ANSWER } },
+      };
+    },
+  },
+  {
+    label: 'Ollama',
+    service: ollamaService,
+    stub: () => {
+      ollamaService.client = { post: async () => ({ data: OLLAMA_ANSWER }) };
+    },
+  },
+];
+
+/** The document came back analysed, whatever the thumbnail step did. */
+function assertAnalysed(analysis) {
+  assert.ok(analysis, 'analyzeDocument returned nothing at all');
+  assert.strictEqual(analysis.error, undefined);
+  assert.strictEqual(
+    analysis.document.correspondent,
+    'Telekom Deutschland GmbH'
+  );
+  assert.deepStrictEqual(analysis.document.tags, ['invoice']);
+}
+
+const originalGetThumbnailImage = paperlessService.getThumbnailImage;
+const writtenCachePaths = [];
+
+(async () => {
+  console.log('\n=== Thumbnail caching is optional for analysis ===');
+
+  try {
+    for (const { label, service, stub } of PROVIDERS) {
+      /* --- ids the cache path helper rejects ----------------------------- */
+
+      // `id || []` is what routes/setup.js hands over when the request omits
+      // one, and String([]) sanitizes to the empty string.
+      for (const id of [[], undefined, null, '', '///']) {
+        await check(
+          `${label}: an unusable id (${JSON.stringify(id)}) still analyses`,
+          async () => {
+            stub();
+            paperlessService.getThumbnailImage = async () => {
+              throw new Error(
+                'Paperless must not be called for an unusable id'
+              );
+            };
+            assertAnalysed(
+              await service.analyzeDocument('Rechnung', [], [], [], id)
+            );
+          }
+        );
+      }
+
+      /* --- Paperless cannot supply the thumbnail ------------------------- */
+
+      await check(
+        `${label}: a thumbnail Paperless does not have still analyses`,
+        async () => {
+          stub();
+          paperlessService.getThumbnailImage = async () => null;
+          assertAnalysed(
+            await service.analyzeDocument(
+              'Rechnung',
+              [],
+              [],
+              [],
+              `thumbmissing${label}`
+            )
+          );
+        }
+      );
+
+      await check(
+        `${label}: a failing thumbnail fetch still analyses`,
+        async () => {
+          stub();
+          paperlessService.getThumbnailImage = async () => {
+            throw new Error('Paperless unreachable');
+          };
+          assertAnalysed(
+            await service.analyzeDocument(
+              'Rechnung',
+              [],
+              [],
+              [],
+              `thumbbroken${label}`
+            )
+          );
+        }
+      );
+
+      /* --- the feature itself still works ------------------------------- */
+
+      await check(
+        `${label}: an available thumbnail is still written to the cache`,
+        async () => {
+          stub();
+          paperlessService.getThumbnailImage = async () =>
+            Buffer.from([0x89, 0x50, 0x4e, 0x47]);
+
+          const id = `thumbok${label}`;
+          const cachePath = getThumbnailCachePath(id);
+          writtenCachePaths.push(cachePath);
+          await fs.rm(cachePath, { force: true });
+
+          assertAnalysed(
+            await service.analyzeDocument('Rechnung', [], [], [], id)
+          );
+          await assert.doesNotReject(
+            fs.access(cachePath),
+            `expected a cached thumbnail at ${cachePath}`
+          );
+        }
+      );
+    }
+  } finally {
+    paperlessService.getThumbnailImage = originalGetThumbnailImage;
+    for (const cachePath of writtenCachePaths) {
+      await fs.rm(cachePath, { force: true });
+    }
+  }
+
+  if (failed > 0) {
+    console.error(`\n${failed} thumbnail independence case(s) failed`);
+    process.exit(1);
+  }
+  console.log(
+    `\nAll thumbnail independence cases passed (cache dir: ${THUMBNAIL_CACHE_DIR})`
+  );
+})();

From 63c51f29d3adf0640a5fa5c9678240468ee084f8 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Tue, 1 Sep 2026 19:27:27 +0200
Subject: [PATCH 29/53] Refactoring Step 1
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three new findings, discovered while writing the tests 7k — Ollama reports a response without tags and correspondents as a success. The other 3 throw "AI could not determine assignable metadata…", which triggers the OCR fallback. Ollama merely logs it and delivers a document full of null values, populated metrics, and no error. As a result, the scan loop writes nulls back to Paperless and never falls back to OCR.7l — Ollama's empty payload reads No response data from Ollama API. This does not match a single OCR marker; the other three say Invalid API response structure, which does match. Same silent consequence. 7m — Custom classifies a socket hang up as a timeout and replaces the message with AI response timeout reached…. This is a consequence of Finding 7e, which is now proven. Both 7k and 7l are silent: the analysis "succeeds," the log looks plausible, and the document disappears. I pinned them as they are today, along with a comment—if the refactoring unifies the path, this case will fail. That is intentional: it forces the person to look at it instead of swallowing the issue. I am intentionally not pinning two things: the exact wording of an error message (Custom and Ollama's retry wrappers rewrite it) and the marker list via import. I copied the list manually because an import would make the test automatically match the source code, which is the exact opposite of pinning.Proof that the tests do not run empty (false positives)Mutation 1 — Renaming an OCR marker in serviceUtils.js → 5 cases fail.Mutation 2 — Removing errorCode: error.code from openaiService.js → 1 case fails.Both files were restored afterward, leaving git diff completely clean.Current Statusllm-analysis-result-contract has 32 cases, llm-ocr-fallback-phrases has 15 cases—both are green. Full run with Node 26.7.0: 74 passed / 7 skipped / 7 failed (the exact same seven pre-existing failures that I verified against a clean tree). ESLint and Prettier are clean.
---
 scripts/run-tests.js                       |   4 +
 services/ManualServiceRefactoring.md       |  50 ++-
 tests/test-llm-analysis-result-contract.js | 360 +++++++++++++++++++++
 tests/test-llm-ocr-fallback-phrases.js     | 305 +++++++++++++++++
 4 files changed, 703 insertions(+), 16 deletions(-)
 create mode 100644 tests/test-llm-analysis-result-contract.js
 create mode 100644 tests/test-llm-ocr-fallback-phrases.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index 11ec0c5a..6c464787 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -94,6 +94,8 @@ const TESTS = {
   'ai-temperature-config': 'test-ai-temperature-config.js',
   'custom-field-monetary-normalization':
     'test-custom-field-monetary-normalization.js',
+  'llm-analysis-result-contract': 'test-llm-analysis-result-contract.js',
+  'llm-ocr-fallback-phrases': 'test-llm-ocr-fallback-phrases.js',
   'mistral-ocr-no-processed-on-update-failure':
     'test-mistral-ocr-no-processed-on-update-failure.js',
   'ocr-provider-lmstudio-compatible':
@@ -186,6 +188,8 @@ const AREAS = {
     'ollama-token-metrics',
     'reconciliation-service',
     'settings-paperless-url-fallback',
+    'llm-analysis-result-contract',
+    'llm-ocr-fallback-phrases',
   ],
   prompts: [
     'restriction-service',
diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 390f7f9c..4c4634cf 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -156,18 +156,27 @@ a record rather than a string.
 The refactor collapses four implementations into one, which forces a choice wherever they differ
 today. These are the actual bugs the work will surface:
 
-| #   | Drift                                                                                                                       | Location                                                                                                                                                                         | Proposed resolution                                                                                                                                                                                 |
-| --- | --------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`                                   | `azureService.js:258`, `:426`                                                                                                                                                    | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                                                     |
-| b   | ~~`return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined`~~ **resolved**          | was `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                          | Fixed ahead of the refactor together with row j — same statement block, same root cause. See the note below the table.                                                                              |
-| c   | "Insufficient content" early return exists only in OpenAI                                                                   | `openaiService.js:310-332`, `:505-527`                                                                                                                                           | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                                                            |
-| d   | Three different JSON parsers                                                                                                | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | One parser built from custom's `_extractFirstJsonValue` + `<think>` stripping + Ollama's sanitize fallback; keep Ollama's object passthrough as a pre-step.                                         |
-| e   | Only custom normalizes timeouts                                                                                             | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                                                                |
-| f   | `generateText` max-token handling differs four ways                                                                         | openai: none; `azureService.js:528`: `max_tokens`; `customService.js:625-654`: clamped; `ollamaService.js:1105`: `num_predict`                                                   | Keep custom's clamping as the shared rule; the transport maps it to its own knob.                                                                                                                   |
-| g   | `checkStatus` differs by nature                                                                                             | openai/custom `models.list()`; `azureService.js:574-580` REST GET; `ollamaService.js:1143` `/api/ps`                                                                             | Stays in the transport. Not business logic.                                                                                                                                                         |
-| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`                                    | `openaiService.js:33-44`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop.                                  |
-| i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt                              | all four                                                                                                                                                                         | Still out of scope as a _placement_ question, but the four copies are now one call to `services/thumbnailCache.js`. Moving it out of `analyzeDocument` is a one-line change once a caller wants it. |
-| j   | ~~`getThumbnailCachePath(id)` runs **outside** the try block, so an invalid id makes `analyzeDocument` throw~~ **resolved** | was `openaiService.js:63`, `azureService.js:52`, `customService.js:147`                                                                                                          | Fixed ahead of the refactor. See the note below the table.                                                                                                                                          |
+| #   | Drift                                                                                                                       | Location                                                                                                                                                                         | Proposed resolution                                                                                                                                                                                                                                         |
+| --- | --------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`                                   | `azureService.js:258`, `:426`                                                                                                                                                    | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                                                                                                             |
+| b   | ~~`return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined`~~ **resolved**          | was `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                          | Fixed ahead of the refactor together with row j — same statement block, same root cause. See the note below the table.                                                                                                                                      |
+| c   | "Insufficient content" early return exists only in OpenAI                                                                   | `openaiService.js:310-332`, `:505-527`                                                                                                                                           | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                                                                                                                    |
+| d   | Three different JSON parsers                                                                                                | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | One parser built from custom's `_extractFirstJsonValue` + `<think>` stripping + Ollama's sanitize fallback; keep Ollama's object passthrough as a pre-step.                                                                                                 |
+| e   | Only custom normalizes timeouts                                                                                             | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                                                                                                                        |
+| f   | `generateText` max-token handling differs four ways                                                                         | openai: none; `azureService.js:528`: `max_tokens`; `customService.js:625-654`: clamped; `ollamaService.js:1105`: `num_predict`                                                   | Keep custom's clamping as the shared rule; the transport maps it to its own knob.                                                                                                                                                                           |
+| g   | `checkStatus` differs by nature                                                                                             | openai/custom `models.list()`; `azureService.js:574-580` REST GET; `ollamaService.js:1143` `/api/ps`                                                                             | Stays in the transport. Not business logic.                                                                                                                                                                                                                 |
+| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`                                    | `openaiService.js:33-44`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop.                                                                                          |
+| i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt                              | all four                                                                                                                                                                         | Still out of scope as a _placement_ question, but the four copies are now one call to `services/thumbnailCache.js`. Moving it out of `analyzeDocument` is a one-line change once a caller wants it.                                                         |
+| j   | ~~`getThumbnailCachePath(id)` runs **outside** the try block, so an invalid id makes `analyzeDocument` throw~~ **resolved** | was `openaiService.js:63`, `azureService.js:52`, `customService.js:147`                                                                                                          | Fixed ahead of the refactor. See the note below the table.                                                                                                                                                                                                  |
+| k   | An answer with neither tags nor a correspondent is a **failure** on three providers and a **success** on Ollama             | `openaiService.js:329-331`, `azureService.js`, `customService.js` vs. `ollamaService.js`                                                                                         | Ollama logs "No tags or correspondent found…" and returns an all-null `document` with populated `metrics` and **no `error`**. So the scan loop writes nulls back to Paperless and never falls back to OCR. Adopt the throwing behaviour of the other three. |
+| l   | An empty provider payload is worded "No response data from Ollama API", which matches **no** OCR marker                     | `ollamaService.js` vs. `openaiService.js:262`, `azureService.js`, `customService.js`                                                                                             | The other three say "Invalid API response structure", which `shouldQueueForOcrOnAiError()` matches. Unify the wording on the shared phrase, or add this one to the marker list.                                                                             |
+| m   | Custom classifies `socket hang up` as a timeout and replaces the message                                                    | `customService.js:331`                                                                                                                                                           | Consequence of row e: `isTimeoutError()` swallows the original text into "AI response timeout reached…". Decide deliberately whether a dropped connection is a timeout when the handling is unified.                                                        |
+
+Rows k and l were found by writing the step-1 safety net, not by reading the code — both are
+silent: the analysis "succeeds", the log looks plausible, and the document simply never reaches
+OCR. Both are pinned as-is in the new tests and marked as known drift, so the case starts failing
+the moment the refactor unifies the path. That is deliberate: it forces whoever unifies it to
+update the expectation rather than to not notice.
 
 **Rows b and j were fixed before the refactor started.** They shared one root cause — a best-effort
 side effect written as a blocking precondition — and sat within three lines of each other, so
@@ -421,10 +430,19 @@ Repo hygiene for every step:
 Each step ends with a green `node scripts/run-tests.js --all`. Steps 1-2 are pure preparation and
 carry no behavioral risk; the risk starts at step 4.
 
-1. **Freeze the contract in tests first (TDD).** Write `llm-analysis-result-contract` and
-   `llm-ocr-fallback-phrases` against the **current** `openaiService`/`ollamaService`. They must pass
-   before any production code moves — that is what makes them a regression net rather than a
-   description of the new code.
+1. ~~**Freeze the contract in tests first (TDD).**~~ **Done.** `llm-analysis-result-contract`
+   (32 cases) and `llm-ocr-fallback-phrases` (15 cases) cover all four providers and are green
+   against the **current** services — which is what makes them a regression net rather than a
+   description of the new code. Both are mutation-verified: rewording an OCR marker in
+   `serviceUtils.js` fails five cases, dropping `errorCode: error.code` from `openaiService.js`
+   fails one. They stub only the provider client and pre-seed the thumbnail cache, so no network
+   and no `require.cache` tricks. They also produced Findings 7k, 7l and 7m.
+
+   Two things they deliberately do **not** pin: the wording of a failure message (Custom and Ollama
+   both rewrite it — rows e/m and the retry wrapper), and the phrases the OCR matcher owns, which
+   are copied into the test rather than imported. Importing that list would make the test agree
+   with whatever the source says, which is the opposite of pinning it.
+
 2. **Extract `customFieldsTemplate.js` and `promptBuilder.js`** with `llm-prompt-builder`, and have
    all four existing services call them. Six duplicates become one; no other behavior changes.
    `prompt-existing-data-serialization` and the restriction tests are the guard.
diff --git a/tests/test-llm-analysis-result-contract.js b/tests/test-llm-analysis-result-contract.js
new file mode 100644
index 00000000..d27ead5b
--- /dev/null
+++ b/tests/test-llm-analysis-result-contract.js
@@ -0,0 +1,360 @@
+/**
+ * The analysis result contract, pinned across all four providers.
+ *
+ * Safety net for the LLM service refactoring (services/ManualServiceRefactoring.md,
+ * step 1). It describes what analyzeDocument() promises its three callers today —
+ * server.js:766-821, services/mistralOcrService.js:988-997, and
+ * routes/setup.js:3167-3177 — so that collapsing four implementations into one
+ * cannot quietly change the shape they read.
+ *
+ * Two properties carry the weight:
+ *
+ *   1. Success returns the full { document, metrics, truncated } record. Every
+ *      field of `document` is written back by paperlessService.updateDocument();
+ *      a narrowed document does not fail loudly, it silently stops setting the
+ *      title, type, date, language, and custom fields.
+ *
+ *   2. Failure returns { document: { tags: [], correspondent: null },
+ *      metrics: null, error, errorCode } and **never throws**. server.js relies
+ *      on this: a throw escapes the per-document try in the scan loop instead of
+ *      routing the document to the OCR queue.
+ *
+ * The provider client is the only collaborator stubbed. The thumbnail cache is
+ * pre-seeded so the analysis path reaches the client without a Paperless call.
+ */
+
+const assert = require('assert');
+const fs = require('fs').promises;
+
+process.env.AI_PROVIDER = 'openai';
+process.env.OPENAI_API_KEY = 'test-key';
+process.env.OPENAI_MODEL = 'gpt-4';
+process.env.AZURE_API_KEY = 'test-key';
+process.env.AZURE_ENDPOINT = 'https://example.invalid';
+process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment';
+process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1';
+process.env.CUSTOM_API_KEY = 'test-key';
+process.env.CUSTOM_MODEL = 'test-model';
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('../services/thumbnailCachePaths');
+const openaiService = require('../services/openaiService');
+const azureService = require('../services/azureService');
+const customService = require('../services/customService');
+const ollamaService = require('../services/ollamaService');
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+/** Own id so the seeded thumbnail cannot collide with a real cached document. */
+const DOCUMENT_ID = 'testLlmResultContract';
+
+/**
+ * Every field the write-back path reads. Deliberately includes the ones a
+ * narrowed result would drop without any error surfacing.
+ */
+const ANSWER = {
+  title: 'Telekom invoice July',
+  correspondent: 'Telekom Deutschland GmbH',
+  tags: ['invoice', 'telecom'],
+  document_type: 'Rechnung',
+  document_date: '2026-09-03',
+  language: 'de',
+  custom_fields: [{ field_name: 'Amount', value: '49.99' }],
+};
+
+function completion(content = JSON.stringify(ANSWER)) {
+  return {
+    choices: [{ message: { content }, finish_reason: 'stop' }],
+    usage: { prompt_tokens: 900, completion_tokens: 120, total_tokens: 1020 },
+  };
+}
+
+/** Replaces the SDK client with one that answers whatever the case needs. */
+function stubClient(service, respond) {
+  service.client = {
+    chat: { completions: { create: respond } },
+    models: { list: async () => ({ data: [] }) },
+  };
+}
+
+/** Same for Ollama, which answers over axios instead of an SDK. */
+function stubOllamaClient(respond) {
+  ollamaService.client = {
+    post: respond,
+    get: async () => ({ status: 200, data: { models: [] } }),
+  };
+}
+
+function openAiCompatibleProvider(label, service) {
+  return {
+    label,
+    service,
+    arrangeSuccess: () => stubClient(service, async () => completion()),
+    arrangeAnswer: (content) =>
+      stubClient(service, async () => completion(content)),
+    arrangeThrow: (error) =>
+      stubClient(service, async () => {
+        throw error;
+      }),
+  };
+}
+
+const PROVIDERS = [
+  openAiCompatibleProvider('OpenAI', openaiService),
+  openAiCompatibleProvider('Azure', azureService),
+  openAiCompatibleProvider('Custom', customService),
+  {
+    label: 'Ollama',
+    service: ollamaService,
+    arrangeSuccess: () =>
+      stubOllamaClient(async () => ({
+        data: {
+          response: JSON.stringify(ANSWER),
+          done_reason: 'stop',
+          eval_count: 120,
+          prompt_eval_count: 900,
+        },
+      })),
+    arrangeAnswer: (content) =>
+      stubOllamaClient(async () => ({
+        data: {
+          response: content,
+          done_reason: 'stop',
+          eval_count: 120,
+          prompt_eval_count: 900,
+        },
+      })),
+    arrangeThrow: (error) =>
+      stubOllamaClient(async () => {
+        throw error;
+      }),
+  },
+];
+
+/** Enough text that no provider takes an "insufficient content" shortcut. */
+const CONTENT = 'Invoice from Telekom Deutschland GmbH. '.repeat(20);
+
+const analyse = (service) =>
+  service.analyzeDocument(CONTENT, [], [], [], DOCUMENT_ID);
+
+/** The failure shape server.js:774-821 branches on. */
+const assertFailureRecord = (analysis) => {
+  assert.ok(analysis, 'analyzeDocument returned nothing at all');
+  assert.deepStrictEqual(
+    analysis.document,
+    { tags: [], correspondent: null },
+    'a failure must still carry an empty, well-formed document'
+  );
+  assert.strictEqual(analysis.metrics, null, 'a failure records no metrics');
+  assert.strictEqual(
+    typeof analysis.error,
+    'string',
+    'the failure reason must be a string — shouldQueueForOcrOnAiError() reads it'
+  );
+  assert.ok(analysis.error.trim(), 'the failure reason must not be empty');
+};
+
+(async () => {
+  console.log('\n=== LLM analysis result contract ===');
+
+  await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+  await fs.writeFile(
+    getThumbnailCachePath(DOCUMENT_ID),
+    Buffer.from([0x89, 0x50, 0x4e, 0x47])
+  );
+
+  for (const {
+    label,
+    service,
+    arrangeSuccess,
+    arrangeAnswer,
+    arrangeThrow,
+  } of PROVIDERS) {
+    /* --- the success record ------------------------------------------- */
+
+    await check(
+      `${label}: a successful analysis returns the full record`,
+      async () => {
+        arrangeSuccess();
+        const analysis = await analyse(service);
+
+        assert.ok(analysis, 'analyzeDocument returned nothing at all');
+        assert.strictEqual(
+          analysis.error,
+          undefined,
+          `unexpected error: ${analysis.error}`
+        );
+        assert.strictEqual(
+          typeof analysis.truncated,
+          'boolean',
+          'truncated must always be a boolean'
+        );
+      }
+    );
+
+    await check(
+      `${label}: every document field survives to the caller`,
+      async () => {
+        arrangeSuccess();
+        const { document } = await analyse(service);
+
+        // Asserted field by field: paperlessService.updateDocument() writes each
+        // of these, and a dropped one produces no error, just a missing value.
+        assert.deepStrictEqual(document.tags, ANSWER.tags);
+        assert.strictEqual(document.correspondent, ANSWER.correspondent);
+        assert.strictEqual(document.title, ANSWER.title);
+        assert.strictEqual(document.document_type, ANSWER.document_type);
+        assert.strictEqual(document.document_date, ANSWER.document_date);
+        assert.strictEqual(document.language, ANSWER.language);
+        assert.deepStrictEqual(document.custom_fields, ANSWER.custom_fields);
+      }
+    );
+
+    await check(
+      `${label}: metrics are reported as named token counts`,
+      async () => {
+        arrangeSuccess();
+        const { metrics } = await analyse(service);
+
+        // documentModel.addOpenAIMetrics() reads these three names.
+        assert.ok(metrics, 'a successful analysis must report metrics');
+        assert.strictEqual(typeof metrics.promptTokens, 'number');
+        assert.strictEqual(typeof metrics.completionTokens, 'number');
+        assert.strictEqual(typeof metrics.totalTokens, 'number');
+      }
+    );
+
+    /* --- the failure record, and the promise not to throw --------------- */
+
+    // The message text itself is not pinned here: Custom rewrites anything
+    // isTimeoutError() recognizes into "AI response timeout reached…", and
+    // Ollama's retry wrapper replaces it with "Ollama request failed (…)".
+    // Only the phrases that steer OCR fallback are pinned, in
+    // test-llm-ocr-fallback-phrases.js.
+    await check(
+      `${label}: a transport failure is returned, not thrown`,
+      async () => {
+        arrangeThrow(new Error('socket hang up'));
+        const analysis = await analyse(service);
+
+        assertFailureRecord(analysis);
+      }
+    );
+
+    await check(
+      `${label}: an unparseable answer is returned, not thrown`,
+      async () => {
+        arrangeAnswer('this is not JSON at all');
+        const analysis = await analyse(service);
+
+        assertFailureRecord(analysis);
+      }
+    );
+
+    await check(
+      `${label}: an error carrying a code passes the code through`,
+      async () => {
+        // The scan loop reads errorCode to decide OCR fallback; a service that
+        // raises the right error but drops the code loses that decision.
+        const coded = new Error('provider stopped generating early');
+        coded.code = 'ai_response_truncated';
+        arrangeThrow(coded);
+        const analysis = await analyse(service);
+
+        assertFailureRecord(analysis);
+        assert.strictEqual(analysis.errorCode, 'ai_response_truncated');
+      }
+    );
+
+    await check(
+      `${label}: an error without a code leaves errorCode unset`,
+      async () => {
+        arrangeThrow(new Error('socket hang up'));
+        const analysis = await analyse(service);
+
+        assert.strictEqual(
+          analysis.errorCode,
+          undefined,
+          'an uncoded failure must not invent a code'
+        );
+      }
+    );
+  }
+
+  /* --- an answer with no usable metadata: the providers disagree --------- */
+
+  /* OpenAI, Azure and Custom raise "AI could not determine assignable
+     metadata: no tags or correspondent found", which is one of the phrases
+     shouldQueueForOcrOnAiError() matches, so the document reaches the OCR
+     queue. Ollama only logs and returns an all-null document as a *success*:
+     no error, metrics populated. The scan loop therefore writes nulls back to
+     Paperless and never falls back to OCR. */
+
+  for (const { label, service, arrangeAnswer } of PROVIDERS.filter(
+    (p) => p.label !== 'Ollama'
+  )) {
+    await check(
+      `${label}: an answer without usable metadata is a failure`,
+      async () => {
+        arrangeAnswer(JSON.stringify({ note: 'nothing usable here' }));
+        const analysis = await analyse(service);
+
+        assertFailureRecord(analysis);
+        assert.match(
+          analysis.error,
+          /could not determine assignable metadata/i
+        );
+      }
+    );
+  }
+
+  /* Pinned as it is today so the divergence is visible rather than absent.
+     When the refactor unifies this path (Finding 7c/7k), this case will start
+     failing — that is the point. Replace it with the assertion above and drop
+     the Ollama exclusion from the loop overhead. */
+  await check(
+    'Ollama: an answer without usable metadata is NOT reported as a failure (known drift)',
+    async () => {
+      const ollama = PROVIDERS.find((p) => p.label === 'Ollama');
+      ollama.arrangeAnswer(JSON.stringify({ note: 'nothing usable here' }));
+      const analysis = await analyse(ollama.service);
+
+      assert.strictEqual(
+        analysis.error,
+        undefined,
+        'Ollama has started reporting this as an error — unify it and update this case'
+      );
+      assert.deepStrictEqual(analysis.document, {
+        tags: [],
+        correspondent: null,
+        title: null,
+        document_date: null,
+        document_type: null,
+        language: null,
+        custom_fields: null,
+      });
+      assert.ok(analysis.metrics, 'the drift includes reporting metrics');
+    }
+  );
+
+  await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
+
+  if (failed) {
+    console.error(`\n${failed} analysis result contract case(s) failed`);
+    process.exitCode = 1;
+    return;
+  }
+  console.log('\nAll analysis result contract cases passed');
+})();
diff --git a/tests/test-llm-ocr-fallback-phrases.js b/tests/test-llm-ocr-fallback-phrases.js
new file mode 100644
index 00000000..5d5c15ad
--- /dev/null
+++ b/tests/test-llm-ocr-fallback-phrases.js
@@ -0,0 +1,305 @@
+/**
+ * The OCR fallback phrases, pinned end to end (Finding 4).
+ *
+ * Safety net for the LLM service refactoring (services/ManualServiceRefactoring.md,
+ * step 1). server.js:774-821 decides whether a failed analysis goes to the OCR
+ * queue by matching analysis.error against a fixed list of **substrings** in
+ * shouldQueueForOcrOnAiError(), and picks the queue reason with
+ * classifyOcrQueueReasonFromAiError() (services/serviceUtils.js:812-861).
+ *
+ * Nothing enforces that link today. Reword a message inside a provider service
+ * and the analysis still fails, the log still looks right, and the document
+ * silently stops reaching OCR. That is the regression this test exists to
+ * catch, so it asserts the full chain per provider:
+ *
+ *     stub a failing client
+ *       -> analyzeDocument() returns analysis.error
+ *       -> shouldQueueForOcrOnAiError(analysis.error) is true
+ *       -> classifyOcrQueueReasonFromAiError(analysis.error) is the right reason
+ *
+ * The matcher's own vocabulary is checked separately, so a phrase deleted from
+ * the list is caught even if no provider happens to produce it.
+ */
+
+const assert = require('assert');
+const fs = require('fs').promises;
+
+process.env.AI_PROVIDER = 'openai';
+process.env.OPENAI_API_KEY = 'test-key';
+process.env.OPENAI_MODEL = 'gpt-4';
+process.env.AZURE_API_KEY = 'test-key';
+process.env.AZURE_ENDPOINT = 'https://example.invalid';
+process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment';
+process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1';
+process.env.CUSTOM_API_KEY = 'test-key';
+process.env.CUSTOM_MODEL = 'test-model';
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+
+const {
+  shouldQueueForOcrOnAiError,
+  classifyOcrQueueReasonFromAiError,
+} = require('../services/serviceUtils');
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('../services/thumbnailCachePaths');
+const openaiService = require('../services/openaiService');
+const azureService = require('../services/azureService');
+const customService = require('../services/customService');
+const ollamaService = require('../services/ollamaService');
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+/** Own id so the seeded thumbnail cannot collide with a real cached document. */
+const DOCUMENT_ID = 'testllmocrphrases';
+
+/** Enough text that no provider takes an "insufficient content" shortcut. */
+const CONTENT = 'Invoice from Telekom Deutschland GmbH. '.repeat(20);
+
+/**
+ * Every marker shouldQueueForOcrOnAiError() knows, with the reason
+ * classifyOcrQueueReasonFromAiError() derives from it. Copied out deliberately
+ * rather than imported: importing the list would make the test agree with
+ * whatever the source says, which is the opposite of pinning it.
+ */
+const MARKERS = [
+  ['insufficient content for ai analysis', 'ai_insufficient_content'],
+  ['invalid response structure', 'ai_invalid_response_structure'],
+  ['could not determine assignable metadata', 'ai_invalid_response_structure'],
+  ['invalid json response from api', 'ai_invalid_json'],
+  ['invalid api response structure', 'ai_invalid_api_response_structure'],
+];
+
+function stubClient(service, respond) {
+  service.client = {
+    chat: { completions: { create: respond } },
+    models: { list: async () => ({ data: [] }) },
+  };
+}
+
+function stubOllamaClient(respond) {
+  ollamaService.client = {
+    post: respond,
+    get: async () => ({ status: 200, data: { models: [] } }),
+  };
+}
+
+function openAiCompatibleProvider(label, service) {
+  return {
+    label,
+    service,
+    /** Answers with `content`, which the service then tries to parse. */
+    answer: (content) =>
+      stubClient(service, async () => ({
+        choices: [{ message: { content }, finish_reason: 'stop' }],
+        usage: {
+          prompt_tokens: 900,
+          completion_tokens: 120,
+          total_tokens: 1020,
+        },
+      })),
+    /** Answers with a payload that has no choices array at all. */
+    answerMalformed: () => stubClient(service, async () => ({ usage: {} })),
+  };
+}
+
+const PROVIDERS = [
+  openAiCompatibleProvider('OpenAI', openaiService),
+  openAiCompatibleProvider('Azure', azureService),
+  openAiCompatibleProvider('Custom', customService),
+  {
+    label: 'Ollama',
+    service: ollamaService,
+    answer: (content) =>
+      stubOllamaClient(async () => ({
+        data: {
+          response: content,
+          done_reason: 'stop',
+          eval_count: 120,
+          prompt_eval_count: 900,
+        },
+      })),
+    answerMalformed: () => stubOllamaClient(async () => ({ data: {} })),
+  },
+];
+
+const analyse = (service, content = CONTENT) =>
+  service.analyzeDocument(content, [], [], [], DOCUMENT_ID);
+
+/**
+ * The assertion that matters: whatever the service worded, the scan loop must
+ * still route the document to OCR under the expected reason.
+ */
+const assertRoutesToOcr = (analysis, expectedReason) => {
+  assert.ok(analysis, 'analyzeDocument returned nothing at all');
+  assert.strictEqual(
+    typeof analysis.error,
+    'string',
+    'no error message to match against'
+  );
+  assert.ok(
+    shouldQueueForOcrOnAiError(analysis.error),
+    `"${analysis.error}" no longer matches any OCR fallback marker`
+  );
+  assert.strictEqual(
+    classifyOcrQueueReasonFromAiError(analysis.error),
+    expectedReason,
+    `"${analysis.error}" classified as the wrong queue reason`
+  );
+};
+
+(async () => {
+  console.log('\n=== LLM OCR fallback phrases ===');
+
+  /* --- the matcher's own vocabulary -------------------------------------- */
+
+  await check('every marker still triggers the fallback', () => {
+    for (const [marker, reason] of MARKERS) {
+      assert.ok(
+        shouldQueueForOcrOnAiError(marker),
+        `marker dropped from shouldQueueForOcrOnAiError: "${marker}"`
+      );
+      assert.strictEqual(
+        classifyOcrQueueReasonFromAiError(marker),
+        reason,
+        `marker reclassified: "${marker}"`
+      );
+    }
+  });
+
+  await check(
+    'markers match case-insensitively inside longer sentences',
+    () => {
+      // Services word these as sentences, not as bare markers.
+      assert.ok(
+        shouldQueueForOcrOnAiError(
+          'AI could not determine assignable metadata: no tags or correspondent found'
+        )
+      );
+      assert.strictEqual(
+        classifyOcrQueueReasonFromAiError(
+          'AI could not determine assignable metadata: no tags or correspondent found'
+        ),
+        'ai_invalid_response_structure'
+      );
+    }
+  );
+
+  await check('an unrelated failure does not reach the OCR queue', () => {
+    // A dead provider must not be retried as an OCR job.
+    assert.strictEqual(shouldQueueForOcrOnAiError('socket hang up'), false);
+    assert.strictEqual(shouldQueueForOcrOnAiError(''), false);
+    assert.strictEqual(shouldQueueForOcrOnAiError(undefined), false);
+    assert.strictEqual(
+      classifyOcrQueueReasonFromAiError('socket hang up'),
+      'ai_failed_unknown'
+    );
+  });
+
+  /* --- the providers still produce them ---------------------------------- */
+
+  await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+  await fs.writeFile(
+    getThumbnailCachePath(DOCUMENT_ID),
+    Buffer.from([0x89, 0x50, 0x4e, 0x47])
+  );
+
+  for (const { label, service, answer } of PROVIDERS) {
+    await check(
+      `${label}: an unparseable answer routes to OCR as invalid JSON`,
+      async () => {
+        answer('the model apologised instead of answering');
+        assertRoutesToOcr(await analyse(service), 'ai_invalid_json');
+      }
+    );
+  }
+
+  for (const { label, service, answerMalformed } of PROVIDERS.filter(
+    (p) => p.label !== 'Ollama'
+  )) {
+    await check(
+      `${label}: a payload with no choices routes to OCR`,
+      async () => {
+        answerMalformed();
+        assertRoutesToOcr(
+          await analyse(service),
+          'ai_invalid_api_response_structure'
+        );
+      }
+    );
+  }
+
+  /* Pinned as it is today so the divergence is visible. Ollama words an empty
+     payload "No response data from Ollama API", which matches no marker, so
+     the document is never queued for OCR while the other three are. When the
+     refactor unifies the wording this case starts failing — fold it into the
+     loop above then. */
+  await check(
+    'Ollama: an empty payload does NOT route to OCR (known drift)',
+    async () => {
+      const ollama = PROVIDERS.find((p) => p.label === 'Ollama');
+      ollama.answerMalformed();
+      const analysis = await analyse(ollama.service);
+
+      assert.match(analysis.error, /No response data from Ollama API/);
+      assert.strictEqual(
+        shouldQueueForOcrOnAiError(analysis.error),
+        false,
+        'Ollama now reaches OCR here — unify the wording and update this case'
+      );
+    }
+  );
+
+  /* Only OpenAI implements the "insufficient content" early return today
+     (Finding 7c), and it triggers on the *answer*, not on the document: an
+     unparseable reply containing "i'm sorry" / "i cannot" / "insufficient".
+     The refactor is meant to extend it to the other three. */
+  await check(
+    'OpenAI: a refusal routes to OCR as insufficient content',
+    async () => {
+      const openai = PROVIDERS.find((p) => p.label === 'OpenAI');
+      openai.answer("I'm sorry, there is insufficient text to analyse.");
+      assertRoutesToOcr(
+        await analyse(openai.service),
+        'ai_insufficient_content'
+      );
+    }
+  );
+
+  /* The three OpenAI-compatible services raise "could not determine
+     assignable metadata"; Ollama returns an all-null document with no error
+     at all, so it never reaches OCR. See test-llm-analysis-result-contract.js,
+     where that drift is pinned. */
+  for (const { label, service, answer } of PROVIDERS.filter(
+    (p) => p.label !== 'Ollama'
+  )) {
+    await check(
+      `${label}: an answer without metadata routes to OCR`,
+      async () => {
+        answer(JSON.stringify({ note: 'nothing usable here' }));
+        assertRoutesToOcr(
+          await analyse(service),
+          'ai_invalid_response_structure'
+        );
+      }
+    );
+  }
+
+  await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
+
+  if (failed) {
+    console.error(`\n${failed} OCR fallback phrase case(s) failed`);
+    process.exitCode = 1;
+    return;
+  }
+  console.log('\nAll OCR fallback phrase cases passed');
+})();

From 38a53524f5f21c57752e769a5a9e4b7f1b3ac72c Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 20 Sep 2026 01:15:44 +0200
Subject: [PATCH 30/53] pre merge upstream

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .gitignore                | 1 +
 models/document.js        | 6 +++++-
 services/customService.js | 9 +++++++--
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index eb0fd26a..259d0aa6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,6 +33,7 @@ dist/
 *.local.*
 .vscode/
 .idea/
+.ollamassist/
 # AI context files (auto-generated)
 COPILOT.md
 .DS_Store
diff --git a/models/document.js b/models/document.js
index 799ddc77..dd932a00 100644
--- a/models/document.js
+++ b/models/document.js
@@ -1203,6 +1203,10 @@ module.exports = {
 
   async addToOcrQueue(documentId, title, reason = 'manual') {
     try {
+      // TODO cleanup status hack, check usages of this method
+      // status hack: this method is called to force ocr to reprocess a document.
+      // keeping the status as done prevents this. I wonder if there are side
+      // effects now when called in a use case which does not imply forced OCR.
       const result = db
         .prepare(
           `
@@ -1211,7 +1215,7 @@ module.exports = {
         ON CONFLICT(document_id) DO UPDATE SET
           title = excluded.title,
           reason = excluded.reason,
-          status = CASE WHEN status = 'done' THEN 'done' ELSE 'pending' END,
+          status = CASE WHEN status = 'done' THEN 'pending' ELSE 'pending' END, 
           added_at = CASE WHEN status = 'done' THEN added_at ELSE CURRENT_TIMESTAMP END
         WHERE status != 'processing'
       `
diff --git a/services/customService.js b/services/customService.js
index 9a758640..0e1ecea4 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -353,7 +353,12 @@ class CustomOpenAIService {
       const message = response?.choices?.[0]?.message;
       let jsonContent = extractChatMessageContent(message, 'Custom OpenAI');
       if (!jsonContent) {
-        throw new Error('Invalid API response structure');
+        if(!response) throw new Error (`API response is empty.`);
+        if(!response.choices) throw new Error (`API response is missing choices.`);
+        if(!response.choices[0]) throw new Error (`API response is missing choices[0].`);
+        if(!response.choices[0].message) throw new Error (`API response is missing choices[0].message.`);
+        if(!response.choices[0].message.content) throw new Error (`API response is missing choices[0].message.content.`);
+        throw new Error(`Invalid API response structure: ${message?.content}`);
       }
 
       // Log token usage
@@ -375,7 +380,7 @@ class CustomOpenAIService {
         parsedResponse = parsedResult.parsed;
         jsonContent = parsedResult.normalized;
       } catch (error) {
-        console.error(`Failed to parse JSON response: ${error.message}`);
+        console.log(`[ERROR] Failed to parse JSON response: ${error.message}`);
         console.debug(error);
         throw error;
       }

From c88a4480a6d5c1f4cc47438efb2bd1e86252ca3f Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 5 Oct 2026 00:34:32 +0200
Subject: [PATCH 31/53] Make the entrypoint readable for the unprivileged user

Background
Since the upstream merge (#241), docker-entrypoint.sh drops privileges with
gosu and then re-executes itself as PUID:PGID. The Dockerfile copies the
script without --chown (owned by root) and only adds the execute bit. A build
context that hands the file over without read permission for others - here a
Samba share with a restrictive create mask - produced mode 771 in the image.
The kernel could exec the script as uid 1000, but /bin/sh could not read it:
"cannot open /app/./docker-entrypoint.sh: Permission denied", and the
container restarted in a loop.

Changes
- Dockerfile: chmod 755 instead of chmod +x for docker-entrypoint.sh and
  start-services.sh, so the mode no longer depends on the build context.

Testing
On the affected host: stat on /app/docker-entrypoint.sh showed 771 root
before and 755 root after the rebuild; the container starts and the restart
loop is gone.

Impact
Images built from a Git checkout already got 755 and are unchanged. Images
built from a context without world-read permission start again.

Upstream Status
Not reported yet. Upstream's CI builds from a Git checkout (644), so the
problem does not show there; worth an issue/PR since any local build from a
restrictive filesystem hits it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 Dockerfile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 3c3329b7..a2608574 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,7 +18,12 @@ COPY --chown=node:node schemas.js swagger.js ./
 COPY --chown=node:node scripts ./scripts/
 COPY docker-entrypoint.sh start-services.sh ./
 
-RUN chmod +x docker-entrypoint.sh start-services.sh
+# An explicit mode rather than +x: the entrypoint re-executes itself after the
+# privilege drop, so the unprivileged user has to be able to read it, too. A
+# build context that hands the files over without read permission for others
+# (e.g. a Samba share with a restrictive create mask) would otherwise produce
+# 0711 and a restart loop with "cannot open ./docker-entrypoint.sh".
+RUN chmod 755 docker-entrypoint.sh start-services.sh
 
 # Configure persistent data volume.
 # Docker seeds a fresh named volume from the image including ownership and mode,

From 355ac29c6de277ae36854a680b357bb8e6434e86 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 5 Oct 2026 00:43:42 +0200
Subject: [PATCH 32/53] Keep the OCR fallback marker in Custom's response
 structure errors

Background
Commit 38a5352 replaced Custom OpenAI's "Invalid API response structure"
with detailed messages such as "API response is missing choices.". The OCR
fallback (shouldQueueForOcrOnAiError) matches on the old phrase, so a
Custom provider answering without choices or content no longer sent the
document to the OCR queue. llm-ocr-fallback-phrases caught it.

Changes
- services/customService.js: the detail stays, but every message now reads
  "Invalid API response structure: <what is missing>.".

Testing
llm-ocr-fallback-phrases, llm-analysis-result-contract,
ocr-fallback-ai-errors and response-truncation-detection pass. ESLint and
Prettier clean.

Impact
Empty or malformed Custom provider answers reach the OCR fallback again.

Upstream Status
Local only; upstream never had the detailed messages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/customService.js | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/services/customService.js b/services/customService.js
index 7d25e578..0f314466 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -382,12 +382,21 @@ class CustomOpenAIService {
       const message = response?.choices?.[0]?.message;
       let jsonContent = extractChatMessageContent(message, 'Custom OpenAI');
       if (!jsonContent) {
-        if(!response) throw new Error (`API response is empty.`);
-        if(!response.choices) throw new Error (`API response is missing choices.`);
-        if(!response.choices[0]) throw new Error (`API response is missing choices[0].`);
-        if(!response.choices[0].message) throw new Error (`API response is missing choices[0].message.`);
-        if(!response.choices[0].message.content) throw new Error (`API response is missing choices[0].message.content.`);
-        throw new Error(`Invalid API response structure: ${message?.content}`);
+        // Every message keeps the "Invalid API response structure" prefix:
+        // shouldQueueForOcrOnAiError() matches on that phrase, and without it
+        // the document no longer reaches the OCR fallback.
+        const missing = !response
+          ? 'the response is empty'
+          : !response.choices
+            ? 'missing choices'
+            : !response.choices[0]
+              ? 'missing choices[0]'
+              : !response.choices[0].message
+                ? 'missing choices[0].message'
+                : !response.choices[0].message.content
+                  ? 'missing choices[0].message.content'
+                  : `unusable content ${message?.content}`;
+        throw new Error(`Invalid API response structure: ${missing}.`);
       }
 
       // Log token usage

From 3e4581c9eb1e0087e80ccfbb18740e6dcbd33745 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 5 Oct 2026 00:47:15 +0200
Subject: [PATCH 33/53] Bring the LLM refactoring plan up to date with the
 upstream merge

Background
The merge of upstream v2026.09.03 (9d337d2) removed the Playground, moved
the write-back into documentProcessingService and widened generateText()
with the options of services/aiGenerateOptions.js. The plan still pointed
at line numbers, endpoints and tests from before.

Changes
- All line references re-checked against the merged tree.
- POST /manual/playground and analyzePlayground() dropped from the
  findings, the target architecture, the parity test and step 7;
  playground-deprecation replaced by ai-provider-streaming in the list of
  acceptance tests.
- Finding 3 names documentProcessingService.buildUpdateData() as the
  write-back the result contract feeds.
- Finding 4 records the Custom regression from 38a5352 and its fix 355ac29.
- Finding 6 adds the generateText(prompt, options) contract the transports
  have to carry; drift row f marked as mostly resolved upstream.
- Out of scope: CachedNameIdEnum's semantic match calls generateAnswer(),
  which no provider has.

Testing
Documentation only; Prettier clean.

Impact
None at runtime.

Upstream Status
Local only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/ManualServiceRefactoring.md | 128 ++++++++++++++++-----------
 1 file changed, 77 insertions(+), 51 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 4c4634cf..c3317a56 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -29,6 +29,7 @@ Rules for this work:
 ## Validation Against the Source Code
 
 The following was verified against the working tree. Each finding constrains the design that follows.
+Line numbers were re-checked after the merge of upstream v2026.09.03 (merge `9d337d2`).
 
 ### 1. `manualService.js` is dead code
 
@@ -39,8 +40,12 @@ The duplication that the "manual" business service actually suffers from lives i
 `routes/setup.js`, which switches on `process.env.AI_PROVIDER` inline over the four _dedicated_
 singletons:
 
-- `routes/setup.js:7511-7578` — `POST /manual/analyze`, four-way switch calling `analyzeDocument()`.
-- `routes/setup.js:7657-7717` — `POST /manual/playground`, four-way switch calling `analyzePlayground()`.
+- `routes/setup.js:7477-7543` — `POST /manual/analyze`, four-way switch (`:7495-7537`) calling
+  `analyzeDocument()`.
+
+The second switch, `POST /manual/playground` over `analyzePlayground()`, is gone: upstream removed
+the Playground (#359), and the merge took that over, including the local `analyzePlayground()` in
+`ollamaService.js`.
 
 Further evidence that `manualService.js` was abandoned rather than maintained:
 
@@ -56,8 +61,8 @@ Further evidence that `manualService.js` was abandoned rather than maintained:
 - Its Ollama path never sends the document content at all — `prompt` is just
   `process.env.SYSTEM_PROMPT` (`manualService.js:218`).
 
-**Decision:** delete `services/manualService.js` as part of this work and point the two
-`/manual/*` endpoints at `AIServiceFactory.getService()`. Do not port its behavior — it is worse
+**Decision:** delete `services/manualService.js` as part of this work and point
+`POST /manual/analyze` at `AIServiceFactory.getService()`. Do not port its behavior — it is worse
 than the dedicated services on every axis: no token budgeting, no content truncation, no restriction
 placeholders, no structured output, no retry, no metrics.
 
@@ -65,15 +70,16 @@ placeholders, no structured output, no retry, no metrics.
 
 | Service            | Lines | Relationship                                    |
 | ------------------ | ----- | ----------------------------------------------- |
-| `openaiService.js` | 629   | ~90 % identical to azure/custom                 |
-| `azureService.js`  | 597   | ~90 % identical to openai/custom                |
-| `customService.js` | 706   | ~90 % identical, plus the strongest JSON parser |
-| `ollamaService.js` | 1164  | genuinely different transport and token model   |
+| `openaiService.js` | 550   | ~90 % identical to azure/custom                 |
+| `azureService.js`  | 568   | ~90 % identical to openai/custom                |
+| `customService.js` | 698   | ~90 % identical, plus the strongest JSON parser |
+| `ollamaService.js` | 1231  | genuinely different transport and token model   |
 
 The largest duplicated block is **prompt construction**: the
 `CUSTOM_FIELDS` → `customFieldsTemplate` → `%CUSTOMFIELDS%` block appears **six times**
-(`openaiService.js:125-160`, `azureService.js:114-149`, `customService.js:209-244`,
-`ollamaService.js:147-179`, `:434-469`, `:585-622`), followed each time by the same
+(`openaiService.js:125-160`, `azureService.js:143-178`, `customService.js:219-254`,
+`ollamaService.js:128-160` in `analyzeDocument()`, `:348-383` in `_buildPrompt()` and `:499-534` in
+`_generateCustomFieldsTemplate()`), followed each time by the same
 `useExistingData` / `mustHavePrompt` / `RestrictionPromptService` / `USE_PROMPT_TAGS` /
 `customPrompt` cascade.
 
@@ -92,17 +98,19 @@ Produced by all four services and consumed by three call sites:
   errorCode?: string }
 ```
 
-Consumers: `server.js:766-821`, `services/mistralOcrService.js:988-997`,
-`routes/setup.js:3167-3177`. Any narrowing of `document` drops title, document type, date, language
-or custom fields silently — the write-back in `paperlessService.updateDocument()` simply stops
-setting those fields.
+Consumers: `server.js:854-890` (`processDocument()`), `services/mistralOcrService.js:1027-1052`
+(`_runAiAnalysis()`), `routes/setup.js:3135-3165` (`processDocument()`) and `POST /manual/analyze`
+(`routes/setup.js:7495-7537`), which returns the result as JSON. The first three hand `document` to
+`documentProcessingService.buildUpdateData()`, which since the merge is the single write-back for
+scan, queue and OCR. Any narrowing of `document` drops title, document type, date, language or
+custom fields silently — `buildUpdateData()` simply stops setting those fields.
 
 ### 4. The error contract is load-bearing and phrase-matched — it must be preserved verbatim
 
 `analyzeDocument()` **never throws**; it returns the failure in `error`/`errorCode`.
-`server.js:774-821` then routes the document to the OCR queue based on
+`server.js:870-890` (and `routes/setup.js:3160`) then routes the document to the OCR queue based on
 `shouldQueueForOcrOnAiError()` / `classifyOcrQueueReasonFromAiError()`
-(`services/serviceUtils.js:812-861`), which match on **exact phrases**:
+(`services/serviceUtils.js:847-900`), which match on **exact phrases**:
 
 - `'insufficient content for ai analysis'`
 - `'invalid response structure'`
@@ -110,9 +118,14 @@ setting those fields.
 - `'invalid json response from api'`
 - `'invalid api response structure'`
 
-Rewording any of these silently disables OCR fallback. Guarded by the `ocr-fallback-ai-errors` and
-`response-truncation-detection` tests. The `error.code = 'ai_response_truncated'` marker
-(`serviceUtils.js:802`, `ollamaService.js:920`) has the same status.
+Rewording any of these silently disables OCR fallback. Guarded by the `ocr-fallback-ai-errors`,
+`response-truncation-detection` and `llm-ocr-fallback-phrases` tests. The
+`error.code = 'ai_response_truncated'` marker (`serviceUtils.js:834`, `ollamaService.js:791`) has the
+same status.
+
+This already happened once: `38a5352` replaced Custom's "Invalid API response structure" with
+detailed messages ("API response is missing choices."), and `llm-ocr-fallback-phrases` went red.
+`355ac29` keeps the detail behind the phrase ("Invalid API response structure: missing choices.").
 
 Consequence for the design: `analyzeDocument()` keeps its non-throwing contract. A version that
 throws on, say, a missing `SYSTEM_PROMPT` would take down the scan loop instead of queueing the
@@ -123,12 +136,12 @@ document.
 Two separate functions, easily conflated:
 
 ```js
-// ollamaService.js:678-683 — conservative 2 chars/token, deliberate and documented
+// ollamaService.js:568-570 — conservative 2 chars/token, deliberate and documented
 _calculatePromptTokenCount(prompt) {
   return Math.ceil(prompt.length / 2);
 }
 
-// ollamaService.js:691-702 — tokens, with the answer reserved
+// ollamaService.js:581-588 — tokens, with the answer reserved
 _calculateNumCtx(promptTokenCount, expectedResponseTokens) {
   return Math.min(promptTokenCount + expectedResponseTokens, Number(config.tokenLimit));
 }
@@ -138,19 +151,29 @@ The `/2` divisor exists because non-English tokenization (CJK, German) produces
 characters per token instead of ~4; `manualService.js:234` used `/4` and would truncate those
 documents. Centralizing the estimators must **preserve `/2` for Ollama**, not unify the divisors.
 
-`_fitContentToContext()` (`ollamaService.js:379-409`) uses the same `/2` estimate by construction, so
+`_fitContentToContext()` (`ollamaService.js:290-320`) uses the same `/2` estimate by construction, so
 the two agree. Whatever holds this arithmetic after the refactor has to keep that property.
 
 ### 6. The transport seam needs a wide signature
 
-Ollama's transport needs `(prompt, systemPrompt, numCtx, schema)` (`ollamaService.js:811`), sends
+Ollama's transport needs `(prompt, systemPrompt, numCtx, schema)` (`ollamaService.js:678`), sends
 `format: schema` for structured output, computes `num_ctx`/`num_predict`, carries a bounded retry for
-transient failures (`:841-861`), and inspects `done_reason` for truncation (`:886-922`). Its response
-may arrive as a **JS object** rather than a string (`:929-949`).
+transient failures (`:643-735`, `_wrapOllamaRequestError()` at `:664`), and inspects `done_reason`
+for truncation (`:739-795`). Its response may arrive as a **JS object** rather than a string
+(`_processOllamaResponse()`, `:803`).
 
 A single-string `callAPI(prompt)` cannot carry any of that, which is why `CompletionRequest` below is
 a record rather than a string.
 
+`generateText(prompt, options)` widened the seam further with the merge. The Duplicates and
+"Simplify tags" features (`entityMatchAiService.js`, `tagSimplifyService.js`) call it with options
+that `services/aiGenerateOptions.js` reads the same way for all four providers: `systemPrompt`,
+`temperature` (where `0` is a value), a completion cap, a model override, an abort signal,
+reasoning on/off and a progress handler that streams partial text (`ai-provider-streaming` test).
+Upstream already unified the option parsing; what is still per provider is how each option reaches
+the API. The transport contract has to carry all of it, or the generation path regresses for those
+features.
+
 ### 7. behavioral drift between the four services — each item needs an explicit decision
 
 The refactor collapses four implementations into one, which forces a choice wherever they differ
@@ -158,18 +181,18 @@ today. These are the actual bugs the work will surface:
 
 | #   | Drift                                                                                                                       | Location                                                                                                                                                                         | Proposed resolution                                                                                                                                                                                                                                         |
 | --- | --------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`                                   | `azureService.js:258`, `:426`                                                                                                                                                    | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                                                                                                             |
+| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`                                   | `azureService.js:286`                                                                                                                                                            | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                                                                                                             |
 | b   | ~~`return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined`~~ **resolved**          | was `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                          | Fixed ahead of the refactor together with row j — same statement block, same root cause. See the note below the table.                                                                                                                                      |
-| c   | "Insufficient content" early return exists only in OpenAI                                                                   | `openaiService.js:310-332`, `:505-527`                                                                                                                                           | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                                                                                                                    |
+| c   | "Insufficient content" early return exists only in OpenAI                                                                   | `openaiService.js:310-332`                                                                                                                                                       | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                                                                                                                    |
 | d   | Three different JSON parsers                                                                                                | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | One parser built from custom's `_extractFirstJsonValue` + `<think>` stripping + Ollama's sanitize fallback; keep Ollama's object passthrough as a pre-step.                                                                                                 |
 | e   | Only custom normalizes timeouts                                                                                             | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                                                                                                                        |
-| f   | `generateText` max-token handling differs four ways                                                                         | openai: none; `azureService.js:528`: `max_tokens`; `customService.js:625-654`: clamped; `ollamaService.js:1105`: `num_predict`                                                   | Keep custom's clamping as the shared rule; the transport maps it to its own knob.                                                                                                                                                                           |
-| g   | `checkStatus` differs by nature                                                                                             | openai/custom `models.list()`; `azureService.js:574-580` REST GET; `ollamaService.js:1143` `/api/ps`                                                                             | Stays in the transport. Not business logic.                                                                                                                                                                                                                 |
-| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`                                    | `openaiService.js:33-44`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop.                                                                                          |
+| f   | `generateText` option handling — **mostly resolved upstream**                                                               | `services/aiGenerateOptions.js`, used by all four `generateText()` implementations                                                                                               | Upstream unified how options are read (completion cap, temperature, system prompt, model, abort, reasoning, progress). Keep `aiGenerateOptions.js` as the shared reader; the transports only map the result onto their own knobs.                           |
+| g   | `checkStatus` differs by nature                                                                                             | openai/custom `models.list()`; Azure REST GET; `ollamaService.js:1210` `/api/ps`                                                                                                 | Stays in the transport. Not business logic.                                                                                                                                                                                                                 |
+| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`                                    | `openaiService.js:46-60`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop.                                                                                          |
 | i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt                              | all four                                                                                                                                                                         | Still out of scope as a _placement_ question, but the four copies are now one call to `services/thumbnailCache.js`. Moving it out of `analyzeDocument` is a one-line change once a caller wants it.                                                         |
 | j   | ~~`getThumbnailCachePath(id)` runs **outside** the try block, so an invalid id makes `analyzeDocument` throw~~ **resolved** | was `openaiService.js:63`, `azureService.js:52`, `customService.js:147`                                                                                                          | Fixed ahead of the refactor. See the note below the table.                                                                                                                                                                                                  |
-| k   | An answer with neither tags nor a correspondent is a **failure** on three providers and a **success** on Ollama             | `openaiService.js:329-331`, `azureService.js`, `customService.js` vs. `ollamaService.js`                                                                                         | Ollama logs "No tags or correspondent found…" and returns an all-null `document` with populated `metrics` and **no `error`**. So the scan loop writes nulls back to Paperless and never falls back to OCR. Adopt the throwing behaviour of the other three. |
-| l   | An empty provider payload is worded "No response data from Ollama API", which matches **no** OCR marker                     | `ollamaService.js` vs. `openaiService.js:262`, `azureService.js`, `customService.js`                                                                                             | The other three say "Invalid API response structure", which `shouldQueueForOcrOnAiError()` matches. Unify the wording on the shared phrase, or add this one to the marker list.                                                                             |
+| k   | An answer with neither tags nor a correspondent is a **failure** on three providers and a **success** on Ollama             | `openaiService.js:316-329`, `azureService.js`, `customService.js` vs. `ollamaService.js:223`                                                                                     | Ollama logs "No tags or correspondent found…" and returns an all-null `document` with populated `metrics` and **no `error`**. So the scan loop writes nulls back to Paperless and never falls back to OCR. Adopt the throwing behaviour of the other three. |
+| l   | An empty provider payload is worded "No response data from Ollama API", which matches **no** OCR marker                     | `ollamaService.js:821` vs. `openaiService.js`, `azureService.js`, `customService.js`                                                                                             | The other three say "Invalid API response structure", which `shouldQueueForOcrOnAiError()` matches. Unify the wording on the shared phrase, or add this one to the marker list.                                                                             |
 | m   | Custom classifies `socket hang up` as a timeout and replaces the message                                                    | `customService.js:331`                                                                                                                                                           | Consequence of row e: `isTimeoutError()` swallows the original text into "AI response timeout reached…". Decide deliberately whether a dropped connection is a timeout when the handling is unified.                                                        |
 
 Rows k and l were found by writing the step-1 safety net, not by reading the code — both are
@@ -305,8 +328,7 @@ class DocumentAnalyzer {
   async analyzeDocument(content, existingTags = [], existingCorrespondentList = [],
                         existingDocumentTypesList = [], id, customPrompt = null, options = {}) { ... }
 
-  async analyzePlayground(content, prompt) { ... }
-  async generateText(prompt) { ... }
+  async generateText(prompt, options = {}) { ... }   // options as read by aiGenerateOptions.js
   checkStatus() { return this.transport.checkStatus(); }
 }
 ```
@@ -409,13 +431,14 @@ Planned tests (each registered in `TESTS` in `scripts/run-tests.js` and in an `A
 | `llm-transport-request-mapping`       | chat transports send `messages[]` with system+user and omit `temperature` for `o3-mini`; Ollama transport sends `format`, `num_ctx`, `num_predict`, `think:false` unless `OLLAMA_THINK=true`                                                                        |
 | `llm-azure-temperature-wiring`        | Azure analysis requests carry `config.aiTemperatureAnalysis`, not a hardcoded `0.3` — Finding 7a                                                                                                                                                                    |
 | ~~`llm-missing-thumbnail-continues`~~ | **already delivered** as `tests/test-thumbnail-optional-analysis.js` (Findings 7b/7j). Keep it green through the refactor instead of writing a new one; it exercises `analyzeDocument` end to end, so the composition roots must keep calling `cacheThumbnail(id)`. |
-| `llm-provider-parity`                 | all four composition roots expose `analyzeDocument`/`analyzePlayground`/`generateText`/`checkStatus` and, given the same fake transport payload, produce byte-identical `document` objects                                                                          |
+| `llm-provider-parity`                 | all four composition roots expose `analyzeDocument`/`generateText`/`checkStatus` and, given the same fake transport payload, produce byte-identical `document` objects                                                                                              |
 
 Existing tests that must stay green without modification (they are the real acceptance criteria):
 `ocr-fallback-ai-errors`, `response-truncation-detection`, `ollama-response-limit`,
 `ollama-upstream-error`, `ollama-temperature-wiring`, `ollama-token-metrics`,
 `prompt-existing-data-serialization`, `restriction-service`, `restricted-document-types-placeholder`,
-`document-type-restriction`, `ai-temperature-config`, `playground-deprecation`.
+`document-type-restriction`, `ai-temperature-config`, `ai-provider-streaming`.
+(`playground-deprecation` was retired upstream together with the Playground, #359.)
 
 Repo hygiene for every step:
 
@@ -459,16 +482,16 @@ carry no behavioral risk; the risk starts at step 4.
 6. **Introduce `documentAnalyzer.js`** and the four composition roots. Reduce
    `services/{openai,azure,custom,ollama}Service.js` to re-export shims. Add `llm-provider-parity`.
    Fix Finding 7a here (Azure temperature) with its test.
-7. **Delete `services/manualService.js`** and repoint `POST /manual/analyze` and
-   `POST /manual/playground` (`routes/setup.js:7511`, `:7657`) at `AIServiceFactory.getService()`,
-   collapsing both four-way switches. Clean up what used to trigger Finding 7j while here: all four
-   branches pass `id || []` (`routes/setup.js:7535`, `:7550`, `:7559`, `:7568`), and `String([])` is
+7. **Delete `services/manualService.js`** and repoint `POST /manual/analyze`
+   (`routes/setup.js:7477`) at `AIServiceFactory.getService()`, collapsing its four-way switch.
+   Clean up what used to trigger Finding 7j while here: all four branches pass `id || []`
+   (`routes/setup.js:7501`, `:7516`, `:7525`, `:7534`), and `String([])` is
    `''`, which `getThumbnailCachePath()` rejects. That no longer costs the analysis — the request now
    succeeds with a warning — but `id || []` is still a nonsense default; pass the id through as-is,
    or require it. Note that
    `/manual/analyze` currently calls
-   `documentModel.addOpenAIMetrics()` only on the OpenAI branch while `/manual/playground` calls it on
-   three of four — decide explicitly whether metrics are recorded for every provider, and if the
+   `documentModel.addOpenAIMetrics()` only on the OpenAI branch — decide explicitly whether metrics
+   are recorded for every provider, and if the
    response shape changes, update the `@swagger` JSDoc and run `node scripts/regen-openapi.js`
    (the CI drift check will fail otherwise).
 8. **Verify Finding 7h** (unreachable `initialize()` branches in the former `openaiService`) and drop
@@ -488,8 +511,11 @@ carry no behavioral risk; the risk starts at step 4.
   single `cacheThumbnail(id)` call per provider rather than four inline copies, so hoisting it to
   the caller is a small change whenever someone wants to make it.
 - **Vision/multimodal analysis**, which is presumably why the thumbnail is cached at all.
-- **`analyzePlayground` deduplication beyond the shared collaborators** — the endpoint is already
-  deprecated (`routes/setup.js:7584`, `playground-deprecation` test); do not invest in it.
+- **`CachedNameIdEnum`'s semantic match calls a method no provider has.** `_findSemanticMatch()`
+  (`services/CachedNameIdEnum.js:176`) calls `ai_service.generateAnswer()`, which exists only in
+  test stubs. The call throws, is caught and returns "no match", so tag/correspondent/document type
+  lookup silently falls back to its text matchers. Either point it at `generateText(prompt, options)`
+  with a JSON schema once step 6 has settled that contract, or drop the AI tier.
 - **A structured-output (`response_format`/`format`) path for the chat transports** to match what
   Ollama already does. Would likely remove most of `responseParser`, but it is a behavior change
   needing its own validation against real providers.
@@ -499,10 +525,10 @@ carry no behavioral risk; the risk starts at step 4.
 
 ## Risks
 
-| Risk                                                                         | Mitigation                                                                                                                           |
-| ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
-| OCR fallback silently stops working because an error phrase changed          | `llm-ocr-fallback-phrases` written in step 1, before any code moves                                                                  |
-| A caller reads a field that the unified `document` object no longer carries  | `llm-provider-parity` compares full objects; `server.js:766`, `mistralOcrService.js:988`, `routes/setup.js:3167` reviewed explicitly |
-| Ollama's token arithmetic subtly changes and long documents start truncating | `charBudget` keeps the `/2` divisor and the real `num_ctx` formula; `ollama-response-limit` and `ollama-token-metrics` guard it      |
-| No test exercises a real provider API                                        | Step 9 keeps a manual smoke run against at least one live provider in the acceptance criteria                                        |
-| The refactor lands as one unreviewable diff                                  | Steps 2-6 are individually green and individually landable                                                                           |
+| Risk                                                                         | Mitigation                                                                                                                            |
+| ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- |
+| OCR fallback silently stops working because an error phrase changed          | `llm-ocr-fallback-phrases` written in step 1, before any code moves                                                                   |
+| A caller reads a field that the unified `document` object no longer carries  | `llm-provider-parity` compares full objects; `server.js:854`, `mistralOcrService.js:1027`, `routes/setup.js:3135` reviewed explicitly |
+| Ollama's token arithmetic subtly changes and long documents start truncating | `charBudget` keeps the `/2` divisor and the real `num_ctx` formula; `ollama-response-limit` and `ollama-token-metrics` guard it       |
+| No test exercises a real provider API                                        | Step 9 keeps a manual smoke run against at least one live provider in the acceptance criteria                                         |
+| The refactor lands as one unreviewable diff                                  | Steps 2-6 are individually green and individually landable                                                                            |

From 56b278fcc9aec40e199ca33a5f3a0cf801b36278 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Mon, 5 Oct 2026 00:57:29 +0200
Subject: [PATCH 34/53] Build the analysis prompt once for all four providers

Background
Step 2 of the LLM service refactoring (services/ManualServiceRefactoring.md).
The %CUSTOMFIELDS% block and the prompt cascade after it were copied into
every provider service: openai, azure and custom once each, Ollama three
times.

Changes
- services/llm/customFieldsTemplate.js: buildCustomFieldsBlock(), the
  %CUSTOMFIELDS% fragment with its per-data-type hints.
- services/llm/promptBuilder.js: buildAnalysisPrompt() returns
  { systemPrompt, promptTags } - existing data, mustHavePrompt, restriction
  placeholders, external API data, USE_PROMPT_TAGS and the custom prompt,
  in the order the providers applied them.
- openai/azure/custom: the inline cascade is replaced by the builder.
- Ollama: _buildPrompt() and _generateCustomFieldsTemplate() delegate to
  the builder (the prompt tests call _buildPrompt() directly); the custom
  prompt path uses it too.
- Ollama: _validateAndTruncateExternalApiData() is synchronous. It awaited
  nothing, but was declared async and never awaited, so Ollama sent
  "[object Promise]" instead of the external API data.
- New test llm-prompt-builder (27 cases, area prompts).
- Plan: step 2 marked done.

Testing
Recorded the requests of all four providers for eight prompt scenarios
before and after: openai/azure/custom are byte-identical (24 of 24).
Ollama differs only in the indentation of its two fixed text blocks (8
instead of 12 spaces, like the others) and in sending the external API
data. node scripts/run-tests.js --all: no new failures;
llm-ocr-fallback-phrases is green since 355ac29. The remaining 15 failures
are the Windows-only ones from before. ESLint and Prettier clean.

Impact
Ollama users with external API data get it into the prompt for the first
time. No other prompt changes.

Upstream Status
Local only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 scripts/run-tests.js                 |   2 +
 services/ManualServiceRefactoring.md |  22 +-
 services/azureService.js             | 105 +-----
 services/customService.js            | 108 +------
 services/llm/customFieldsTemplate.js |  71 ++++
 services/llm/promptBuilder.js        | 113 +++++++
 services/ollamaService.js            | 181 +----------
 services/openaiService.js            | 105 +-----
 tests/test-llm-prompt-builder.js     | 468 +++++++++++++++++++++++++++
 9 files changed, 702 insertions(+), 473 deletions(-)
 create mode 100644 services/llm/customFieldsTemplate.js
 create mode 100644 services/llm/promptBuilder.js
 create mode 100644 tests/test-llm-prompt-builder.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index 064352f4..eac689c6 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -99,6 +99,7 @@ const TESTS = {
     'test-custom-field-monetary-normalization.js',
   'llm-analysis-result-contract': 'test-llm-analysis-result-contract.js',
   'llm-ocr-fallback-phrases': 'test-llm-ocr-fallback-phrases.js',
+  'llm-prompt-builder': 'test-llm-prompt-builder.js',
   'custom-field-longtext': 'test-custom-field-longtext.js',
   'document-processing-service': 'test-document-processing-service.js',
   'settings-custom-field-type-conflict':
@@ -243,6 +244,7 @@ const AREAS = {
     'updated-service',
     'prompt-existing-data-serialization',
     'restricted-document-types-placeholder',
+    'llm-prompt-builder',
     'token-limit-config',
     'truncate-returns-string',
   ],
diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index c3317a56..769775d1 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -466,9 +466,25 @@ carry no behavioral risk; the risk starts at step 4.
    are copied into the test rather than imported. Importing that list would make the test agree
    with whatever the source says, which is the opposite of pinning it.
 
-2. **Extract `customFieldsTemplate.js` and `promptBuilder.js`** with `llm-prompt-builder`, and have
-   all four existing services call them. Six duplicates become one; no other behavior changes.
-   `prompt-existing-data-serialization` and the restriction tests are the guard.
+2. ~~**Extract `customFieldsTemplate.js` and `promptBuilder.js`**~~ **Done.**
+   `services/llm/customFieldsTemplate.js` (`buildCustomFieldsBlock()`) and
+   `services/llm/promptBuilder.js` (`buildAnalysisPrompt()` → `{ systemPrompt, promptTags }`) replace
+   the six copies. `llm-prompt-builder` (27 cases) pins the cascade and checks that every provider
+   sends what the builder built. `ollamaService._buildPrompt()` stays as a thin wrapper, because
+   `prompt-existing-data-serialization` and `restricted-document-types-placeholder` call it.
+
+   Verified by recording the requests of all four providers for eight prompt scenarios before and
+   after: the 24 requests of openai/azure/custom are byte-identical. Ollama differs in two
+   deliberate ways. Its two fixed text blocks are now indented with 8 spaces instead of 12, like
+   the other three; the wording is unchanged. And it sends the external API data instead of
+   `[object Promise]`: its `_validateAndTruncateExternalApiData()` was `async` but never awaited
+   by either caller. It awaited nothing itself, so it is synchronous now.
+
+   Not changed on purpose: a `CUSTOM_FIELDS` that parses but has no `custom_fields` array still
+   throws inside `buildCustomFieldsBlock()` (and so fails the analysis), as all six copies did.
+   Ollama still ignores `PROMPT_TAGS` as user content; the chat providers only use it for the
+   token count.
+
 3. **Extract `responseParser.js`** (custom's brace matcher + `<think>` stripping + Ollama's sanitize
    fallback + Ollama's object passthrough) with `llm-response-parser`, and route all four services
    through it. Resolves Finding 7d.
diff --git a/services/azureService.js b/services/azureService.js
index 182b96d4..f501db10 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -5,7 +5,6 @@ const {
   writePromptToFile,
   extractChatMessageContent,
   assertCompletionNotTruncated,
-  toNameList,
 } = require('./serviceUtils');
 const {
   abortSignal,
@@ -27,7 +26,7 @@ const config = require('../config/config');
 const fs = require('fs').promises;
 const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
-const RestrictionPromptService = require('./restrictionPromptService');
+const { buildAnalysisPrompt } = require('./llm/promptBuilder');
 const responseLogPath = path.join(
   process.cwd(),
   'data',
@@ -106,15 +105,6 @@ class AzureOpenAIService {
       // Cache the thumbnail for the UI; never let it affect the analysis.
       await cacheThumbnail(id);
 
-      // Format existing data - callers may hand over entity objects or plain names
-      const existingTagNames = toNameList(existingTags).join(', ');
-      const existingCorrespondentNames = toNameList(
-        existingCorrespondentList
-      ).join(', ');
-      const existingDocumentTypeNames = toNameList(
-        existingDocumentTypesList
-      ).join(', ');
-
       // Get external API data if available and validate it
       let externalApiData = options.externalApiData || null;
       let validatedExternalApiData = null;
@@ -133,101 +123,18 @@ class AzureOpenAIService {
         }
       }
 
-      let systemPrompt = '';
-      let promptTags = '';
       const model = process.env.AZURE_DEPLOYMENT_NAME;
-
-      // Parse CUSTOM_FIELDS from environment variable
-      let customFieldsObj;
-      try {
-        customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-      } catch (error) {
-        console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
-        console.debug(error);
-        customFieldsObj = { custom_fields: [] };
-      }
-
-      // Generate custom fields template for the prompt
-      const customFieldsTemplate = {};
-
-      customFieldsObj.custom_fields.forEach((field, index) => {
-        let valueHint;
-        if (field.data_type === 'date') {
-          valueHint =
-            'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
-        } else if (field.data_type === 'boolean') {
-          valueHint = "Fill in 'true' or 'false' based on your analysis";
-        } else {
-          valueHint = 'Fill in the value based on your analysis';
-        }
-        customFieldsTemplate[index] = {
-          field_name: field.value,
-          value: valueHint,
-        };
-      });
-
-      // Convert template to string for replacement and wrap in custom_fields
-      const customFieldsStr =
-        '"custom_fields": ' +
-        JSON.stringify(customFieldsTemplate, null, 2)
-          .split('\n')
-          .map((line) => '    ' + line) // Add proper indentation
-          .join('\n');
-
-      // Get system prompt and model
-      if (
-        config.useExistingData === 'yes' &&
-        config.restrictToExistingTags === 'no' &&
-        config.restrictToExistingCorrespondents === 'no'
-      ) {
-        systemPrompt =
-          `
-        Pre-existing tags: ${existingTagNames}\n\n
-        Pre-existing correspondents: ${existingCorrespondentNames}\n\n
-        Pre-existing document types: ${existingDocumentTypeNames}\n\n
-        ` +
-          process.env.SYSTEM_PROMPT +
-          '\n\n' +
-          config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
-        promptTags = '';
-      } else {
-        const mustHavePrompt = config.mustHavePrompt.replace(
-          '%CUSTOMFIELDS%',
-          customFieldsStr
-        );
-        systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt;
-        promptTags = '';
-      }
-
-      // Process placeholder replacements in system prompt
-      systemPrompt = RestrictionPromptService.processRestrictionsInPrompt(
-        systemPrompt,
+      const { systemPrompt, promptTags } = buildAnalysisPrompt({
         existingTags,
         existingCorrespondentList,
-        existingDocumentTypesList
-      );
-
-      // Include validated external API data if available
-      if (validatedExternalApiData) {
-        systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`;
-      }
-
-      if (process.env.USE_PROMPT_TAGS === 'yes') {
-        promptTags = process.env.PROMPT_TAGS;
-        systemPrompt =
-          `
-        Take these tags and try to match one or more to the document content.\n\n
-        ` + config.specialPromptPreDefinedTags;
-      }
-
+        existingDocumentTypesList,
+        externalApiData: validatedExternalApiData,
+        customPrompt,
+      });
       if (customPrompt) {
         console.log(
           '[DEBUG] Replace system prompt with custom prompt via WebHook'
         );
-        systemPrompt =
-          customPrompt +
-          '\n\n' +
-          config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
       }
 
       // Calculate tokens AFTER all prompt modifications are complete
diff --git a/services/customService.js b/services/customService.js
index 0f314466..b11444c4 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -6,7 +6,6 @@ const {
   assertCompletionNotTruncated,
   isTimeoutError,
   buildTimeoutErrorMessage,
-  toNameList,
 } = require('./serviceUtils');
 const {
   abortSignal,
@@ -27,7 +26,7 @@ const config = require('../config/config');
 const fs = require('fs').promises;
 const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
-const RestrictionPromptService = require('./restrictionPromptService');
+const { buildAnalysisPrompt } = require('./llm/promptBuilder');
 const responseLogPath = path.join(
   process.cwd(),
   'data',
@@ -182,15 +181,6 @@ class CustomOpenAIService {
       // Cache the thumbnail for the UI; never let it affect the analysis.
       await cacheThumbnail(id);
 
-      // Format existing data - callers may hand over entity objects or plain names
-      const existingTagNames = toNameList(existingTags).join(', ');
-      const existingCorrespondentNames = toNameList(
-        existingCorrespondentList
-      ).join(', ');
-      const existingDocumentTypeNames = toNameList(
-        existingDocumentTypesList
-      ).join(', ');
-
       // Get external API data if available and validate it
       let externalApiData = options.externalApiData || null;
       let validatedExternalApiData = null;
@@ -209,100 +199,16 @@ class CustomOpenAIService {
         }
       }
 
-      let systemPrompt = '';
-      let promptTags = '';
       const model = config.custom.model;
-
-      // Parse CUSTOM_FIELDS from environment variable
-      let customFieldsObj;
-      try {
-        customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-      } catch (error) {
-        console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
-        console.debug(error);
-        customFieldsObj = { custom_fields: [] };
-      }
-
-      // Generate custom fields template for the prompt
-      const customFieldsTemplate = {};
-
-      customFieldsObj.custom_fields.forEach((field, index) => {
-        let valueHint;
-        if (field.data_type === 'date') {
-          valueHint =
-            'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
-        } else if (field.data_type === 'boolean') {
-          valueHint = "Fill in 'true' or 'false' based on your analysis";
-        } else {
-          valueHint = 'Fill in the value based on your analysis';
-        }
-        customFieldsTemplate[index] = {
-          field_name: field.value,
-          value: valueHint,
-        };
-      });
-
-      // Convert template to string for replacement and wrap in custom_fields
-      const customFieldsStr =
-        '"custom_fields": ' +
-        JSON.stringify(customFieldsTemplate, null, 2)
-          .split('\n')
-          .map((line) => '    ' + line) // Add proper indentation
-          .join('\n');
-
-      // Get system prompt based on configuration
-      if (
-        config.useExistingData === 'yes' &&
-        config.restrictToExistingTags === 'no' &&
-        config.restrictToExistingCorrespondents === 'no'
-      ) {
-        systemPrompt =
-          `
-        Pre-existing tags: ${existingTagNames}\n\n
-        Pre-existing correspondents: ${existingCorrespondentNames}\n\n
-        Pre-existing document types: ${existingDocumentTypeNames}\n\n
-        ` +
-          process.env.SYSTEM_PROMPT +
-          '\n\n' +
-          config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
-        promptTags = '';
-      } else {
-        const mustHavePrompt = config.mustHavePrompt.replace(
-          '%CUSTOMFIELDS%',
-          customFieldsStr
-        );
-        systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt;
-        promptTags = '';
-      }
-
-      // Process placeholder replacements in system prompt
-      systemPrompt = RestrictionPromptService.processRestrictionsInPrompt(
-        systemPrompt,
+      const { systemPrompt, promptTags } = buildAnalysisPrompt({
         existingTags,
         existingCorrespondentList,
-        existingDocumentTypesList
-      );
-
-      // Include validated external API data if available
-      if (validatedExternalApiData) {
-        systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`;
-      }
-
-      if (process.env.USE_PROMPT_TAGS === 'yes') {
-        promptTags = process.env.PROMPT_TAGS;
-        systemPrompt =
-          `
-        Take these tags and try to match one or more to the document content.\n\n
-        ` + config.specialPromptPreDefinedTags;
-      }
-
-      // Custom prompt override if provided
+        existingDocumentTypesList,
+        externalApiData: validatedExternalApiData,
+        customPrompt,
+      });
       if (customPrompt) {
         console.log('[DEBUG] Replace system prompt with custom prompt');
-        systemPrompt =
-          customPrompt +
-          '\n\n' +
-          config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
       }
 
       // Calculate tokens AFTER all prompt modifications are complete
@@ -351,8 +257,6 @@ class CustomOpenAIService {
       // console.log(`[DEBUG] Prompt tags: ${promptTags}`);
       // console.log(`[DEBUG] Model: ${model}`);
       // console.log(`[DEBUG] Custom fields: ${customFieldsStr}`);
-      // console.log(`[DEBUG] Existing tags: ${existingTagNames}`);
-      // console.log(`[DEBUG] Existing correspondents: ${existingCorrespondentNames}`);
       // console.log(`[DEBUG] Custom prompt: ${customPrompt}`);
       // console.log(`[DEBUG] External API data: ${validatedExternalApiData}`);
       // console.log('######################################################################');
diff --git a/services/llm/customFieldsTemplate.js b/services/llm/customFieldsTemplate.js
new file mode 100644
index 00000000..7da1012a
--- /dev/null
+++ b/services/llm/customFieldsTemplate.js
@@ -0,0 +1,71 @@
+'use strict';
+
+/**
+ * The %CUSTOMFIELDS% block of the analysis prompt.
+ *
+ * Renders the custom fields configured in CUSTOM_FIELDS as the JSON fragment
+ * the model is asked to fill in, one entry per field with a hint for its data
+ * type. Every provider used to carry its own copy of this; Ollama had three.
+ */
+
+const DATE_HINT =
+  'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
+const BOOLEAN_HINT = "Fill in 'true' or 'false' based on your analysis";
+const VALUE_HINT = 'Fill in the value based on your analysis';
+
+/**
+ * The value hint the model gets for one data type.
+ *
+ * @param {string} dataType Paperless-ngx custom field data type.
+ * @returns {string}
+ */
+function valueHintFor(dataType) {
+  if (dataType === 'date') return DATE_HINT;
+  if (dataType === 'boolean') return BOOLEAN_HINT;
+  return VALUE_HINT;
+}
+
+/**
+ * The configured custom fields, or none when the setting cannot be read.
+ *
+ * @param {string|undefined} rawCustomFields CUSTOM_FIELDS as stored.
+ * @returns {{custom_fields: Array<{value: string, data_type: string}>}}
+ */
+function parseCustomFields(rawCustomFields) {
+  try {
+    return JSON.parse(rawCustomFields);
+  } catch (error) {
+    console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
+    console.debug(error);
+    return { custom_fields: [] };
+  }
+}
+
+/**
+ * Build the `"custom_fields": {...}` fragment that replaces %CUSTOMFIELDS%.
+ *
+ * The fragment is indented so it sits inside the JSON example of
+ * mustHavePrompt; the model copies that layout back.
+ *
+ * @param {string|undefined} [rawCustomFields=process.env.CUSTOM_FIELDS]
+ * @returns {string}
+ */
+function buildCustomFieldsBlock(rawCustomFields = process.env.CUSTOM_FIELDS) {
+  const template = {};
+  parseCustomFields(rawCustomFields).custom_fields.forEach((field, index) => {
+    template[index] = {
+      field_name: field.value,
+      value: valueHintFor(field.data_type),
+    };
+  });
+
+  return (
+    '"custom_fields": ' +
+    JSON.stringify(template, null, 2)
+      .split('\n')
+      .map((line) => '    ' + line)
+      .join('\n')
+  );
+}
+
+module.exports = { buildCustomFieldsBlock };
diff --git a/services/llm/promptBuilder.js b/services/llm/promptBuilder.js
new file mode 100644
index 00000000..5c8059dd
--- /dev/null
+++ b/services/llm/promptBuilder.js
@@ -0,0 +1,113 @@
+'use strict';
+
+/**
+ * The analysis instructions, assembled once for every provider.
+ *
+ * openai, azure and custom send the result as their system message; Ollama
+ * puts it in front of the document in its prompt and keeps its own short
+ * system prompt. Either way the cascade below is the same, and it used to be
+ * copied into each of them.
+ */
+
+const config = require('../../config/config');
+const RestrictionPromptService = require('../restrictionPromptService');
+const { toNameList } = require('../serviceUtils');
+const { buildCustomFieldsBlock } = require('./customFieldsTemplate');
+
+// The layout of the two literal blocks below is what the providers have always
+// sent, down to the whitespace. It is kept as it was so that moving the code
+// here does not change a single prompt.
+const BLOCK_INDENT = '        ';
+
+function preExistingDataBlock(tags, correspondents, documentTypes) {
+  return (
+    `\n${BLOCK_INDENT}Pre-existing tags: ${tags}\n\n\n` +
+    `${BLOCK_INDENT}Pre-existing correspondents: ${correspondents}\n\n\n` +
+    `${BLOCK_INDENT}Pre-existing document types: ${documentTypes}\n\n\n` +
+    BLOCK_INDENT
+  );
+}
+
+const PREDEFINED_TAGS_INTRO =
+  `\n${BLOCK_INDENT}Take these tags and try to match one or more to the document content.\n\n\n` +
+  BLOCK_INDENT;
+
+/**
+ * Build the instructions for one document analysis.
+ *
+ * In order, each step overriding or extending the previous one:
+ *  1. SYSTEM_PROMPT plus mustHavePrompt with the custom fields block; with
+ *     USE_EXISTING_DATA (and no restriction) the existing tags,
+ *     correspondents and document types in front of it.
+ *  2. The %RESTRICTED_*% placeholders resolved.
+ *  3. The external API data appended.
+ *  4. With USE_PROMPT_TAGS, all of the above replaced by the predefined-tags
+ *     prompt; PROMPT_TAGS is handed out as promptTags.
+ *  5. A custom prompt (webhook) replaces all of it, followed by mustHavePrompt.
+ *
+ * @param {Object} input
+ * @param {Array<Object|string>} [input.existingTags]
+ * @param {Array<Object|string>} [input.existingCorrespondentList]
+ * @param {Array<Object|string>} [input.existingDocumentTypesList]
+ * @param {string|null} [input.externalApiData] Already validated and truncated.
+ * @param {string|null} [input.customPrompt]
+ * @param {string} [input.customFieldsBlock] Defaults to the configured fields.
+ * @returns {{systemPrompt: string, promptTags: string}}
+ */
+function buildAnalysisPrompt({
+  existingTags = [],
+  existingCorrespondentList = [],
+  existingDocumentTypesList = [],
+  externalApiData = null,
+  customPrompt = null,
+  customFieldsBlock = buildCustomFieldsBlock(),
+} = {}) {
+  const mustHavePrompt = config.mustHavePrompt.replace(
+    '%CUSTOMFIELDS%',
+    customFieldsBlock
+  );
+  let systemPrompt;
+  let promptTags = '';
+
+  if (
+    config.useExistingData === 'yes' &&
+    config.restrictToExistingTags === 'no' &&
+    config.restrictToExistingCorrespondents === 'no'
+  ) {
+    systemPrompt =
+      preExistingDataBlock(
+        toNameList(existingTags).join(', '),
+        toNameList(existingCorrespondentList).join(', '),
+        toNameList(existingDocumentTypesList).join(', ')
+      ) +
+      process.env.SYSTEM_PROMPT +
+      '\n\n' +
+      mustHavePrompt;
+  } else {
+    systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt;
+  }
+
+  systemPrompt = RestrictionPromptService.processRestrictionsInPrompt(
+    systemPrompt,
+    existingTags,
+    existingCorrespondentList,
+    existingDocumentTypesList
+  );
+
+  if (externalApiData) {
+    systemPrompt += `\n\nAdditional context from external API:\n${externalApiData}`;
+  }
+
+  if (process.env.USE_PROMPT_TAGS === 'yes') {
+    promptTags = process.env.PROMPT_TAGS;
+    systemPrompt = PREDEFINED_TAGS_INTRO + config.specialPromptPreDefinedTags;
+  }
+
+  if (customPrompt) {
+    systemPrompt = customPrompt + '\n\n' + mustHavePrompt;
+  }
+
+  return { systemPrompt, promptTags };
+}
+
+module.exports = { buildAnalysisPrompt };
diff --git a/services/ollamaService.js b/services/ollamaService.js
index 1aadac55..a891e73e 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -1,4 +1,4 @@
-const { writePromptToFile, toNameList } = require('./serviceUtils');
+const { writePromptToFile } = require('./serviceUtils');
 const {
   abortSignal,
   abortedGenerationError,
@@ -14,7 +14,8 @@ const {
 const axios = require('axios');
 const config = require('../config/config');
 const { cacheThumbnail } = require('./thumbnailCache');
-const RestrictionPromptService = require('./restrictionPromptService');
+const { buildAnalysisPrompt } = require('./llm/promptBuilder');
+const { buildCustomFieldsBlock } = require('./llm/customFieldsTemplate');
 
 /**
  * Service for document analysis using Ollama
@@ -122,45 +123,8 @@ class OllamaService {
           options
         );
       } else {
-        // Parse CUSTOM_FIELDS for custom prompt
-        let customFieldsObj;
-        try {
-          customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-        } catch (error) {
-          console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
-          console.debug(error);
-          customFieldsObj = { custom_fields: [] };
-        }
-
-        const customFieldsTemplate = {};
-        customFieldsObj.custom_fields.forEach((field, index) => {
-          let valueHint;
-          if (field.data_type === 'date') {
-            valueHint =
-              'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
-          } else if (field.data_type === 'boolean') {
-            valueHint = "Fill in 'true' or 'false' based on your analysis";
-          } else {
-            valueHint = 'Fill in the value based on your analysis';
-          }
-          customFieldsTemplate[index] = {
-            field_name: field.value,
-            value: valueHint,
-          };
-        });
-
-        const customFieldsStr =
-          '"custom_fields": ' +
-          JSON.stringify(customFieldsTemplate, null, 2)
-            .split('\n')
-            .map((line) => '    ' + line)
-            .join('\n');
-
         promptPrefix =
-          customPrompt +
-          '\n\n' +
-          config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr) +
-          '\n\n';
+          buildAnalysisPrompt({ customPrompt }).systemPrompt + '\n\n';
         console.log('[DEBUG] Ollama Service started with custom prompt');
       }
 
@@ -335,85 +299,11 @@ class OllamaService {
     existingDocumentTypes = [],
     options = {}
   ) {
-    let systemPrompt;
-
     // Validate that existingCorrespondent is an array and handle if it's not
     const correspondentList = Array.isArray(existingCorrespondent)
       ? existingCorrespondent
       : [];
 
-    // Parse CUSTOM_FIELDS from environment variable
-    let customFieldsObj;
-    try {
-      customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-    } catch (error) {
-      console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
-      console.debug(error);
-      customFieldsObj = { custom_fields: [] };
-    }
-
-    // Generate custom fields template for the prompt
-    const customFieldsTemplate = {};
-
-    customFieldsObj.custom_fields.forEach((field, index) => {
-      let valueHint;
-      if (field.data_type === 'date') {
-        valueHint =
-          'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
-      } else if (field.data_type === 'boolean') {
-        valueHint = "Fill in 'true' or 'false' based on your analysis";
-      } else {
-        valueHint = 'Fill in the value based on your analysis';
-      }
-      customFieldsTemplate[index] = {
-        field_name: field.value,
-        value: valueHint,
-      };
-    });
-
-    // Convert template to string for replacement and wrap in custom_fields
-    const customFieldsStr =
-      '"custom_fields": ' +
-      JSON.stringify(customFieldsTemplate, null, 2)
-        .split('\n')
-        .map((line) => '    ' + line) // Add proper indentation
-        .join('\n');
-
-    // Get system prompt based on configuration
-    if (
-      config.useExistingData === 'yes' &&
-      config.restrictToExistingTags === 'no' &&
-      config.restrictToExistingCorrespondents === 'no'
-    ) {
-      // Format existing tags
-      const existingTagsList = toNameList(existingTags).join(', ');
-
-      // Format existing correspondents
-      const existingCorrespondentList =
-        toNameList(correspondentList).join(', ');
-
-      // Format existing document types
-      const existingDocumentTypesList = toNameList(existingDocumentTypes).join(
-        ', '
-      );
-
-      systemPrompt =
-        `
-            Pre-existing tags: ${existingTagsList}\n\n
-            Pre-existing correspondents: ${existingCorrespondentList}\n\n
-            Pre-existing document types: ${existingDocumentTypesList}\n\n
-            ` +
-        process.env.SYSTEM_PROMPT +
-        '\n\n' +
-        config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
-    } else {
-      const mustHavePrompt = config.mustHavePrompt.replace(
-        '%CUSTOMFIELDS%',
-        customFieldsStr
-      );
-      systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt;
-    }
-
     // Get validated external API data if available
     let validatedExternalApiData = null;
     if (options.externalApiData) {
@@ -431,26 +321,12 @@ class OllamaService {
       }
     }
 
-    // Process placeholder replacements in system prompt
-    systemPrompt = RestrictionPromptService.processRestrictionsInPrompt(
-      systemPrompt,
+    const { systemPrompt } = buildAnalysisPrompt({
       existingTags,
-      correspondentList,
-      existingDocumentTypes,
-      config
-    );
-
-    // Include validated external API data if available
-    if (validatedExternalApiData) {
-      systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`;
-    }
-
-    if (process.env.USE_PROMPT_TAGS === 'yes') {
-      systemPrompt =
-        `
-            Take these tags and try to match one or more to the document content.\n\n
-            ` + config.specialPromptPreDefinedTags;
-    }
+      existingCorrespondentList: correspondentList,
+      existingDocumentTypesList: existingDocumentTypes,
+      externalApiData: validatedExternalApiData,
+    });
 
     return `${systemPrompt}
         ${JSON.stringify(content)}
@@ -463,7 +339,7 @@ class OllamaService {
    * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500)
    * @returns {string} - Validated and potentially truncated data string
    */
-  async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
+  _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
     if (!apiData) {
       return null;
     }
@@ -494,42 +370,7 @@ class OllamaService {
    * @returns {string} Custom fields template as a string
    */
   _generateCustomFieldsTemplate() {
-    let customFieldsObj;
-    try {
-      customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-    } catch (error) {
-      console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
-      console.debug(error);
-      customFieldsObj = { custom_fields: [] };
-    }
-
-    // Generate custom fields template for the prompt
-    const customFieldsTemplate = {};
-
-    customFieldsObj.custom_fields.forEach((field, index) => {
-      let valueHint;
-      if (field.data_type === 'date') {
-        valueHint =
-          'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
-      } else if (field.data_type === 'boolean') {
-        valueHint = "Fill in 'true' or 'false' based on your analysis";
-      } else {
-        valueHint = 'Fill in the value based on your analysis';
-      }
-      customFieldsTemplate[index] = {
-        field_name: field.value,
-        value: valueHint,
-      };
-    });
-
-    // Convert template to string for replacement and wrap in custom_fields
-    return (
-      '"custom_fields": ' +
-      JSON.stringify(customFieldsTemplate, null, 2)
-        .split('\n')
-        .map((line) => '    ' + line) // Add proper indentation
-        .join('\n')
-    );
+    return buildCustomFieldsBlock();
   }
 
   /**
diff --git a/services/openaiService.js b/services/openaiService.js
index de2664a9..c7c8b61f 100644
--- a/services/openaiService.js
+++ b/services/openaiService.js
@@ -5,7 +5,6 @@ const {
   writePromptToFile,
   extractChatMessageContent,
   assertCompletionNotTruncated,
-  toNameList,
 } = require('./serviceUtils');
 const {
   abortSignal,
@@ -26,7 +25,7 @@ const config = require('../config/config');
 const fs = require('fs').promises;
 const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
-const RestrictionPromptService = require('./restrictionPromptService');
+const { buildAnalysisPrompt } = require('./llm/promptBuilder');
 const responseLogPath = path.join(
   process.cwd(),
   'data',
@@ -88,15 +87,6 @@ class OpenAIService {
       // Cache the thumbnail for the UI; never let it affect the analysis.
       await cacheThumbnail(id);
 
-      // Format existing data - callers may hand over entity objects or plain names
-      const existingTagNames = toNameList(existingTags).join(', ');
-      const existingCorrespondentNames = toNameList(
-        existingCorrespondentList
-      ).join(', ');
-      const existingDocumentTypeNames = toNameList(
-        existingDocumentTypesList
-      ).join(', ');
-
       // Get external API data if available and validate it
       let externalApiData = options.externalApiData || null;
       let validatedExternalApiData = null;
@@ -115,101 +105,18 @@ class OpenAIService {
         }
       }
 
-      let systemPrompt = '';
-      let promptTags = '';
       const model = process.env.OPENAI_MODEL;
-
-      // Parse CUSTOM_FIELDS from environment variable
-      let customFieldsObj;
-      try {
-        customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS);
-      } catch (error) {
-        console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`);
-        console.debug(error);
-        customFieldsObj = { custom_fields: [] };
-      }
-
-      // Generate custom fields template for the prompt
-      const customFieldsTemplate = {};
-
-      customFieldsObj.custom_fields.forEach((field, index) => {
-        let valueHint;
-        if (field.data_type === 'date') {
-          valueHint =
-            'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis';
-        } else if (field.data_type === 'boolean') {
-          valueHint = "Fill in 'true' or 'false' based on your analysis";
-        } else {
-          valueHint = 'Fill in the value based on your analysis';
-        }
-        customFieldsTemplate[index] = {
-          field_name: field.value,
-          value: valueHint,
-        };
-      });
-
-      // Convert template to string for replacement and wrap in custom_fields
-      const customFieldsStr =
-        '"custom_fields": ' +
-        JSON.stringify(customFieldsTemplate, null, 2)
-          .split('\n')
-          .map((line) => '    ' + line) // Add proper indentation
-          .join('\n');
-
-      // Get system prompt and model
-      if (
-        config.useExistingData === 'yes' &&
-        config.restrictToExistingTags === 'no' &&
-        config.restrictToExistingCorrespondents === 'no'
-      ) {
-        systemPrompt =
-          `
-        Pre-existing tags: ${existingTagNames}\n\n
-        Pre-existing correspondents: ${existingCorrespondentNames}\n\n
-        Pre-existing document types: ${existingDocumentTypeNames}\n\n
-        ` +
-          process.env.SYSTEM_PROMPT +
-          '\n\n' +
-          config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
-        promptTags = '';
-      } else {
-        const mustHavePrompt = config.mustHavePrompt.replace(
-          '%CUSTOMFIELDS%',
-          customFieldsStr
-        );
-        systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt;
-        promptTags = '';
-      }
-
-      // Process placeholder replacements in system prompt
-      systemPrompt = RestrictionPromptService.processRestrictionsInPrompt(
-        systemPrompt,
+      const { systemPrompt, promptTags } = buildAnalysisPrompt({
         existingTags,
         existingCorrespondentList,
-        existingDocumentTypesList
-      );
-
-      // Include validated external API data if available
-      if (validatedExternalApiData) {
-        systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`;
-      }
-
-      if (process.env.USE_PROMPT_TAGS === 'yes') {
-        promptTags = process.env.PROMPT_TAGS;
-        systemPrompt =
-          `
-        Take these tags and try to match one or more to the document content.\n\n
-        ` + config.specialPromptPreDefinedTags;
-      }
-
+        existingDocumentTypesList,
+        externalApiData: validatedExternalApiData,
+        customPrompt,
+      });
       if (customPrompt) {
         console.log(
           '[DEBUG] Replace system prompt with custom prompt via WebHook'
         );
-        systemPrompt =
-          customPrompt +
-          '\n\n' +
-          config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr);
       }
 
       // Calculate tokens AFTER all prompt modifications are complete
diff --git a/tests/test-llm-prompt-builder.js b/tests/test-llm-prompt-builder.js
new file mode 100644
index 00000000..15547df2
--- /dev/null
+++ b/tests/test-llm-prompt-builder.js
@@ -0,0 +1,468 @@
+/**
+ * The analysis prompt, built once for all four providers.
+ *
+ * Step 2 of the LLM service refactoring (services/ManualServiceRefactoring.md).
+ * The %CUSTOMFIELDS% block and the prompt cascade after it — pre-existing
+ * data, mustHavePrompt, restriction placeholders, external API data,
+ * USE_PROMPT_TAGS and the webhook's custom prompt — used to be copied into
+ * every provider service, six times in all. They now live in
+ * services/llm/customFieldsTemplate.js and services/llm/promptBuilder.js.
+ *
+ * The first half pins what the builder produces. The second half checks that
+ * every provider really sends what the builder produced, so a provider that
+ * keeps a private copy, or drops part of the builder's output, fails here.
+ *
+ * One drift is resolved on the way and pinned as such: Ollama called its async
+ * _validateAndTruncateExternalApiData() without awaiting it and sent
+ * "[object Promise]" instead of the external API data.
+ */
+
+const assert = require('assert');
+const fs = require('fs').promises;
+
+process.env.AI_PROVIDER = 'openai';
+process.env.OPENAI_API_KEY = 'test-key';
+process.env.OPENAI_MODEL = 'gpt-4';
+process.env.AZURE_API_KEY = 'test-key';
+process.env.AZURE_ENDPOINT = 'https://example.invalid';
+process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment';
+process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1';
+process.env.CUSTOM_API_KEY = 'test-key';
+process.env.CUSTOM_MODEL = 'test-model';
+
+const config = require('../config/config');
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('../services/thumbnailCachePaths');
+const {
+  buildCustomFieldsBlock,
+} = require('../services/llm/customFieldsTemplate');
+const { buildAnalysisPrompt } = require('../services/llm/promptBuilder');
+const openaiService = require('../services/openaiService');
+const azureService = require('../services/azureService');
+const customService = require('../services/customService');
+const ollamaService = require('../services/ollamaService');
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+/** Own id so the seeded thumbnail cannot collide with a real cached document. */
+const DOCUMENT_ID = 'testLlmPromptBuilder';
+
+const SYSTEM_PROMPT =
+  'Analyse the document. Tags: %RESTRICTED_TAGS% | Correspondents: %RESTRICTED_CORRESPONDENTS% | Types: %RESTRICTED_DOCUMENT_TYPES%';
+
+const CUSTOM_FIELDS = JSON.stringify({
+  custom_fields: [
+    { value: 'Due date', data_type: 'date' },
+    { value: 'Paid', data_type: 'boolean' },
+    { value: 'Amount', data_type: 'monetary' },
+  ],
+});
+
+// Both shapes reach the services (issue #262): entity objects and plain names.
+const TAGS = [
+  { id: 1, name: 'invoice' },
+  { id: 2, name: 'contract' },
+];
+const CORRESPONDENTS = [{ id: 7, name: 'Acme Corp' }];
+const DOCUMENT_TYPES = ['Offer', 'Notice'];
+
+const ANSWER = JSON.stringify({
+  title: 'Telekom invoice July',
+  correspondent: 'Telekom Deutschland GmbH',
+  tags: ['invoice'],
+  document_type: 'Rechnung',
+  document_date: '2026-09-03',
+  language: 'de',
+});
+
+const SETTINGS = {
+  env: ['SYSTEM_PROMPT', 'CUSTOM_FIELDS', 'USE_PROMPT_TAGS', 'PROMPT_TAGS'],
+  config: [
+    'useExistingData',
+    'restrictToExistingTags',
+    'restrictToExistingCorrespondents',
+  ],
+};
+
+/** Puts every setting the prompt reads into a known state for one case. */
+function arrange({ env = {}, settings = {} } = {}) {
+  const values = {
+    SYSTEM_PROMPT,
+    CUSTOM_FIELDS,
+    USE_PROMPT_TAGS: 'no',
+    PROMPT_TAGS: '',
+    ...env,
+  };
+  for (const key of SETTINGS.env) {
+    if (values[key] === undefined) delete process.env[key];
+    else process.env[key] = values[key];
+  }
+  Object.assign(
+    config,
+    {
+      useExistingData: 'no',
+      restrictToExistingTags: 'no',
+      restrictToExistingCorrespondents: 'no',
+    },
+    settings
+  );
+}
+
+function build(overrides = {}) {
+  return buildAnalysisPrompt({
+    existingTags: TAGS,
+    existingCorrespondentList: CORRESPONDENTS,
+    existingDocumentTypesList: DOCUMENT_TYPES,
+    ...overrides,
+  });
+}
+
+/* --- the providers, each with a client that records what it was sent ----- */
+
+function recordingChatClient(service, requests) {
+  service.client = {
+    chat: {
+      completions: {
+        create: async (request) => {
+          requests.push(request);
+          return {
+            choices: [{ message: { content: ANSWER }, finish_reason: 'stop' }],
+            usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 },
+          };
+        },
+      },
+    },
+    models: { list: async () => ({ data: [] }) },
+  };
+}
+
+function recordingOllamaClient(requests) {
+  ollamaService.client = {
+    post: async (url, body) => {
+      requests.push(body);
+      return {
+        data: {
+          response: ANSWER,
+          done_reason: 'stop',
+          eval_count: 3,
+          prompt_eval_count: 9,
+        },
+      };
+    },
+    get: async () => ({ status: 200, data: { models: [] } }),
+  };
+}
+
+const CHAT_PROVIDERS = [
+  ['OpenAI', openaiService],
+  ['Azure', azureService],
+  ['Custom', customService],
+];
+
+/** The system message a chat provider sends for one analysis. */
+async function chatSystemPrompt(service, customPrompt = null, options = {}) {
+  const requests = [];
+  recordingChatClient(service, requests);
+  const analysis = await service.analyzeDocument(
+    'Document body text.',
+    TAGS,
+    CORRESPONDENTS,
+    DOCUMENT_TYPES,
+    DOCUMENT_ID,
+    customPrompt,
+    options
+  );
+  assert.strictEqual(analysis.error, undefined, 'the analysis succeeded');
+  assert.strictEqual(requests.length, 1, 'one request was sent');
+  const system = requests[0].messages.find((m) => m.role === 'system');
+  assert.ok(system, 'the request carries a system message');
+  return system.content;
+}
+
+/** The prompt Ollama sends for one analysis. */
+async function ollamaPrompt(customPrompt = null, options = {}) {
+  const requests = [];
+  recordingOllamaClient(requests);
+  const analysis = await ollamaService.analyzeDocument(
+    'Document body text.',
+    TAGS,
+    CORRESPONDENTS,
+    DOCUMENT_TYPES,
+    DOCUMENT_ID,
+    customPrompt,
+    options
+  );
+  assert.strictEqual(analysis.error, undefined, 'the analysis succeeded');
+  assert.strictEqual(requests.length, 1, 'one request was sent');
+  return requests[0].prompt;
+}
+
+(async () => {
+  console.log('\n=== LLM prompt builder ===');
+
+  /* --- the %CUSTOMFIELDS% block ------------------------------------------ */
+
+  await check('the custom fields block gives each data type its hint', () => {
+    arrange();
+    const block = buildCustomFieldsBlock();
+    assert.ok(block.startsWith('"custom_fields": '), block);
+    assert.match(block, /"field_name": "Due date"/);
+    assert.match(
+      block,
+      /"value": "Fill in the date in ISO 8601 format \(YYYY-MM-DD\) based on your analysis"/
+    );
+    assert.match(block, /"field_name": "Paid"/);
+    assert.match(
+      block,
+      /"value": "Fill in 'true' or 'false' based on your analysis"/
+    );
+    assert.match(block, /"field_name": "Amount"/);
+    assert.match(block, /"value": "Fill in the value based on your analysis"/);
+  });
+
+  await check(
+    'the custom fields block is indented to sit inside the JSON example',
+    () => {
+      arrange();
+      const lines = buildCustomFieldsBlock().split('\n');
+      assert.strictEqual(lines[0], '"custom_fields":     {');
+      assert.strictEqual(lines[1], '      "0": {');
+      assert.strictEqual(lines[lines.length - 1], '    }');
+    }
+  );
+
+  await check(
+    'unreadable or missing CUSTOM_FIELDS yield an empty block',
+    () => {
+      for (const value of ['{not json', undefined]) {
+        arrange({ env: { CUSTOM_FIELDS: value } });
+        assert.strictEqual(buildCustomFieldsBlock(), '"custom_fields":     {}');
+      }
+    }
+  );
+
+  /* --- the cascade ------------------------------------------------------- */
+
+  await check(
+    'by default: SYSTEM_PROMPT, then mustHavePrompt with the block',
+    () => {
+      arrange();
+      const { systemPrompt, promptTags } = build();
+      assert.ok(systemPrompt.startsWith('Analyse the document.'), systemPrompt);
+      assert.ok(
+        systemPrompt.includes(
+          config.mustHavePrompt
+            .replace('%CUSTOMFIELDS%', buildCustomFieldsBlock())
+            .slice(0, 60)
+        ),
+        'mustHavePrompt follows the system prompt'
+      );
+      assert.ok(!systemPrompt.includes('%CUSTOMFIELDS%'));
+      assert.ok(!systemPrompt.includes('Pre-existing tags'));
+      assert.strictEqual(promptTags, '');
+    }
+  );
+
+  await check('USE_EXISTING_DATA lists what Paperless-ngx already has', () => {
+    arrange({ settings: { useExistingData: 'yes' } });
+    const { systemPrompt } = build();
+    assert.ok(systemPrompt.includes('Pre-existing tags: invoice, contract'));
+    assert.ok(systemPrompt.includes('Pre-existing correspondents: Acme Corp'));
+    assert.ok(
+      systemPrompt.includes('Pre-existing document types: Offer, Notice')
+    );
+    assert.ok(!systemPrompt.includes('[object Object]'));
+    assert.ok(
+      systemPrompt.indexOf('Pre-existing tags') <
+        systemPrompt.indexOf('Analyse the document.'),
+      'the existing data comes before the system prompt'
+    );
+  });
+
+  await check('a restriction switches the existing-data block off', () => {
+    for (const restriction of [
+      'restrictToExistingTags',
+      'restrictToExistingCorrespondents',
+    ]) {
+      arrange({ settings: { useExistingData: 'yes', [restriction]: 'yes' } });
+      assert.ok(
+        !build().systemPrompt.includes('Pre-existing tags'),
+        restriction
+      );
+    }
+  });
+
+  await check('restriction placeholders resolve to the existing names', () => {
+    arrange();
+    const { systemPrompt } = build();
+    assert.ok(systemPrompt.includes('Tags: invoice, contract |'), systemPrompt);
+    assert.ok(systemPrompt.includes('Correspondents: Acme Corp |'));
+    assert.ok(systemPrompt.includes('Types: Offer, Notice'));
+    assert.ok(!systemPrompt.includes('%RESTRICTED_'));
+  });
+
+  await check(
+    'external API data is appended after the placeholders are resolved',
+    () => {
+      arrange();
+      const { systemPrompt } = build({
+        externalApiData: 'IBAN DE00, see %RESTRICTED_TAGS%',
+      });
+      assert.ok(
+        systemPrompt.endsWith(
+          '\n\nAdditional context from external API:\nIBAN DE00, see %RESTRICTED_TAGS%'
+        ),
+        systemPrompt
+      );
+    }
+  );
+
+  await check(
+    'USE_PROMPT_TAGS replaces the prompt and hands out PROMPT_TAGS',
+    () => {
+      arrange({
+        env: { USE_PROMPT_TAGS: 'yes', PROMPT_TAGS: 'invoice, bank' },
+      });
+      const { systemPrompt, promptTags } = build({
+        externalApiData: 'ignored',
+      });
+      assert.ok(
+        systemPrompt.includes(
+          'Take these tags and try to match one or more to the document content.'
+        )
+      );
+      assert.ok(systemPrompt.endsWith(config.specialPromptPreDefinedTags));
+      assert.ok(!systemPrompt.includes('Analyse the document.'));
+      assert.ok(!systemPrompt.includes('ignored'));
+      assert.strictEqual(promptTags, 'invoice, bank');
+    }
+  );
+
+  await check(
+    'a custom prompt wins over everything, mustHavePrompt still follows',
+    () => {
+      arrange({
+        env: { USE_PROMPT_TAGS: 'yes', PROMPT_TAGS: 'invoice' },
+        settings: { useExistingData: 'yes' },
+      });
+      const { systemPrompt, promptTags } = build({
+        customPrompt: 'Webhook prompt.',
+        externalApiData: 'ignored',
+      });
+      assert.strictEqual(
+        systemPrompt,
+        'Webhook prompt.\n\n' +
+          config.mustHavePrompt.replace(
+            '%CUSTOMFIELDS%',
+            buildCustomFieldsBlock()
+          )
+      );
+      assert.strictEqual(
+        promptTags,
+        'invoice',
+        'the tag list is still reported'
+      );
+    }
+  );
+
+  /* --- every provider sends what the builder built ------------------------ */
+
+  await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+  await fs.writeFile(
+    getThumbnailCachePath(DOCUMENT_ID),
+    Buffer.from([0x89, 0x50, 0x4e, 0x47])
+  );
+
+  const PROVIDER_CASES = [
+    ['by default', {}, {}],
+    ['with existing data', { settings: { useExistingData: 'yes' } }, {}],
+    ['with a custom prompt', {}, { customPrompt: 'Webhook prompt.' }],
+    [
+      'with external API data',
+      {},
+      { externalApiData: { iban: 'DE00', note: 'x' } },
+    ],
+  ];
+
+  for (const [label, service] of CHAT_PROVIDERS) {
+    for (const [name, arrangement, input] of PROVIDER_CASES) {
+      await check(
+        `${label}: the system message is the builder's ${name}`,
+        async () => {
+          arrange(arrangement);
+          const sent = await chatSystemPrompt(
+            service,
+            input.customPrompt ?? null,
+            input.externalApiData
+              ? { externalApiData: input.externalApiData }
+              : {}
+          );
+          const expected = build({
+            customPrompt: input.customPrompt ?? null,
+            externalApiData: input.externalApiData
+              ? JSON.stringify(input.externalApiData, null, 2)
+              : null,
+          }).systemPrompt;
+          assert.strictEqual(sent, expected);
+        }
+      );
+    }
+  }
+
+  for (const [name, arrangement, input] of PROVIDER_CASES) {
+    await check(
+      `Ollama: the prompt starts with the builder's ${name}`,
+      async () => {
+        arrange(arrangement);
+        const sent = await ollamaPrompt(
+          input.customPrompt ?? null,
+          input.externalApiData
+            ? { externalApiData: input.externalApiData }
+            : {}
+        );
+        const expected = build({
+          customPrompt: input.customPrompt ?? null,
+          externalApiData: input.externalApiData
+            ? JSON.stringify(input.externalApiData, null, 2)
+            : null,
+        }).systemPrompt;
+        assert.ok(sent.startsWith(expected), `sent:\n${sent}`);
+        assert.ok(
+          sent.includes('"Document body text."'),
+          'the content follows'
+        );
+      }
+    );
+  }
+
+  await check(
+    'Ollama: external API data is the data, not "[object Promise]"',
+    async () => {
+      arrange();
+      const sent = await ollamaPrompt(null, {
+        externalApiData: { iban: 'DE00' },
+      });
+      assert.ok(!sent.includes('[object Promise]'), sent);
+      assert.ok(sent.includes('"iban": "DE00"'), sent);
+    }
+  );
+
+  await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
+
+  if (failed > 0) {
+    console.error(`\n${failed} prompt builder case(s) failed`);
+    process.exit(1);
+  }
+  console.log('\nAll prompt builder cases passed');
+})();

From 49a04fc1b9153da31b91a028c6e052b68b07de4c Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 11 Oct 2026 19:11:59 +0200
Subject: [PATCH 35/53] Parse the analysis answer once for all four providers

Background
Step 3 of the LLM service refactoring (services/ManualServiceRefactoring.md),
Finding 7d: there were three JSON parsers. Custom matched braces and
stripped <think> blocks; openai and azure stripped fences and parsed the
rest as is; Ollama took everything from the first "{" to the last "}" and
repaired trailing commas and unquoted keys. The same answer could succeed
on one provider and fail on the next.

Changes
- services/llm/responseParser.js: parseAnalysisResponse() returns
  { parsed, normalized }. An object passes through (Ollama's structured
  output); <think> blocks and fences (any case) are stripped; the text is
  parsed as it stands; then each top-level {...} is tried as written and
  repaired. An unreadable object is skipped whole, so a nested
  custom_fields entry is never taken for the answer. Every failure starts
  with "Invalid JSON response from API", the OCR fallback phrase.
  stripResponseWrapping() is exported for OpenAI's "insufficient content"
  check, which still runs only after a failed parse.
- openai/azure: the inline strip-and-parse is replaced by the parser.
- custom: _extractFirstJsonValue() and _parseJsonResponse() removed.
- ollama: _parseResponse() and _sanitizeJsonString() removed;
  _processOllamaResponse() applies its field defaults to both the object
  and the text path.
- tests/test-llm-response-parser.js (57 cases, area processing): seven
  readable and three unreadable answers through the parser and through
  all four providers.
- Plan: step 3 and row 7d marked done.

Testing
Against the services before this change 12 provider cases failed:
OpenAI and Azure could not read an upper-case fence, prose around the
object, a trailing comma, or prose plus braces in a string; Custom could
not read a trailing comma; Ollama could not read a <think> block or prose
after the object, and its repair branch dropped document_type and
custom_fields. All 57 pass now.
node scripts/run-tests.js --all: 93 passed, 7 skipped, 25 failed; the same
25 fail on the parent commit (92 passed) - better-sqlite3 built for a
different Node ABI, Windows temp-dir EPERM, changelog/version mismatch.
llm-*, ollama-*, response-truncation-detection and ocr-fallback-ai-errors
are green. ESLint and Prettier clean on all changed files.

Impact
Answers that one provider used to reject are now read by all four; an
answer Ollama repaired keeps its document type and custom fields. Failure
messages keep the OCR fallback phrase; their suffix may differ. No API or
OpenAPI change.

Upstream Status
Local only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 scripts/run-tests.js                 |   2 +
 services/ManualServiceRefactoring.md |  29 ++-
 services/azureService.js             |  14 +-
 services/customService.js            |  96 +---------
 services/llm/responseParser.js       | 166 ++++++++++++++++
 services/ollamaService.js            | 128 +++----------
 services/openaiService.js            |  18 +-
 tests/test-llm-response-parser.js    | 277 +++++++++++++++++++++++++++
 8 files changed, 510 insertions(+), 220 deletions(-)
 create mode 100644 services/llm/responseParser.js
 create mode 100644 tests/test-llm-response-parser.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index eac689c6..64fa40ff 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -100,6 +100,7 @@ const TESTS = {
   'llm-analysis-result-contract': 'test-llm-analysis-result-contract.js',
   'llm-ocr-fallback-phrases': 'test-llm-ocr-fallback-phrases.js',
   'llm-prompt-builder': 'test-llm-prompt-builder.js',
+  'llm-response-parser': 'test-llm-response-parser.js',
   'custom-field-longtext': 'test-custom-field-longtext.js',
   'document-processing-service': 'test-document-processing-service.js',
   'settings-custom-field-type-conflict':
@@ -237,6 +238,7 @@ const AREAS = {
     'settings-paperless-url-fallback',
     'llm-analysis-result-contract',
     'llm-ocr-fallback-phrases',
+    'llm-response-parser',
     'rescan-fetch-before-delete',
   ],
   prompts: [
diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 769775d1..aca6e518 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -184,7 +184,7 @@ today. These are the actual bugs the work will surface:
 | a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`                                   | `azureService.js:286`                                                                                                                                                            | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                                                                                                             |
 | b   | ~~`return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined`~~ **resolved**          | was `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                          | Fixed ahead of the refactor together with row j — same statement block, same root cause. See the note below the table.                                                                                                                                      |
 | c   | "Insufficient content" early return exists only in OpenAI                                                                   | `openaiService.js:310-332`                                                                                                                                                       | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                                                                                                                    |
-| d   | Three different JSON parsers                                                                                                | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | One parser built from custom's `_extractFirstJsonValue` + `<think>` stripping + Ollama's sanitize fallback; keep Ollama's object passthrough as a pre-step.                                                                                                 |
+| d   | ~~Three different JSON parsers~~ **resolved**                                                                               | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | Done in step 3: `services/llm/responseParser.js` — custom's brace matcher, `<think>` stripping, Ollama's repair and object passthrough.                                                                                                                     |
 | e   | Only custom normalizes timeouts                                                                                             | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                                                                                                                        |
 | f   | `generateText` option handling — **mostly resolved upstream**                                                               | `services/aiGenerateOptions.js`, used by all four `generateText()` implementations                                                                                               | Upstream unified how options are read (completion cap, temperature, system prompt, model, abort, reasoning, progress). Keep `aiGenerateOptions.js` as the shared reader; the transports only map the result onto their own knobs.                           |
 | g   | `checkStatus` differs by nature                                                                                             | openai/custom `models.list()`; Azure REST GET; `ollamaService.js:1210` `/api/ps`                                                                                                 | Stays in the transport. Not business logic.                                                                                                                                                                                                                 |
@@ -485,9 +485,30 @@ carry no behavioral risk; the risk starts at step 4.
    Ollama still ignores `PROMPT_TAGS` as user content; the chat providers only use it for the
    token count.
 
-3. **Extract `responseParser.js`** (custom's brace matcher + `<think>` stripping + Ollama's sanitize
-   fallback + Ollama's object passthrough) with `llm-response-parser`, and route all four services
-   through it. Resolves Finding 7d.
+3. ~~**Extract `responseParser.js`**~~ **Done.** `services/llm/responseParser.js`
+   (`parseAnalysisResponse()` → `{ parsed, normalized }`, plus `stripResponseWrapping()`) replaces
+   custom's `_extractFirstJsonValue`/`_parseJsonResponse`, the inline strip-and-parse of openai and
+   azure, and Ollama's `_parseResponse`/`_sanitizeJsonString`. Order: an object passes through,
+   `<think>` blocks and fences (any case) are stripped, the text is parsed as it stands, then each
+   **top-level** `{...}` is tried as written and repaired. An unreadable object is skipped whole, so
+   a nested `custom_fields` entry is never mistaken for the answer — on Ollama that would have been
+   a silent success (row k). Every failure starts with "Invalid JSON response from API".
+   `llm-response-parser` (57 cases) sends seven readable and three unreadable answers through the
+   parser and through all four providers.
+
+   Against the services before this step 12 provider cases failed, which is Finding 7d measured:
+   OpenAI and Azure could not read an upper-case fence, prose around the object, a trailing comma
+   or prose plus braces in a string; Custom could not read a trailing comma; Ollama could not read
+   a `<think>` block or prose after the object, and its repair branch **dropped `document_type`
+   and `custom_fields`** from an answer it did repair. All of those now read the same everywhere.
+
+   Deliberately not moved: the field defaults Ollama applies (`tags` → `[]`, the rest → `null`)
+   stay in `_processOllamaResponse()`, now for both the object and the text path. They are result
+   shaping and belong to `analysisResult.js` in step 4, so the parser returns the object as read
+   rather than the "document object" the target architecture lists. OpenAI's "insufficient
+   content" early return still runs only on a failed parse, on the stripped text. Row l ("No
+   response data from Ollama API") is untouched and still pinned as drift.
+
 4. **Extract `analysisResult.js`** and the two `tokenBudget` strategies with `llm-token-budget`.
    Findings 7b/7j are already fixed; `services/thumbnailCache.js` is the collaborator the
    composition roots inject in step 6, and `thumbnail-optional-analysis` must stay green throughout.
diff --git a/services/azureService.js b/services/azureService.js
index f501db10..094642ec 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -27,6 +27,7 @@ const fs = require('fs').promises;
 const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
+const { parseAnalysisResponse } = require('./llm/responseParser');
 const responseLogPath = path.join(
   process.cwd(),
   'data',
@@ -216,20 +217,15 @@ class AzureOpenAIService {
         totalTokens: usage.total_tokens,
       };
 
-      // Strip <think>...</think> reasoning tags from models like Qwen3, DeepSeek-R1
-      jsonContent = jsonContent.replace(/<think>[\s\S]*?<\/think>/g, '');
-      jsonContent = jsonContent
-        .replace(/```json\n?/g, '')
-        .replace(/```\n?/g, '')
-        .trim();
-
       let parsedResponse;
       try {
-        parsedResponse = JSON.parse(jsonContent);
+        const parsedResult = parseAnalysisResponse(jsonContent);
+        parsedResponse = parsedResult.parsed;
+        jsonContent = parsedResult.normalized;
       } catch (error) {
         console.error(`Failed to parse JSON response: ${error.message}`);
         console.debug(error);
-        throw new Error('Invalid JSON response from API', { cause: error });
+        throw error;
       }
 
       try {
diff --git a/services/customService.js b/services/customService.js
index b11444c4..62c7d366 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -27,6 +27,7 @@ const fs = require('fs').promises;
 const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
+const { parseAnalysisResponse } = require('./llm/responseParser');
 const responseLogPath = path.join(
   process.cwd(),
   'data',
@@ -54,99 +55,6 @@ class CustomOpenAIService {
     this.tokenizer = null;
   }
 
-  _extractFirstJsonValue(text) {
-    if (!text) {
-      return null;
-    }
-
-    const starts = [];
-    const firstObject = text.indexOf('{');
-    const firstArray = text.indexOf('[');
-
-    if (firstObject !== -1) {
-      starts.push(firstObject);
-    }
-    if (firstArray !== -1) {
-      starts.push(firstArray);
-    }
-
-    if (starts.length === 0) {
-      return null;
-    }
-
-    const startIndex = Math.min(...starts);
-    const opening = text[startIndex];
-    const closing = opening === '{' ? '}' : ']';
-
-    let depth = 0;
-    let inString = false;
-    let escaping = false;
-
-    for (let i = startIndex; i < text.length; i += 1) {
-      const char = text[i];
-
-      if (inString) {
-        if (escaping) {
-          escaping = false;
-        } else if (char === '\\') {
-          escaping = true;
-        } else if (char === '"') {
-          inString = false;
-        }
-        continue;
-      }
-
-      if (char === '"') {
-        inString = true;
-        continue;
-      }
-
-      if (char === opening) {
-        depth += 1;
-      } else if (char === closing) {
-        depth -= 1;
-        if (depth === 0) {
-          return text.slice(startIndex, i + 1);
-        }
-      }
-    }
-
-    return null;
-  }
-
-  _parseJsonResponse(rawContent) {
-    const sanitizedContent = String(rawContent || '')
-      .replace(/<think>[\s\S]*?<\/think>/g, '')
-      .replace(/```json\n?/gi, '')
-      .replace(/```\n?/g, '')
-      .trim();
-
-    try {
-      return {
-        parsed: JSON.parse(sanitizedContent),
-        normalized: sanitizedContent,
-      };
-    } catch (directParseError) {
-      const extractedJson = this._extractFirstJsonValue(sanitizedContent);
-      if (!extractedJson) {
-        throw new Error('Invalid JSON response from API', {
-          cause: directParseError,
-        });
-      }
-
-      try {
-        return {
-          parsed: JSON.parse(extractedJson),
-          normalized: extractedJson,
-        };
-      } catch (extractedParseError) {
-        throw new Error('Invalid JSON response from API', {
-          cause: extractedParseError,
-        });
-      }
-    }
-  }
-
   initialize() {
     if (!this.client && config.aiProvider === 'custom') {
       this.client = new OpenAI({
@@ -318,7 +226,7 @@ class CustomOpenAIService {
 
       let parsedResponse;
       try {
-        const parsedResult = this._parseJsonResponse(jsonContent);
+        const parsedResult = parseAnalysisResponse(jsonContent);
         parsedResponse = parsedResult.parsed;
         jsonContent = parsedResult.normalized;
       } catch (error) {
diff --git a/services/llm/responseParser.js b/services/llm/responseParser.js
new file mode 100644
index 00000000..0c29e7ac
--- /dev/null
+++ b/services/llm/responseParser.js
@@ -0,0 +1,166 @@
+'use strict';
+
+/**
+ * Reads the JSON object out of a document analysis answer, once for every
+ * provider.
+ *
+ * Models wrap their answer: reasoning models put a <think> block in front,
+ * many fence the JSON in ```json, some add a sentence before or after it, and
+ * smaller local models leave trailing commas or unquoted keys behind. Each
+ * provider used to cope with a different subset of that. This is the union:
+ *
+ *  1. an answer that already is an object (Ollama's structured output) is
+ *     used as it is;
+ *  2. reasoning blocks and code fences are stripped;
+ *  3. the rest is parsed as it stands;
+ *  4. failing that, every top-level {...} in it is tried in turn, as written
+ *     and then repaired, and the first one that parses wins.
+ *
+ * Every failure is an Error whose message starts with "Invalid JSON response
+ * from API" — shouldQueueForOcrOnAiError() matches on that phrase, and
+ * without it the document no longer reaches the OCR fallback.
+ */
+
+const INVALID_JSON = 'Invalid JSON response from API';
+
+/** Drop reasoning blocks and code fences around the answer. */
+function stripResponseWrapping(text) {
+  return String(text ?? '')
+    .replace(/<think>[\s\S]*?<\/think>/g, '')
+    .replace(/```json\n?/gi, '')
+    .replace(/```\n?/g, '')
+    .trim();
+}
+
+/**
+ * The end index of the balanced {...} that opens at `start`, or -1 when the
+ * text ends first. Braces inside string literals do not count.
+ */
+function findObjectEnd(text, start) {
+  let depth = 0;
+  let inString = false;
+  let escaping = false;
+
+  for (let i = start; i < text.length; i += 1) {
+    const char = text[i];
+
+    if (inString) {
+      if (escaping) {
+        escaping = false;
+      } else if (char === '\\') {
+        escaping = true;
+      } else if (char === '"') {
+        inString = false;
+      }
+      continue;
+    }
+
+    if (char === '"') {
+      inString = true;
+    } else if (char === '{') {
+      depth += 1;
+    } else if (char === '}') {
+      depth -= 1;
+      if (depth === 0) {
+        return i;
+      }
+    }
+  }
+
+  return -1;
+}
+
+/**
+ * Every top-level {...} in the text, left to right. An object that does not
+ * parse is skipped as a whole: what is nested inside it is part of a broken
+ * answer, not an answer of its own.
+ */
+function* topLevelObjects(text) {
+  let start = text.indexOf('{');
+  while (start !== -1) {
+    const end = findObjectEnd(text, start);
+    if (end === -1) {
+      return;
+    }
+    yield text.slice(start, end + 1);
+    start = text.indexOf('{', end + 1);
+  }
+}
+
+/** The repairs Ollama has always applied to an answer that would not parse. */
+function repairJson(jsonStr) {
+  return jsonStr
+    .replace(/,\s*}/g, '}') // Remove trailing commas before closing braces
+    .replace(/,\s*]/g, ']') // Remove trailing commas before closing brackets
+    .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":'); // Ensure property names are quoted
+}
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+/** The parsed object, or undefined when the text is not one. */
+function tryParseObject(text) {
+  try {
+    const parsed = JSON.parse(text);
+    return isPlainObject(parsed) ? parsed : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Parse a document analysis answer.
+ *
+ * @param {string|object} raw - The model's answer
+ * @returns {{ parsed: object, normalized: string }} The answer object, and
+ *   the JSON it was read from (what goes to the response log)
+ * @throws {Error} "Invalid JSON response from API: …" when no object can be
+ *   read; the first parse error is attached as `cause`
+ */
+function parseAnalysisResponse(raw) {
+  if (isPlainObject(raw)) {
+    return { parsed: raw, normalized: JSON.stringify(raw) };
+  }
+
+  const text = stripResponseWrapping(raw);
+  if (!text) {
+    throw new Error(`${INVALID_JSON}: the answer is empty`);
+  }
+
+  let firstError;
+  try {
+    const parsed = JSON.parse(text);
+    if (isPlainObject(parsed)) {
+      return { parsed, normalized: text };
+    }
+  } catch (error) {
+    firstError = error;
+  }
+
+  let sawObject = false;
+  for (const candidate of topLevelObjects(text)) {
+    sawObject = true;
+    for (const attempt of [candidate, repairJson(candidate)]) {
+      const parsed = tryParseObject(attempt);
+      if (parsed) {
+        return { parsed, normalized: attempt };
+      }
+    }
+  }
+
+  if (!sawObject) {
+    throw new Error(`${INVALID_JSON}: no JSON object in the answer`, {
+      cause: firstError,
+    });
+  }
+  throw new Error(
+    `${INVALID_JSON}: unreadable even after repair${firstError ? `: ${firstError.message}` : ''}`,
+    { cause: firstError }
+  );
+}
+
+module.exports = {
+  parseAnalysisResponse,
+  stripResponseWrapping,
+};
diff --git a/services/ollamaService.js b/services/ollamaService.js
index a891e73e..ae113499 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -15,6 +15,7 @@ const axios = require('axios');
 const config = require('../config/config');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
+const { parseAnalysisResponse } = require('./llm/responseParser');
 const { buildCustomFieldsBlock } = require('./llm/customFieldsTemplate');
 
 /**
@@ -638,28 +639,35 @@ class OllamaService {
 
   /**
    * Process Ollama API response
+   *
+   * The answer arrives as an object when Ollama applied the format schema,
+   * and as text otherwise; responseParser handles both. Every exit either
+   * returns something the caller can use or throws. Handing back
+   * { tags: [], correspondent: null } for an unreadable answer is
+   * indistinguishable from a model that genuinely found nothing, so the
+   * document would be written back untouched and marked processed.
+   *
    * @param {Object} responseData - Ollama API response data
    * @returns {Object} Parsed response
    */
   _processOllamaResponse(responseData) {
-    if (responseData.response && typeof responseData.response === 'object') {
-      console.log('Using response JS object directly');
+    if (!responseData.response) {
+      throw new Error('No response data from Ollama API');
+    }
+    try {
+      const { parsed } = parseAnalysisResponse(responseData.response);
       return {
-        tags: Array.isArray(responseData.response.tags)
-          ? responseData.response.tags
-          : [],
-        correspondent: responseData.response.correspondent || null,
-        title: responseData.response.title || null,
-        document_date: responseData.response.document_date || null,
-        document_type: responseData.response.document_type || null,
-        language: responseData.response.language || null,
-        custom_fields: responseData.response.custom_fields || null,
+        tags: Array.isArray(parsed.tags) ? parsed.tags : [],
+        correspondent: parsed.correspondent || null,
+        title: parsed.title || null,
+        document_date: parsed.document_date || null,
+        document_type: parsed.document_type || null,
+        language: parsed.language || null,
+        custom_fields: parsed.custom_fields || null,
       };
-    } else if (responseData.response) {
-      console.log('No JS object yet, parsing JSON');
-      return this._parseResponse(responseData.response);
-    } else {
-      throw new Error('No response data from Ollama API');
+    } catch (error) {
+      console.error('Error parsing Ollama response:', error.message);
+      throw error;
     }
   }
 
@@ -679,94 +687,6 @@ class OllamaService {
     };
   }
 
-  /**
-   * Parse text response to extract JSON
-   * @param {string} response - Response text
-   * @returns {Object} Parsed object
-   */
-  /* Every exit from here either returns something the caller can use or
-     throws. It used to hand back { tags: [], correspondent: null } on each of
-     its three failure paths, which is indistinguishable from a model that
-     genuinely found nothing — so a document whose answer could not be read at
-     all was written back untouched and then marked processed, never to be
-     looked at again. openaiService has thrown at the same point all along. */ _parseResponse(
-    response
-  ) {
-    try {
-      // Find JSON in response using regex
-      const jsonMatch = response.match(/\{[\s\S]*\}/);
-      if (!jsonMatch) {
-        /* Worded to carry the marker serviceUtils already classifies, so a
-           document whose answer was unreadable takes the same OCR fallback it
-           takes under OpenAI. Truncation deliberately does not — see
-           _assertNotTruncated, where a second pass cannot help. */
-        throw new Error(
-          'Invalid JSON response from API: no JSON object in the answer'
-        );
-      }
-
-      let jsonStr = jsonMatch[0];
-      console.log('Extracted JSON String:', jsonStr);
-
-      try {
-        // Attempt to parse the JSON
-        const result = JSON.parse(jsonStr);
-
-        // Validate and return the result
-        return {
-          tags: Array.isArray(result.tags) ? result.tags : [],
-          correspondent: result.correspondent || null,
-          title: result.title || null,
-          document_date: result.document_date || null,
-          document_type: result.document_type || null,
-          language: result.language || null,
-          custom_fields: result.custom_fields || null,
-        };
-      } catch (jsonError) {
-        console.warn('Error parsing JSON from response:', jsonError.message);
-        console.warn('Attempting to sanitize the JSON...');
-
-        // Sanitize the JSON
-        jsonStr = this._sanitizeJsonString(jsonStr);
-
-        try {
-          const sanitizedResult = JSON.parse(jsonStr);
-          return {
-            tags: Array.isArray(sanitizedResult.tags)
-              ? sanitizedResult.tags
-              : [],
-            correspondent: sanitizedResult.correspondent || null,
-            title: sanitizedResult.title || null,
-            document_date: sanitizedResult.document_date || null,
-            language: sanitizedResult.language || null,
-          };
-        } catch {
-          console.error(
-            'Final JSON parsing failed after sanitization. This happens when the JSON structure is too complex or invalid. That indicates an issue with the generated JSON string by Ollama. Switch to OpenAI for better results or fine tune your prompt.'
-          );
-          throw new Error(
-            `Invalid JSON response from API, unreadable even after sanitizing: ${jsonError.message}`
-          );
-        }
-      }
-    } catch (error) {
-      console.error('Error parsing Ollama response:', error.message);
-      throw error;
-    }
-  }
-
-  /**
-   * Sanitize a JSON string
-   * @param {string} jsonStr - JSON string to sanitize
-   * @returns {string} Sanitized JSON string
-   */
-  _sanitizeJsonString(jsonStr) {
-    return jsonStr
-      .replace(/,\s*}/g, '}') // Remove trailing commas before closing braces
-      .replace(/,\s*]/g, ']') // Remove trailing commas before closing brackets
-      .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":'); // Ensure property names are quoted
-  }
-
   /**
    * Logs the prompt and response to a file. Combines the system prompt, user
    * prompt, and response into a formatted string and writes it to a file.
diff --git a/services/openaiService.js b/services/openaiService.js
index c7c8b61f..3e0289fc 100644
--- a/services/openaiService.js
+++ b/services/openaiService.js
@@ -26,6 +26,10 @@ const fs = require('fs').promises;
 const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
+const {
+  parseAnalysisResponse,
+  stripResponseWrapping,
+} = require('./llm/responseParser');
 const responseLogPath = path.join(
   process.cwd(),
   'data',
@@ -200,20 +204,16 @@ class OpenAIService {
         totalTokens: usage.total_tokens,
       };
 
-      // Strip <think>...</think> reasoning tags from models like Qwen3, DeepSeek-R1
-      jsonContent = jsonContent.replace(/<think>[\s\S]*?<\/think>/g, '');
-      jsonContent = jsonContent
-        .replace(/```json\n?/g, '')
-        .replace(/```\n?/g, '')
-        .trim();
-
       let parsedResponse;
       try {
-        parsedResponse = JSON.parse(jsonContent);
+        const parsedResult = parseAnalysisResponse(jsonContent);
+        parsedResponse = parsedResult.parsed;
+        jsonContent = parsedResult.normalized;
       } catch (error) {
         console.error(`Failed to parse JSON response: ${error.message}`);
         console.debug(error);
         // Check if the response indicates the content is too minimal or API can't process it
+        jsonContent = stripResponseWrapping(jsonContent);
         if (
           jsonContent &&
           (jsonContent.toLowerCase().includes("i'm sorry") ||
@@ -236,7 +236,7 @@ class OpenAIService {
             error: 'Insufficient content for AI analysis',
           };
         }
-        throw new Error('Invalid JSON response from API', { cause: error });
+        throw error;
       }
 
       try {
diff --git a/tests/test-llm-response-parser.js b/tests/test-llm-response-parser.js
new file mode 100644
index 00000000..321d7850
--- /dev/null
+++ b/tests/test-llm-response-parser.js
@@ -0,0 +1,277 @@
+/**
+ * The analysis answer, parsed once for all four providers.
+ *
+ * Step 3 of the LLM service refactoring (services/ManualServiceRefactoring.md).
+ * Finding 7d: there were three parsers. Custom matched braces and stripped
+ * <think> blocks, openai and azure stripped fences and called JSON.parse on
+ * the rest, Ollama took everything from the first "{" to the last "}" and
+ * repaired trailing commas and unquoted keys. The same answer could therefore
+ * succeed on one provider and fail on the next. services/llm/responseParser.js
+ * is the union of the three.
+ *
+ * The first half pins what the parser accepts and how it fails. The second
+ * half sends the same answers through every provider, so a provider that
+ * keeps a private parser, or narrows the result again, fails here.
+ *
+ * Every failure must keep the "Invalid JSON response from API" phrase: the
+ * OCR fallback matches on it (Finding 4).
+ */
+
+const assert = require('assert');
+const fs = require('fs').promises;
+
+process.env.AI_PROVIDER = 'openai';
+process.env.OPENAI_API_KEY = 'test-key';
+process.env.OPENAI_MODEL = 'gpt-4';
+process.env.AZURE_API_KEY = 'test-key';
+process.env.AZURE_ENDPOINT = 'https://example.invalid';
+process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment';
+process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1';
+process.env.CUSTOM_API_KEY = 'test-key';
+process.env.CUSTOM_MODEL = 'test-model';
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+delete process.env.CUSTOM_FIELDS;
+
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('../services/thumbnailCachePaths');
+const {
+  parseAnalysisResponse,
+  stripResponseWrapping,
+} = require('../services/llm/responseParser');
+const { shouldQueueForOcrOnAiError } = require('../services/serviceUtils');
+const openaiService = require('../services/openaiService');
+const azureService = require('../services/azureService');
+const customService = require('../services/customService');
+const ollamaService = require('../services/ollamaService');
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+/** Own id so the seeded thumbnail cannot collide with a real cached document. */
+const DOCUMENT_ID = 'testLlmResponseParser';
+
+const RESULT = {
+  title: 'Telekom invoice July',
+  correspondent: 'Telekom Deutschland GmbH',
+  tags: ['invoice'],
+  document_type: 'Rechnung',
+  document_date: '2026-09-03',
+  language: 'de',
+  custom_fields: { 0: { field_name: 'Amount', value: '12.50' } },
+};
+const JSON_ANSWER = JSON.stringify(RESULT, null, 2);
+
+/** Answers every provider has to read, and what each one is a case of. */
+const READABLE_ANSWERS = [
+  ['plain JSON', JSON_ANSWER],
+  ['a fenced block', '```json\n' + JSON_ANSWER + '\n```'],
+  ['an upper-case fence', '```JSON\n' + JSON_ANSWER + '\n```'],
+  [
+    'a <think> block in front',
+    '<think>The sender is "Telekom" {maybe}.</think>\n' + JSON_ANSWER,
+  ],
+  [
+    'prose around the object',
+    'Here is the analysis:\n' + JSON_ANSWER + '\nLet me know {if} it helps.',
+  ],
+  ['a trailing comma', JSON_ANSWER.replace(/\n}$/, ',\n}')],
+  [
+    'braces inside a string value',
+    'Result: ' +
+      JSON_ANSWER.replace('Telekom invoice July', 'Invoice {July} }') +
+      ' (done)',
+  ],
+];
+
+/** The object a READABLE_ANSWERS entry stands for. */
+function expectedFor(name) {
+  return name === 'braces inside a string value'
+    ? { ...RESULT, title: 'Invoice {July} }' }
+    : RESULT;
+}
+
+/** Answers no provider can read; none of them may count as a success. */
+const UNREADABLE_ANSWERS = [
+  ['no JSON at all', 'The document is a scanned letter.'],
+  ['an object cut off mid-way', '{"title": "Telekom", "tags": ["inv'],
+  ['an object beyond repair', '{"title": "Telekom" "tags" ["invoice"]}'],
+];
+
+/* --- the providers, each with a client that answers a fixed text ---------- */
+
+function chatClient(service, content) {
+  service.client = {
+    chat: {
+      completions: {
+        create: async () => ({
+          choices: [{ message: { content }, finish_reason: 'stop' }],
+          usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 },
+        }),
+      },
+    },
+    models: { list: async () => ({ data: [] }) },
+  };
+}
+
+function ollamaClient(response) {
+  ollamaService.client = {
+    post: async () => ({
+      data: {
+        response,
+        done_reason: 'stop',
+        eval_count: 3,
+        prompt_eval_count: 9,
+      },
+    }),
+    get: async () => ({ status: 200, data: { models: [] } }),
+  };
+}
+
+const PROVIDERS = [
+  ['OpenAI', openaiService, (answer) => chatClient(openaiService, answer)],
+  ['Azure', azureService, (answer) => chatClient(azureService, answer)],
+  ['Custom', customService, (answer) => chatClient(customService, answer)],
+  ['Ollama', ollamaService, ollamaClient],
+];
+
+async function analyze(service, answerWith, answer) {
+  answerWith(answer);
+  return service.analyzeDocument(
+    'Document body text.',
+    [],
+    [],
+    [],
+    DOCUMENT_ID
+  );
+}
+
+/** Ollama fills the fields a model left out; the chat providers pass them on. */
+function sameDocument(label, actual, expected) {
+  if (label !== 'Ollama') {
+    assert.deepStrictEqual(actual, expected);
+    return;
+  }
+  for (const [key, value] of Object.entries(expected)) {
+    assert.deepStrictEqual(actual[key], value, key);
+  }
+}
+
+(async () => {
+  console.log('\n=== LLM response parser ===');
+
+  /* --- the parser on its own --------------------------------------------- */
+
+  for (const [name, answer] of READABLE_ANSWERS) {
+    await check(`reads ${name}`, () => {
+      const { parsed } = parseAnalysisResponse(answer);
+      assert.deepStrictEqual(parsed, expectedFor(name));
+    });
+  }
+
+  await check('normalized is the JSON that was parsed', () => {
+    const { normalized } = parseAnalysisResponse(
+      '<think>x</think>Sure:\n' + JSON_ANSWER + '\nDone.'
+    );
+    assert.deepStrictEqual(JSON.parse(normalized), RESULT);
+    assert.ok(normalized.startsWith('{'), normalized);
+    assert.ok(normalized.endsWith('}'), normalized);
+  });
+
+  await check('an answer that is already an object passes through', () => {
+    const { parsed, normalized } = parseAnalysisResponse(RESULT);
+    assert.strictEqual(parsed, RESULT);
+    assert.deepStrictEqual(JSON.parse(normalized), RESULT);
+  });
+
+  await check('the first top-level object wins, not one nested in it', () => {
+    // The outer object is beyond repair; its custom_fields entry would parse
+    // on its own, but it is not the answer and must not be taken for one.
+    const answer = '{"title": "A" "custom_fields": {"0": {"value": "1"}}}';
+    assert.throws(() => parseAnalysisResponse(answer), /Invalid JSON/);
+  });
+
+  await check(
+    'an unreadable object does not hide a readable one after it',
+    () => {
+      const { parsed } = parseAnalysisResponse(
+        'Example: {broken "x"}\nAnswer: ' + JSON_ANSWER
+      );
+      assert.deepStrictEqual(parsed, RESULT);
+    }
+  );
+
+  for (const [name, answer] of UNREADABLE_ANSWERS) {
+    await check(`fails on ${name} with the OCR fallback phrase`, () => {
+      assert.throws(
+        () => parseAnalysisResponse(answer),
+        (error) =>
+          /^Invalid JSON response from API/.test(error.message) &&
+          shouldQueueForOcrOnAiError(error.message)
+      );
+    });
+  }
+
+  await check('fails on an empty answer with the OCR fallback phrase', () => {
+    for (const answer of ['', '   ', null, undefined]) {
+      assert.throws(
+        () => parseAnalysisResponse(answer),
+        (error) => /^Invalid JSON response from API/.test(error.message)
+      );
+    }
+  });
+
+  await check('stripResponseWrapping drops reasoning and fences only', () => {
+    assert.strictEqual(
+      stripResponseWrapping('<think>a\nb</think>\n```json\n{"a":1}\n```\n'),
+      '{"a":1}'
+    );
+    assert.strictEqual(stripResponseWrapping("I'm sorry."), "I'm sorry.");
+  });
+
+  /* --- every provider reads the same answers ----------------------------- */
+
+  await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+  await fs.writeFile(getThumbnailCachePath(DOCUMENT_ID), 'thumbnail');
+
+  for (const [label, service, answerWith] of PROVIDERS) {
+    for (const [name, answer] of READABLE_ANSWERS) {
+      await check(`${label}: reads ${name}`, async () => {
+        const analysis = await analyze(service, answerWith, answer);
+        assert.strictEqual(analysis.error, undefined, analysis.error);
+        sameDocument(label, analysis.document, expectedFor(name));
+      });
+    }
+
+    for (const [name, answer] of UNREADABLE_ANSWERS) {
+      await check(`${label}: ${name} reaches the OCR fallback`, async () => {
+        const analysis = await analyze(service, answerWith, answer);
+        assert.match(analysis.error || '', /Invalid JSON response from API/);
+        assert.ok(shouldQueueForOcrOnAiError(analysis.error), analysis.error);
+      });
+    }
+  }
+
+  await check('Ollama: an answer object is used as it is', async () => {
+    const analysis = await analyze(ollamaService, ollamaClient, RESULT);
+    assert.strictEqual(analysis.error, undefined, analysis.error);
+    sameDocument('Ollama', analysis.document, RESULT);
+  });
+
+  await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
+
+  if (failed > 0) {
+    console.error(`\n${failed} response parser case(s) failed`);
+    process.exit(1);
+  }
+  console.log('\nAll response parser cases passed');
+})();

From aec628d8387f1185457352a3fff0d45a4625fa12 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 11 Oct 2026 20:49:49 +0200
Subject: [PATCH 36/53] Shape the analysis result and spend the token budget in
 one place each

Background
Step 4 of the LLM service refactoring (services/ManualServiceRefactoring.md).
openai, azure and custom carried a byte-identical reserve-then-truncate
block, and all four built the success and failure records inline. Ollama's
character-based arithmetic (Finding 5) sat in three private methods.

Changes
- services/llm/tokenBudget/tiktokenBudget.js: reserve() counts the prompt
  with calculateTotalPromptTokens(), reserves RESPONSE_TOKENS and throws
  "Token limit exceeded: prompt too large for available token limit" when
  nothing (or no finite number) is left; fit() is truncateToTokenLimit().
- services/llm/tokenBudget/charBudget.js: Ollama's /2 estimate, num_ctx as
  min(prompt + answer, TOKEN_LIMIT), and reserve()/fit() in place of
  _fitContentToContext(), which is removed. Same reserve()/fit() shape as
  the tiktoken budget. _calculatePromptTokenCount()/_calculateNumCtx() stay
  as one-line wrappers; ollama-temperature-wiring stubs them.
- services/llm/analysisResult.js: buildSuccess(), buildFailure(error,
  message?), buildInsufficientContent() (OpenAI only, row 7c),
  assertAssignableMetadata() (the three chat providers; Ollama's row 7k is
  untouched) and withDocumentDefaults() (Ollama's field defaults).
- The four services use them; ~150 lines of duplicates removed.
- tests/test-llm-token-budget.js (23 cases, area prompts): both budgets on
  their own, and every provider spending the budget it was given.
- tests/test-llm-analysis-result-contract.js: a case for OpenAI's
  insufficient-content placeholder document, pinned nowhere before.
- Ollama's "exceeds the context budget" debug line was single-quoted and
  printed ${text.length} literally; it is a template literal now.
- Plan: step 4 marked done, TokenBudget typedef matched to the code.

Testing
The provider half of llm-token-budget and the new contract case pass
against the parent commit as well - a pure extraction. Mutation-checked:
divisor /4, dropping the answer reservation, a provider fitting to a larger
budget (Azure, Ollama), a dropped errorCode, a dropped placeholder field and
a dropped custom_fields default each fail at least one case.
node scripts/run-tests.js --all: 94 passed, 7 skipped, 25 failed; the same
25 fail on the parent commit (better-sqlite3 built for a different Node
ABI, Windows temp-dir EPERM, changelog/version mismatch). All llm-*,
ollama-*, thumbnail-optional-analysis, truncate-returns-string,
token-limit-config and ai-provider-streaming are green. ESLint and
Prettier clean on all changed files.

Impact
None intended at runtime: same prompts, same cuts, same num_ctx, same
result records and error messages. No API or OpenAPI change.

Upstream Status
Local only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 scripts/run-tests.js                       |   2 +
 services/ManualServiceRefactoring.md       |  34 +-
 services/azureService.js                   |  61 +---
 services/customService.js                  |  62 +---
 services/llm/analysisResult.js             | 107 ++++++
 services/llm/tokenBudget/charBudget.js     |  93 +++++
 services/llm/tokenBudget/tiktokenBudget.js |  69 ++++
 services/ollamaService.js                  | 103 ++----
 services/openaiService.js                  |  76 +---
 tests/test-llm-analysis-result-contract.js |  32 ++
 tests/test-llm-token-budget.js             | 385 +++++++++++++++++++++
 11 files changed, 784 insertions(+), 240 deletions(-)
 create mode 100644 services/llm/analysisResult.js
 create mode 100644 services/llm/tokenBudget/charBudget.js
 create mode 100644 services/llm/tokenBudget/tiktokenBudget.js
 create mode 100644 tests/test-llm-token-budget.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index 64fa40ff..6397e2f9 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -101,6 +101,7 @@ const TESTS = {
   'llm-ocr-fallback-phrases': 'test-llm-ocr-fallback-phrases.js',
   'llm-prompt-builder': 'test-llm-prompt-builder.js',
   'llm-response-parser': 'test-llm-response-parser.js',
+  'llm-token-budget': 'test-llm-token-budget.js',
   'custom-field-longtext': 'test-custom-field-longtext.js',
   'document-processing-service': 'test-document-processing-service.js',
   'settings-custom-field-type-conflict':
@@ -247,6 +248,7 @@ const AREAS = {
     'prompt-existing-data-serialization',
     'restricted-document-types-placeholder',
     'llm-prompt-builder',
+    'llm-token-budget',
     'token-limit-config',
     'truncate-returns-string',
   ],
diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index aca6e518..ba4cddc7 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -300,8 +300,8 @@ Written as JSDoc typedefs in `services/llm/contracts.js`; enforced by tests, not
 
 /**
  * @typedef {Object} TokenBudget
- * @property {(systemPrompt:string, extraPrompts:string[], model:string) => Promise<{totalPromptTokens:number, availableTokens:number}>} reserve
- * @property {(content:string, availableTokens:number, model:string) => Promise<string>} fit
+ * @property {(systemPrompt:string, extraPrompts?:string[], model?:string) => Promise<{totalPromptTokens:number, reservedTokens:number, availableTokens:number}>} reserve
+ * @property {(content:string, availableTokens:number, model?:string) => Promise<string>} fit
  */
 ```
 
@@ -509,9 +509,33 @@ carry no behavioral risk; the risk starts at step 4.
    content" early return still runs only on a failed parse, on the stripped text. Row l ("No
    response data from Ollama API") is untouched and still pinned as drift.
 
-4. **Extract `analysisResult.js`** and the two `tokenBudget` strategies with `llm-token-budget`.
-   Findings 7b/7j are already fixed; `services/thumbnailCache.js` is the collaborator the
-   composition roots inject in step 6, and `thumbnail-optional-analysis` must stay green throughout.
+4. ~~**Extract `analysisResult.js`** and the two `tokenBudget` strategies~~ **Done.**
+   `services/llm/tokenBudget/tiktokenBudget.js` replaces the reserve-then-truncate block that was
+   byte-identical in openai, azure and custom. `services/llm/tokenBudget/charBudget.js` holds
+   Ollama's arithmetic (Finding 5): `estimateTokens()` with the `/2` divisor, `numCtx()` as
+   `min(prompt + answer, TOKEN_LIMIT)`, and `reserve()`/`fit()` in place of
+   `_fitContentToContext()`, which is gone. Both budgets expose `reserve()` → `{ totalPromptTokens,
+reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char budget's
+   `reserve()` takes the prompt prefix alone, since it has no model and no extra messages to count.
+   `_calculatePromptTokenCount()`/`_calculateNumCtx()` stay as one-line wrappers because
+   `ollama-temperature-wiring` stubs them; `generateText()` reaches the char budget through them.
+
+   `services/llm/analysisResult.js` is the one place the result shape lives: `buildSuccess()`,
+   `buildFailure(error, message?)` (Custom passes its normalized timeout text),
+   `buildInsufficientContent()` (OpenAI only, row c), `assertAssignableMetadata()` (the three chat
+   providers; Ollama's row k is untouched and still pinned) and `withDocumentDefaults()` (Ollama's
+   field defaults; whether the chat providers adopt them is the parity decision of step 6).
+
+   `llm-token-budget` (23 cases) pins both budgets and that every provider spends the one it was
+   given. Its provider half was green against the services before this step, which is the
+   point of a pure extraction. `llm-analysis-result-contract` gained a case for OpenAI's
+   insufficient-content placeholder, which was pinned nowhere before. Mutation-checked:
+   divisor `/4`, a dropped answer reservation, a provider that fits to a larger budget, a dropped
+   `errorCode`, a dropped placeholder field and a dropped `custom_fields` default each fail a case.
+
+   One log fixed on the way: Ollama's "exceeds the context budget" debug line was written in single
+   quotes and printed `${text.length}` literally.
+
 5. **Introduce the transports.** `chatCompletionsTransport.js` covering openai/azure/custom, and
    `ollamaGenerateTransport.js` carrying the retry, `format`, `num_ctx`/`num_predict` and
    `done_reason` truncation check moved verbatim from `ollamaService.js:811-922`. Cover with
diff --git a/services/azureService.js b/services/azureService.js
index 094642ec..175c05b7 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -1,6 +1,5 @@
 const {
   calculateTokens,
-  calculateTotalPromptTokens,
   truncateToTokenLimit,
   writePromptToFile,
   extractChatMessageContent,
@@ -27,6 +26,12 @@ const fs = require('fs').promises;
 const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
+const tiktokenBudget = require('./llm/tokenBudget/tiktokenBudget');
+const {
+  buildSuccess,
+  buildFailure,
+  assertAssignableMetadata,
+} = require('./llm/analysisResult');
 const { parseAnalysisResponse } = require('./llm/responseParser');
 const responseLogPath = path.join(
   process.cwd(),
@@ -139,31 +144,11 @@ class AzureOpenAIService {
       }
 
       // Calculate tokens AFTER all prompt modifications are complete
-      const totalPromptTokens = await calculateTotalPromptTokens(
+      const { availableTokens } = await tiktokenBudget.reserve(
         systemPrompt,
         process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [],
         model
       );
-
-      const maxTokens = Number(config.tokenLimit);
-      const reservedTokens = totalPromptTokens + Number(config.responseTokens);
-      const availableTokens = maxTokens - reservedTokens;
-
-      // Validate that we have positive available tokens. A non-finite budget
-      // is the same failure: it slips past a bare `<= 0` and truncates the
-      // document down to nothing instead of stopping.
-      if (!Number.isFinite(availableTokens) || availableTokens <= 0) {
-        console.warn(
-          `[WARNING] No available tokens for content. Reserved: ${reservedTokens}, Max: ${maxTokens}`
-        );
-        throw new Error(
-          'Token limit exceeded: prompt too large for available token limit'
-        );
-      }
-
-      console.log(
-        `[DEBUG] Token calculation - Prompt: ${totalPromptTokens}, Reserved: ${reservedTokens}, Available: ${availableTokens}`
-      );
       console.log(
         `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
       );
@@ -171,7 +156,7 @@ class AzureOpenAIService {
         `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`
       );
 
-      const truncatedContent = await truncateToTokenLimit(
+      const truncatedContent = await tiktokenBudget.fit(
         content,
         availableTokens,
         model
@@ -235,33 +220,17 @@ class AzureOpenAIService {
         console.warn('Failed to write AI response log:', logError.message);
       }
 
-      if (
-        !parsedResponse ||
-        !Array.isArray(parsedResponse.tags) ||
-        (typeof parsedResponse.correspondent !== 'string' &&
-          parsedResponse.correspondent !== null)
-      ) {
-        throw new Error(
-          'AI could not determine assignable metadata: no tags or correspondent found'
-        );
-      }
+      assertAssignableMetadata(parsedResponse);
 
-      return {
-        document: parsedResponse,
-        metrics: mappedUsage,
-        truncated: truncatedContent.length < content.length,
-      };
+      return buildSuccess(
+        parsedResponse,
+        mappedUsage,
+        truncatedContent.length < content.length
+      );
     } catch (error) {
       console.error(`Failed to analyze document: ${error.message}`);
       console.debug(error);
-      return {
-        document: { tags: [], correspondent: null },
-        metrics: null,
-        error: error.message,
-        // Undefined for everything that is not one of ours; the scan loop
-        // falls back to its generic reason then.
-        errorCode: error.code,
-      };
+      return buildFailure(error);
     }
   }
 
diff --git a/services/customService.js b/services/customService.js
index 62c7d366..0e3ef633 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -1,6 +1,5 @@
 const {
   calculateTokens,
-  calculateTotalPromptTokens,
   truncateToTokenLimit,
   extractChatMessageContent,
   assertCompletionNotTruncated,
@@ -27,6 +26,12 @@ const fs = require('fs').promises;
 const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
+const tiktokenBudget = require('./llm/tokenBudget/tiktokenBudget');
+const {
+  buildSuccess,
+  buildFailure,
+  assertAssignableMetadata,
+} = require('./llm/analysisResult');
 const { parseAnalysisResponse } = require('./llm/responseParser');
 const responseLogPath = path.join(
   process.cwd(),
@@ -120,31 +125,11 @@ class CustomOpenAIService {
       }
 
       // Calculate tokens AFTER all prompt modifications are complete
-      const totalPromptTokens = await calculateTotalPromptTokens(
+      const { availableTokens } = await tiktokenBudget.reserve(
         systemPrompt,
         process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [],
         model
       );
-
-      const maxTokens = Number(config.tokenLimit);
-      const reservedTokens = totalPromptTokens + Number(config.responseTokens);
-      const availableTokens = maxTokens - reservedTokens;
-
-      // Validate that we have positive available tokens. A non-finite budget
-      // is the same failure: it slips past a bare `<= 0` and truncates the
-      // document down to nothing instead of stopping.
-      if (!Number.isFinite(availableTokens) || availableTokens <= 0) {
-        console.warn(
-          `[WARNING] No available tokens for content. Reserved: ${reservedTokens}, Max: ${maxTokens}`
-        );
-        throw new Error(
-          'Token limit exceeded: prompt too large for available token limit'
-        );
-      }
-
-      console.log(
-        `[DEBUG] Token calculation - Prompt: ${totalPromptTokens}, Reserved: ${reservedTokens}, Available: ${availableTokens}`
-      );
       console.log(
         `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
       );
@@ -152,7 +137,7 @@ class CustomOpenAIService {
         `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`
       );
 
-      const truncatedContent = await truncateToTokenLimit(
+      const truncatedContent = await tiktokenBudget.fit(
         content,
         availableTokens,
         model
@@ -242,23 +227,13 @@ class CustomOpenAIService {
         console.warn('Failed to write AI response log:', logError.message);
       }
 
-      // Validate response structure
-      if (
-        !parsedResponse ||
-        !Array.isArray(parsedResponse.tags) ||
-        (typeof parsedResponse.correspondent !== 'string' &&
-          parsedResponse.correspondent !== null)
-      ) {
-        throw new Error(
-          'AI could not determine assignable metadata: no tags or correspondent found'
-        );
-      }
+      assertAssignableMetadata(parsedResponse);
 
-      return {
-        document: parsedResponse,
-        metrics: mappedUsage,
-        truncated: truncatedContent.length < content.length,
-      };
+      return buildSuccess(
+        parsedResponse,
+        mappedUsage,
+        truncatedContent.length < content.length
+      );
     } catch (error) {
       const normalizedMessage = isTimeoutError(error)
         ? buildTimeoutErrorMessage('AI')
@@ -272,14 +247,7 @@ class CustomOpenAIService {
 
       console.error(`Failed to analyze document: ${normalizedMessage}`);
       console.debug(error);
-      return {
-        document: { tags: [], correspondent: null },
-        metrics: null,
-        error: normalizedMessage,
-        // Undefined for everything that is not one of ours; the scan loop
-        // falls back to its generic reason then.
-        errorCode: error.code,
-      };
+      return buildFailure(error, normalizedMessage);
     }
   }
 
diff --git a/services/llm/analysisResult.js b/services/llm/analysisResult.js
new file mode 100644
index 00000000..d95d6a47
--- /dev/null
+++ b/services/llm/analysisResult.js
@@ -0,0 +1,107 @@
+'use strict';
+
+/**
+ * The one place the analyzeDocument() result shape lives.
+ *
+ * Three callers read it — server.js processDocument(),
+ * mistralOcrService._runAiAnalysis() and routes/setup.js processDocument() —
+ * and hand `document` to documentProcessingService.buildUpdateData(). A
+ * narrowed `document` does not fail, it silently stops setting fields.
+ *
+ * Failures are returned, never thrown: the scan loop routes a failed analysis
+ * to the OCR queue by matching `error` against fixed phrases
+ * (shouldQueueForOcrOnAiError), and a throw would skip that entirely.
+ */
+
+/**
+ * @param {Object} document - What the model assigned
+ * @param {{promptTokens:number, completionTokens:number, totalTokens:number}|null} metrics
+ * @param {boolean} truncated - Whether the document was cut to fit
+ */
+function buildSuccess(document, metrics, truncated) {
+  return { document, metrics, truncated };
+}
+
+/**
+ * @param {Error} error - Why the analysis failed
+ * @param {string} [message] - Replaces error.message, e.g. a normalized timeout
+ */
+function buildFailure(error, message = error.message) {
+  return {
+    document: { tags: [], correspondent: null },
+    metrics: null,
+    error: message,
+    // Undefined for everything that is not one of ours; the scan loop falls
+    // back to its generic reason then.
+    errorCode: error.code,
+  };
+}
+
+/**
+ * The answer of a model that declined the document as too thin to analyse.
+ * Returned with a placeholder document instead of thrown, so the retry loop
+ * does not ask again; the error phrase routes it to OCR.
+ *
+ * @param {string|number} id - Paperless document id
+ * @param {Object|null} metrics - The tokens the refusal cost
+ */
+function buildInsufficientContent(id, metrics) {
+  return {
+    document: {
+      tags: [],
+      correspondent: 'Unknown',
+      title: `Document ${id}`,
+      document_date: new Date().toISOString().split('T')[0],
+      document_type: 'Document',
+      language: 'und',
+    },
+    metrics,
+    truncated: false,
+    error: 'Insufficient content for AI analysis',
+  };
+}
+
+/**
+ * Throw unless the answer assigns something: a tags array and a correspondent
+ * that is a string or null.
+ *
+ * @param {Object} parsed - The parsed answer
+ * @throws {Error} "AI could not determine assignable metadata: …"
+ */
+function assertAssignableMetadata(parsed) {
+  if (
+    !parsed ||
+    !Array.isArray(parsed.tags) ||
+    (typeof parsed.correspondent !== 'string' && parsed.correspondent !== null)
+  ) {
+    throw new Error(
+      'AI could not determine assignable metadata: no tags or correspondent found'
+    );
+  }
+}
+
+/**
+ * Every document field present: tags an array, everything else a value or
+ * null. Ollama's shaping; the chat providers pass the answer on as parsed.
+ *
+ * @param {Object} parsed - The parsed answer
+ */
+function withDocumentDefaults(parsed) {
+  return {
+    tags: Array.isArray(parsed.tags) ? parsed.tags : [],
+    correspondent: parsed.correspondent || null,
+    title: parsed.title || null,
+    document_date: parsed.document_date || null,
+    document_type: parsed.document_type || null,
+    language: parsed.language || null,
+    custom_fields: parsed.custom_fields || null,
+  };
+}
+
+module.exports = {
+  buildSuccess,
+  buildFailure,
+  buildInsufficientContent,
+  assertAssignableMetadata,
+  withDocumentDefaults,
+};
diff --git a/services/llm/tokenBudget/charBudget.js b/services/llm/tokenBudget/charBudget.js
new file mode 100644
index 00000000..f26512ba
--- /dev/null
+++ b/services/llm/tokenBudget/charBudget.js
@@ -0,0 +1,93 @@
+'use strict';
+
+/**
+ * Ollama's token budget, counted in characters.
+ *
+ * Two characters make one token. That is deliberately conservative: German
+ * and CJK text tokenizes at roughly two characters per token rather than the
+ * ~4 of English, and a ÷4 estimate under-sizes num_ctx for exactly those
+ * documents. Everything here uses the same divisor, so the prompt estimate,
+ * the context window and the document cut agree by construction (Finding 5
+ * of services/ManualServiceRefactoring.md — do not unify it with tiktoken's).
+ *
+ * Shares its reserve()/fit() shape with tiktokenBudget.js.
+ */
+
+const config = require('../../../config/config');
+
+const CHARS_PER_TOKEN = 2;
+
+/** Estimated tokens for a text. */
+function estimateTokens(text) {
+  return Math.ceil(String(text).length / CHARS_PER_TOKEN);
+}
+
+/**
+ * The context window to request: the prompt plus the reserved answer, but
+ * never more than TOKEN_LIMIT.
+ *
+ * @param {number} promptTokens - From estimateTokens()
+ * @param {number} responseTokens - Tokens reserved for the answer
+ * @returns {number}
+ */
+function numCtx(promptTokens, responseTokens) {
+  return Math.min(promptTokens + responseTokens, Number(config.tokenLimit));
+}
+
+/**
+ * Reserve room for the fixed prompt parts and the answer.
+ *
+ * A TOKEN_LIMIT or RESPONSE_TOKENS that is not a number leaves the document
+ * uncut (availableTokens is Infinity), as Ollama always did; it is the
+ * context window that is then wrong, not the document.
+ *
+ * @param {string} promptPrefix - The fully built prompt without the document
+ * @returns {Promise<{totalPromptTokens:number, reservedTokens:number, availableTokens:number}>}
+ * @throws {Error} "Token limit exceeded: …" when nothing is left for the document
+ */
+async function reserve(promptPrefix) {
+  const maxWindow = Number(config.tokenLimit);
+  const responseTokens = Number(config.responseTokens);
+  const totalPromptTokens = estimateTokens(promptPrefix);
+  const reservedTokens = totalPromptTokens + responseTokens;
+
+  if (!Number.isFinite(maxWindow) || !Number.isFinite(responseTokens)) {
+    return { totalPromptTokens, reservedTokens, availableTokens: Infinity };
+  }
+
+  const availableTokens = maxWindow - reservedTokens;
+  if (availableTokens <= 0) {
+    throw new Error(
+      'Token limit exceeded: the prompt alone does not fit the configured ' +
+        'TOKEN_LIMIT. Raise TOKEN_LIMIT or shorten the system prompt.'
+    );
+  }
+  return { totalPromptTokens, reservedTokens, availableTokens };
+}
+
+/**
+ * Cut the document to the tokens reserve() left for it.
+ *
+ * A document that is not a string is measured as its JSON. One that fits is
+ * returned as it was handed in; one that does not comes back as a string.
+ *
+ * @param {string|Object} content - Document content
+ * @param {number} availableTokens - From reserve()
+ * @returns {Promise<string|Object>}
+ */
+async function fit(content, availableTokens) {
+  const text = typeof content === 'string' ? content : JSON.stringify(content);
+  const maxChars = availableTokens * CHARS_PER_TOKEN;
+  if (text.length <= maxChars) {
+    return content;
+  }
+
+  console.log(
+    `[DEBUG] Document content (${text.length} chars) exceeds the context ` +
+      `budget (${maxChars} chars for a ${config.tokenLimit}-token window minus prompt` +
+      ` and ${config.responseTokens} reserved answer tokens); truncating`
+  );
+  return text.substring(0, maxChars);
+}
+
+module.exports = { estimateTokens, numCtx, reserve, fit };
diff --git a/services/llm/tokenBudget/tiktokenBudget.js b/services/llm/tokenBudget/tiktokenBudget.js
new file mode 100644
index 00000000..6ea58460
--- /dev/null
+++ b/services/llm/tokenBudget/tiktokenBudget.js
@@ -0,0 +1,69 @@
+'use strict';
+
+/**
+ * The token budget of the chat providers (openai, azure, custom).
+ *
+ * The prompt is counted with tiktoken (a ÷4 estimate for models tiktoken does
+ * not know), the configured answer is reserved, and the document gets what is
+ * left of TOKEN_LIMIT. This block used to be copied into all three services.
+ *
+ * Shares its reserve()/fit() shape with charBudget.js, which is Ollama's.
+ */
+
+const config = require('../../../config/config');
+const {
+  calculateTotalPromptTokens,
+  truncateToTokenLimit,
+} = require('../../serviceUtils');
+
+/**
+ * Reserve room for the prompt and the answer.
+ *
+ * @param {string} systemPrompt - The fully built system prompt
+ * @param {string[]} extraPrompts - Further prompt parts counted as messages
+ * @param {string} model - Model the prompt is counted for
+ * @returns {Promise<{totalPromptTokens:number, reservedTokens:number, availableTokens:number}>}
+ * @throws {Error} "Token limit exceeded: …" when nothing is left for the document
+ */
+async function reserve(systemPrompt, extraPrompts, model) {
+  const totalPromptTokens = await calculateTotalPromptTokens(
+    systemPrompt,
+    extraPrompts,
+    model
+  );
+
+  const maxTokens = Number(config.tokenLimit);
+  const reservedTokens = totalPromptTokens + Number(config.responseTokens);
+  const availableTokens = maxTokens - reservedTokens;
+
+  // A non-finite budget is the same failure as an exhausted one: it slips
+  // past a bare `<= 0` and truncates the document down to nothing instead of
+  // stopping.
+  if (!Number.isFinite(availableTokens) || availableTokens <= 0) {
+    console.warn(
+      `[WARNING] No available tokens for content. Reserved: ${reservedTokens}, Max: ${maxTokens}`
+    );
+    throw new Error(
+      'Token limit exceeded: prompt too large for available token limit'
+    );
+  }
+
+  console.log(
+    `[DEBUG] Token calculation - Prompt: ${totalPromptTokens}, Reserved: ${reservedTokens}, Available: ${availableTokens}`
+  );
+  return { totalPromptTokens, reservedTokens, availableTokens };
+}
+
+/**
+ * Cut the document to the tokens reserve() left for it.
+ *
+ * @param {string} content - Document content
+ * @param {number} availableTokens - From reserve()
+ * @param {string} model - Model the content is counted for
+ * @returns {Promise<string>}
+ */
+function fit(content, availableTokens, model) {
+  return truncateToTokenLimit(content, availableTokens, model);
+}
+
+module.exports = { reserve, fit };
diff --git a/services/ollamaService.js b/services/ollamaService.js
index ae113499..0e3a1aa1 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -16,6 +16,12 @@ const config = require('../config/config');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
 const { parseAnalysisResponse } = require('./llm/responseParser');
+const charBudget = require('./llm/tokenBudget/charBudget');
+const {
+  buildSuccess,
+  buildFailure,
+  withDocumentDefaults,
+} = require('./llm/analysisResult');
 const { buildCustomFieldsBlock } = require('./llm/customFieldsTemplate');
 
 /**
@@ -129,9 +135,16 @@ class OllamaService {
         console.log('[DEBUG] Ollama Service started with custom prompt');
       }
 
-      // Fit the document into what is left of the context window
+      // Fit the document into what is left of the context window. Without
+      // this an oversized prompt silently changed the meaning of the request:
+      // Ollama clamps num_ctx to the model's real limit, the long document
+      // filled the window, and the answer was cut off mid-JSON — reported as
+      // ai_response_truncated on exactly the long documents. With the content
+      // fitted, prompt + reserved answer always fit the configured
+      // TOKEN_LIMIT.
       const originalContentLength = String(content).length;
-      content = this._fitContentToContext(content, promptPrefix.length);
+      const { availableTokens } = await charBudget.reserve(promptPrefix);
+      content = await charBudget.fit(content, availableTokens);
 
       if (!customPrompt) {
         prompt = this._buildPrompt(
@@ -193,22 +206,11 @@ class OllamaService {
       await this._logPromptAndResponse(systemPrompt, prompt, parsedResponse);
 
       // Return results in consistent format
-      return {
-        document: parsedResponse,
-        metrics,
-        truncated: contentWasTruncated,
-      };
+      return buildSuccess(parsedResponse, metrics, contentWasTruncated);
     } catch (error) {
       console.error(`Error analyzing document with Ollama: ${error.message}`);
       console.debug(error);
-      return {
-        document: { tags: [], correspondent: null },
-        metrics: null,
-        error: error.message,
-        // Undefined for everything that is not one of ours; the scan loop
-        // falls back to its generic reason then.
-        errorCode: error.code,
-      };
+      return buildFailure(error);
     }
   }
 
@@ -233,57 +235,6 @@ class OllamaService {
     return content;
   }
 
-  /**
-   * Fit the document content into what is left of the context window once
-   * the fixed prompt parts and the reserved answer have taken their share.
-   *
-   * Without this an oversized prompt silently changed the meaning of the
-   * request: Ollama clamps num_ctx to the model's real limit, the long
-   * document filled the window, and the answer was cut off mid-JSON —
-   * reported as ai_response_truncated on exactly the long documents. With
-   * the content fitted, prompt + reserved answer always fit the configured
-   * TOKEN_LIMIT, so setting that to the model's true context removes the
-   * silent clamp entirely.
-   *
-   * Uses the same conservative 2-chars-per-token estimate as
-   * _calculatePromptTokenCount(), so the numbers agree by construction.
-   *
-   * @param {string} content - Document content
-   * @param {number} prefixChars - Length of the fully built prompt without the content tail
-   * @returns {string} Content shortened to the remaining budget when needed
-   */
-  _fitContentToContext(content, prefixChars) {
-    const maxWindow = Number(config.tokenLimit);
-    const responseTokens = Number(config.responseTokens);
-    if (!Number.isFinite(maxWindow) || !Number.isFinite(responseTokens)) {
-      return content;
-    }
-
-    const text =
-      typeof content === 'string' ? content : JSON.stringify(content);
-    const overheadTokens = Math.ceil(prefixChars / 2);
-    const availableTokens = maxWindow - responseTokens - overheadTokens;
-
-    if (availableTokens <= 0) {
-      throw new Error(
-        'Token limit exceeded: the prompt alone does not fit the configured ' +
-          'TOKEN_LIMIT. Raise TOKEN_LIMIT or shorten the system prompt.'
-      );
-    }
-
-    const maxChars = availableTokens * 2;
-    if (text.length <= maxChars) {
-      return content;
-    }
-
-    console.log(
-      '[DEBUG] Document content (${text.length} chars) exceeds the context ' +
-        'budget (${maxChars} chars for a ${maxWindow}-token window minus prompt' +
-        ' and ${responseTokens} reserved answer tokens); truncating'
-    );
-    return text.substring(0, maxChars);
-  }
-
   /**
    * Builds and returns a system prompt string combining configuration, content, and metadata.
    * @param {string} content - The main content of the document to be processed.
@@ -408,10 +359,7 @@ class OllamaService {
    * @returns {number} Estimated token count
    */
   _calculatePromptTokenCount(prompt) {
-    // Use conservative 2 chars/token estimate to avoid truncation
-    // with non-English models (CJK, German, etc.) where tokenization
-    // produces roughly 2 chars per token instead of ~4 for English
-    return Math.ceil(prompt.length / 2);
+    return charBudget.estimateTokens(prompt);
   }
 
   /**
@@ -421,10 +369,7 @@ class OllamaService {
    * @returns {number} Context window size
    */
   _calculateNumCtx(promptTokenCount, expectedResponseTokens) {
-    const totalTokenUsage = promptTokenCount + expectedResponseTokens;
-    const maxCtxLimit = Number(config.tokenLimit);
-
-    const numCtx = Math.min(totalTokenUsage, maxCtxLimit);
+    const numCtx = charBudget.numCtx(promptTokenCount, expectedResponseTokens);
 
     console.log('Prompt Token Count:', promptTokenCount);
     console.log('Expected Response Tokens:', expectedResponseTokens);
@@ -656,15 +601,7 @@ class OllamaService {
     }
     try {
       const { parsed } = parseAnalysisResponse(responseData.response);
-      return {
-        tags: Array.isArray(parsed.tags) ? parsed.tags : [],
-        correspondent: parsed.correspondent || null,
-        title: parsed.title || null,
-        document_date: parsed.document_date || null,
-        document_type: parsed.document_type || null,
-        language: parsed.language || null,
-        custom_fields: parsed.custom_fields || null,
-      };
+      return withDocumentDefaults(parsed);
     } catch (error) {
       console.error('Error parsing Ollama response:', error.message);
       throw error;
diff --git a/services/openaiService.js b/services/openaiService.js
index 3e0289fc..b30f90e5 100644
--- a/services/openaiService.js
+++ b/services/openaiService.js
@@ -1,6 +1,5 @@
 const {
   calculateTokens,
-  calculateTotalPromptTokens,
   truncateToTokenLimit,
   writePromptToFile,
   extractChatMessageContent,
@@ -26,6 +25,13 @@ const fs = require('fs').promises;
 const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
+const tiktokenBudget = require('./llm/tokenBudget/tiktokenBudget');
+const {
+  buildSuccess,
+  buildFailure,
+  buildInsufficientContent,
+  assertAssignableMetadata,
+} = require('./llm/analysisResult');
 const {
   parseAnalysisResponse,
   stripResponseWrapping,
@@ -124,31 +130,11 @@ class OpenAIService {
       }
 
       // Calculate tokens AFTER all prompt modifications are complete
-      const totalPromptTokens = await calculateTotalPromptTokens(
+      const { availableTokens } = await tiktokenBudget.reserve(
         systemPrompt,
         process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [],
         model
       );
-
-      const maxTokens = Number(config.tokenLimit);
-      const reservedTokens = totalPromptTokens + Number(config.responseTokens);
-      const availableTokens = maxTokens - reservedTokens;
-
-      // Validate that we have positive available tokens. A non-finite budget
-      // is the same failure: it slips past a bare `<= 0` and truncates the
-      // document down to nothing instead of stopping.
-      if (!Number.isFinite(availableTokens) || availableTokens <= 0) {
-        console.warn(
-          `[WARNING] No available tokens for content. Reserved: ${reservedTokens}, Max: ${maxTokens}`
-        );
-        throw new Error(
-          'Token limit exceeded: prompt too large for available token limit'
-        );
-      }
-
-      console.log(
-        `[DEBUG] Token calculation - Prompt: ${totalPromptTokens}, Reserved: ${reservedTokens}, Available: ${availableTokens}`
-      );
       console.log(
         `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
       );
@@ -156,7 +142,7 @@ class OpenAIService {
         `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`
       );
 
-      const truncatedContent = await truncateToTokenLimit(
+      const truncatedContent = await tiktokenBudget.fit(
         content,
         availableTokens,
         model
@@ -222,19 +208,7 @@ class OpenAIService {
         ) {
           console.warn(`Document ${id} has insufficient content for analysis`);
           // Return a default structure instead of throwing to prevent retry loops
-          return {
-            document: {
-              tags: [],
-              correspondent: 'Unknown',
-              title: `Document ${id}`,
-              document_date: new Date().toISOString().split('T')[0],
-              document_type: 'Document',
-              language: 'und',
-            },
-            metrics: mappedUsage,
-            truncated: false,
-            error: 'Insufficient content for AI analysis',
-          };
+          return buildInsufficientContent(id, mappedUsage);
         }
         throw error;
       }
@@ -246,33 +220,17 @@ class OpenAIService {
         console.warn('Failed to write AI response log:', logError.message);
       }
 
-      if (
-        !parsedResponse ||
-        !Array.isArray(parsedResponse.tags) ||
-        (typeof parsedResponse.correspondent !== 'string' &&
-          parsedResponse.correspondent !== null)
-      ) {
-        throw new Error(
-          'AI could not determine assignable metadata: no tags or correspondent found'
-        );
-      }
+      assertAssignableMetadata(parsedResponse);
 
-      return {
-        document: parsedResponse,
-        metrics: mappedUsage,
-        truncated: truncatedContent.length < content.length,
-      };
+      return buildSuccess(
+        parsedResponse,
+        mappedUsage,
+        truncatedContent.length < content.length
+      );
     } catch (error) {
       console.error(`Failed to analyze document: ${error.message}`);
       console.debug(error);
-      return {
-        document: { tags: [], correspondent: null },
-        metrics: null,
-        error: error.message,
-        // Undefined for everything that is not one of ours; the scan loop
-        // falls back to its generic reason then.
-        errorCode: error.code,
-      };
+      return buildFailure(error);
     }
   }
 
diff --git a/tests/test-llm-analysis-result-contract.js b/tests/test-llm-analysis-result-contract.js
index d27ead5b..c4fd0e93 100644
--- a/tests/test-llm-analysis-result-contract.js
+++ b/tests/test-llm-analysis-result-contract.js
@@ -349,6 +349,38 @@ const assertFailureRecord = (analysis) => {
     }
   );
 
+  /* OpenAI alone (Finding 7c) answers a refusal with a placeholder document
+     instead of the failure record, so the retry loop does not ask again. The
+     error phrase still routes it to OCR; the document is what the write-back
+     would set, so it is pinned field by field. */
+  await check(
+    'OpenAI: a refusal returns the insufficient-content placeholder',
+    async () => {
+      const openai = PROVIDERS.find((p) => p.label === 'OpenAI');
+      openai.arrangeAnswer("I'm sorry, I cannot analyse this document.");
+      const analysis = await analyse(openai.service);
+
+      assert.strictEqual(
+        analysis.error,
+        'Insufficient content for AI analysis'
+      );
+      assert.deepStrictEqual(analysis.document, {
+        tags: [],
+        correspondent: 'Unknown',
+        title: `Document ${DOCUMENT_ID}`,
+        document_date: new Date().toISOString().split('T')[0],
+        document_type: 'Document',
+        language: 'und',
+      });
+      assert.deepStrictEqual(analysis.metrics, {
+        promptTokens: 900,
+        completionTokens: 120,
+        totalTokens: 1020,
+      });
+      assert.strictEqual(analysis.truncated, false);
+    }
+  );
+
   await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
 
   if (failed) {
diff --git a/tests/test-llm-token-budget.js b/tests/test-llm-token-budget.js
new file mode 100644
index 00000000..add7e1c0
--- /dev/null
+++ b/tests/test-llm-token-budget.js
@@ -0,0 +1,385 @@
+/**
+ * The two token budgets, each written once.
+ *
+ * Step 4 of the LLM service refactoring (services/ManualServiceRefactoring.md).
+ * openai, azure and custom carried the same reserve-then-truncate block:
+ * tiktoken counts for the prompt, the configured answer reserved, the
+ * document cut to what is left. It lives in
+ * services/llm/tokenBudget/tiktokenBudget.js now. Ollama's arithmetic is a
+ * different model on purpose (Finding 5) — 2 chars per token, num_ctx sized
+ * as min(prompt + answer, TOKEN_LIMIT), the document fitted by characters —
+ * and lives in services/llm/tokenBudget/charBudget.js. Both expose the same
+ * reserve()/fit() pair, so the analyzer of step 6 can take either.
+ *
+ * The first half pins each budget on its own, the second half checks that
+ * every provider spends the budget it was given.
+ */
+
+const assert = require('assert');
+const fs = require('fs').promises;
+
+process.env.AI_PROVIDER = 'openai';
+process.env.OPENAI_API_KEY = 'test-key';
+process.env.OPENAI_MODEL = 'gpt-4';
+process.env.AZURE_API_KEY = 'test-key';
+process.env.AZURE_ENDPOINT = 'https://example.invalid';
+process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment';
+process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1';
+process.env.CUSTOM_API_KEY = 'test-key';
+process.env.CUSTOM_MODEL = 'test-model';
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+process.env.USE_PROMPT_TAGS = 'no';
+delete process.env.CUSTOM_FIELDS;
+delete process.env.CONTENT_MAX_LENGTH;
+
+const config = require('../config/config');
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('../services/thumbnailCachePaths');
+const {
+  calculateTotalPromptTokens,
+  truncateToTokenLimit,
+} = require('../services/serviceUtils');
+const tiktokenBudget = require('../services/llm/tokenBudget/tiktokenBudget');
+const charBudget = require('../services/llm/tokenBudget/charBudget');
+const openaiService = require('../services/openaiService');
+const azureService = require('../services/azureService');
+const customService = require('../services/customService');
+const ollamaService = require('../services/ollamaService');
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+/** Own id so the seeded thumbnail cannot collide with a real cached document. */
+const DOCUMENT_ID = 'testLlmTokenBudget';
+
+const CHAT_LIMIT_MESSAGE =
+  'Token limit exceeded: prompt too large for available token limit';
+
+const ANSWER = JSON.stringify({
+  title: 'Telekom invoice July',
+  correspondent: 'Telekom Deutschland GmbH',
+  tags: ['invoice'],
+  document_type: 'Rechnung',
+  document_date: '2026-09-03',
+  language: 'de',
+});
+
+/** Long enough that every limit below has to cut it. */
+const LONG_CONTENT = 'Invoice from Telekom Deutschland GmbH, July. '.repeat(
+  400
+);
+
+function limits(tokenLimit, responseTokens) {
+  Object.assign(config, { tokenLimit, responseTokens });
+}
+
+/* --- the providers, each with a client that records what it was sent ----- */
+
+function recordingChatClient(service, requests) {
+  service.client = {
+    chat: {
+      completions: {
+        create: async (request) => {
+          requests.push(request);
+          return {
+            choices: [{ message: { content: ANSWER }, finish_reason: 'stop' }],
+            usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 },
+          };
+        },
+      },
+    },
+    models: { list: async () => ({ data: [] }) },
+  };
+}
+
+function recordingOllamaClient(requests) {
+  ollamaService.client = {
+    post: async (url, body) => {
+      requests.push(body);
+      return {
+        data: {
+          response: ANSWER,
+          done_reason: 'stop',
+          eval_count: 3,
+          prompt_eval_count: 9,
+        },
+      };
+    },
+    get: async () => ({ status: 200, data: { models: [] } }),
+  };
+}
+
+const CHAT_PROVIDERS = [
+  ['OpenAI', openaiService, () => process.env.OPENAI_MODEL],
+  ['Azure', azureService, () => process.env.AZURE_DEPLOYMENT_NAME],
+  ['Custom', customService, () => config.custom.model],
+];
+
+async function analyzeRecorded(service, content, record) {
+  const requests = [];
+  record(requests);
+  const analysis = await service.analyzeDocument(
+    content,
+    [],
+    [],
+    [],
+    DOCUMENT_ID
+  );
+  return { analysis, requests };
+}
+
+(async () => {
+  console.log('\n=== LLM token budget ===');
+
+  /* --- tiktokenBudget ----------------------------------------------------- */
+
+  await check(
+    'tiktoken: reserve counts the prompt and the answer',
+    async () => {
+      limits(4000, 500);
+      const prompt = 'Analyse the document and answer in JSON.';
+      const budget = await tiktokenBudget.reserve(
+        prompt,
+        ['extra tags'],
+        'gpt-4'
+      );
+      const counted = await calculateTotalPromptTokens(
+        prompt,
+        ['extra tags'],
+        'gpt-4'
+      );
+      assert.deepStrictEqual(budget, {
+        totalPromptTokens: counted,
+        reservedTokens: counted + 500,
+        availableTokens: 4000 - counted - 500,
+      });
+    }
+  );
+
+  await check('tiktoken: no room left is "Token limit exceeded"', async () => {
+    limits(100, 100);
+    await assert.rejects(
+      () => tiktokenBudget.reserve('Analyse.', [], 'gpt-4'),
+      (error) => error.message === CHAT_LIMIT_MESSAGE
+    );
+  });
+
+  await check('tiktoken: a broken limit is refused, not spent', async () => {
+    for (const [tokenLimit, responseTokens] of [
+      [Number.NaN, 100],
+      [4000, Number.NaN],
+      [undefined, 100],
+    ]) {
+      limits(tokenLimit, responseTokens);
+      await assert.rejects(
+        () => tiktokenBudget.reserve('Analyse.', [], 'gpt-4'),
+        (error) => error.message === CHAT_LIMIT_MESSAGE
+      );
+    }
+  });
+
+  await check('tiktoken: fit is truncateToTokenLimit', async () => {
+    assert.strictEqual(
+      await tiktokenBudget.fit(LONG_CONTENT, 50, 'gpt-4'),
+      await truncateToTokenLimit(LONG_CONTENT, 50, 'gpt-4')
+    );
+    assert.strictEqual(await tiktokenBudget.fit('short', 50, 'gpt-4'), 'short');
+  });
+
+  /* --- charBudget (Finding 5) --------------------------------------------- */
+
+  await check('char: two characters make one token, rounded up', () => {
+    assert.strictEqual(charBudget.estimateTokens(''), 0);
+    assert.strictEqual(charBudget.estimateTokens('a'), 1);
+    assert.strictEqual(charBudget.estimateTokens('abcd'), 2);
+    assert.strictEqual(charBudget.estimateTokens('abcde'), 3);
+  });
+
+  await check('char: num_ctx is min(prompt + answer, TOKEN_LIMIT)', () => {
+    limits(8192, 1000);
+    assert.strictEqual(charBudget.numCtx(2000, 1000), 3000);
+    assert.strictEqual(charBudget.numCtx(7500, 1000), 8192);
+  });
+
+  await check(
+    'char: reserve leaves TOKEN_LIMIT - answer - prompt/2',
+    async () => {
+      limits(8192, 1000);
+      const budget = await charBudget.reserve('x'.repeat(301));
+      assert.deepStrictEqual(budget, {
+        totalPromptTokens: 151,
+        reservedTokens: 1151,
+        availableTokens: 8192 - 1151,
+      });
+    }
+  );
+
+  await check('char: no room left is "Token limit exceeded"', async () => {
+    limits(1000, 900);
+    await assert.rejects(
+      () => charBudget.reserve('x'.repeat(200)),
+      /^Error: Token limit exceeded: the prompt alone does not fit the configured TOKEN_LIMIT/
+    );
+  });
+
+  await check('char: a broken limit leaves the document whole', async () => {
+    limits(Number.NaN, 1000);
+    const budget = await charBudget.reserve('x'.repeat(200));
+    assert.strictEqual(budget.availableTokens, Infinity);
+    assert.strictEqual(
+      await charBudget.fit(LONG_CONTENT, budget.availableTokens),
+      LONG_CONTENT
+    );
+  });
+
+  await check(
+    'char: fit cuts to two characters per available token',
+    async () => {
+      assert.strictEqual(
+        await charBudget.fit(LONG_CONTENT, 100),
+        LONG_CONTENT.slice(0, 200)
+      );
+      assert.strictEqual(await charBudget.fit('short', 100), 'short');
+    }
+  );
+
+  await check('char: fit measures a non-string document as JSON', async () => {
+    const doc = { text: 'x'.repeat(500) };
+    const cut = await charBudget.fit(doc, 50);
+    assert.strictEqual(cut, JSON.stringify(doc).slice(0, 100));
+    // One that fits comes back as it was handed in, not as its JSON.
+    const small = { a: 1 };
+    assert.strictEqual(await charBudget.fit(small, 50), small);
+  });
+
+  /* --- every provider spends the budget it was given ---------------------- */
+
+  await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+  await fs.writeFile(getThumbnailCachePath(DOCUMENT_ID), 'thumbnail');
+
+  for (const [label, service, model] of CHAT_PROVIDERS) {
+    const record = (requests) => recordingChatClient(service, requests);
+
+    await check(
+      `${label}: the document is cut to the tiktoken budget`,
+      async () => {
+        limits(600, 200);
+        const { analysis, requests } = await analyzeRecorded(
+          service,
+          LONG_CONTENT,
+          record
+        );
+        assert.strictEqual(analysis.error, undefined, analysis.error);
+        assert.strictEqual(requests.length, 1);
+        const system = requests[0].messages.find((m) => m.role === 'system');
+        const user = requests[0].messages.find((m) => m.role === 'user');
+        const budget = await tiktokenBudget.reserve(
+          system.content,
+          [],
+          model()
+        );
+        assert.strictEqual(
+          user.content,
+          await tiktokenBudget.fit(
+            LONG_CONTENT,
+            budget.availableTokens,
+            model()
+          )
+        );
+        assert.strictEqual(analysis.truncated, true);
+      }
+    );
+
+    await check(`${label}: a document that fits is sent whole`, async () => {
+      limits(8000, 200);
+      const { analysis, requests } = await analyzeRecorded(
+        service,
+        'Short invoice.',
+        record
+      );
+      const user = requests[0].messages.find((m) => m.role === 'user');
+      assert.strictEqual(user.content, 'Short invoice.');
+      assert.strictEqual(analysis.truncated, false);
+    });
+
+    await check(`${label}: no room left fails before any request`, async () => {
+      limits(100, 100);
+      const { analysis, requests } = await analyzeRecorded(
+        service,
+        LONG_CONTENT,
+        record
+      );
+      assert.strictEqual(analysis.error, CHAT_LIMIT_MESSAGE);
+      assert.strictEqual(requests.length, 0);
+    });
+  }
+
+  await check('Ollama: the document is cut to the char budget', async () => {
+    limits(2000, 500);
+    const { analysis, requests } = await analyzeRecorded(
+      ollamaService,
+      LONG_CONTENT,
+      recordingOllamaClient
+    );
+    assert.strictEqual(analysis.error, undefined, analysis.error);
+    const prompt = requests[0].prompt;
+    // The prompt ends with the fitted document as a JSON string literal.
+    const start = prompt.indexOf('"Invoice from');
+    const sentContent = JSON.parse(prompt.slice(start).trim());
+    const prefix = prompt.slice(0, start - '\n        '.length);
+    const budget = await charBudget.reserve(prefix + '\n        ');
+    assert.ok(sentContent.length < LONG_CONTENT.length, 'the document was cut');
+    assert.ok(
+      sentContent.length <= budget.availableTokens * 2,
+      `${sentContent.length} chars sent for ${budget.availableTokens} tokens`
+    );
+    assert.strictEqual(analysis.truncated, true);
+  });
+
+  await check(
+    'Ollama: num_ctx is min(prompt/2 + answer, TOKEN_LIMIT)',
+    async () => {
+      limits(2000, 500);
+      const { requests } = await analyzeRecorded(
+        ollamaService,
+        'Short invoice.',
+        recordingOllamaClient
+      );
+      const { prompt, options } = requests[0];
+      assert.strictEqual(
+        options.num_ctx,
+        Math.min(Math.ceil(prompt.length / 2) + 500, 2000)
+      );
+      assert.strictEqual(options.num_predict, 500);
+    }
+  );
+
+  await check('Ollama: no room left fails before any request', async () => {
+    limits(600, 500);
+    const { analysis, requests } = await analyzeRecorded(
+      ollamaService,
+      LONG_CONTENT,
+      recordingOllamaClient
+    );
+    assert.match(analysis.error || '', /^Token limit exceeded/);
+    assert.strictEqual(requests.length, 0);
+  });
+
+  await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
+
+  if (failed > 0) {
+    console.error(`\n${failed} token budget case(s) failed`);
+    process.exit(1);
+  }
+  console.log('\nAll token budget cases passed');
+})();

From 992b2ea1c3e93f62fcbbb879025934a12f346e7d Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 11 Oct 2026 21:03:29 +0200
Subject: [PATCH 37/53] Send the analysis request through one transport per API
 family
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 5a of the LLM service refactoring (services/ManualServiceRefactoring.md).
Each of the four services built its own analysis request, checked the
answer for a cut-off, pulled the text out and mapped the usage. The three
chat providers did it identically up to the label; Ollama's retry, error
wrapping, headers and done_reason check sat in the service.

Changes
- services/llm/transports/chatCompletionsTransport.js: owns the SDK client
  (createClient() factory, built once on first use) and complete(), which
  sends system + user messages, omits an undefined temperature, raises
  ai_response_truncated on finish_reason "length", and returns
  { payload, metrics }. describeMissingContent() lets Custom keep its
  detailed "Invalid API response structure: …" message.
- services/llm/transports/ollamaGenerateTransport.js: /api/generate with
  format, num_ctx, num_predict, think:false unless OLLAMA_THINK, bearer
  header, one retry on a 5xx or no response, the wrapped upstream error and
  the done_reason check - moved verbatim. Returns { payload, metrics, raw }.
  The axios client is handed in by the service, where tests mock it.
- The services own a `transport`; `client` (and Ollama's apiUrl/model) are
  accessors onto it, so tests that replace service.client are unchanged.
  _callOllamaAPI(), _assertNotTruncated(), _extractOllamaMetrics() and
  _buildRequestHeaders() delegate; the private retry helpers are gone.
- Provider-specific values stay with the provider: OpenAI sends no
  temperature for o3-mini, Azure still sends 0.3 (row 7a, step 6).
- One comment corrected on the way: the cut-off check said the window was
  sized from "length / 4"; it is length / 2.
- tests/test-llm-transport-request-mapping.js (31 cases, area processing).
- Plan: step 5 split into 5a (done) and 5b (generateText, checkStatus).

Testing
Mutation-checked: always sending temperature, a different system role, a
changed top_k, a fixed Ollama temperature, no retry and a different
truncation code each fail a case (llm-transport-request-mapping,
llm-prompt-builder, ollama-temperature-wiring, ollama-upstream-error,
response-truncation-detection).
node scripts/run-tests.js --all: 95 passed, 7 skipped, 25 failed; the same
25 fail on the parent commit (better-sqlite3 built for a different Node
ABI, Windows temp-dir EPERM, changelog/version mismatch). ESLint and
Prettier clean on all changed files.

Impact
None intended at runtime: the same requests, retries, errors and metrics.
generateText() and checkStatus() are untouched. No API or OpenAPI change.

Upstream Status
Local only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 scripts/run-tests.js                          |   2 +
 services/ManualServiceRefactoring.md          |  21 +-
 services/azureService.js                      |  80 +--
 services/customService.js                     | 111 ++--
 .../transports/chatCompletionsTransport.js    | 110 ++++
 .../llm/transports/ollamaGenerateTransport.js | 278 +++++++++
 services/ollamaService.js                     | 255 ++------
 services/openaiService.js                     | 109 ++--
 tests/test-llm-transport-request-mapping.js   | 580 ++++++++++++++++++
 9 files changed, 1166 insertions(+), 380 deletions(-)
 create mode 100644 services/llm/transports/chatCompletionsTransport.js
 create mode 100644 services/llm/transports/ollamaGenerateTransport.js
 create mode 100644 tests/test-llm-transport-request-mapping.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index 6397e2f9..2861340b 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -102,6 +102,7 @@ const TESTS = {
   'llm-prompt-builder': 'test-llm-prompt-builder.js',
   'llm-response-parser': 'test-llm-response-parser.js',
   'llm-token-budget': 'test-llm-token-budget.js',
+  'llm-transport-request-mapping': 'test-llm-transport-request-mapping.js',
   'custom-field-longtext': 'test-custom-field-longtext.js',
   'document-processing-service': 'test-document-processing-service.js',
   'settings-custom-field-type-conflict':
@@ -240,6 +241,7 @@ const AREAS = {
     'llm-analysis-result-contract',
     'llm-ocr-fallback-phrases',
     'llm-response-parser',
+    'llm-transport-request-mapping',
     'rescan-fetch-before-delete',
   ],
   prompts: [
diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index ba4cddc7..1a47c18a 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -538,8 +538,25 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
 
 5. **Introduce the transports.** `chatCompletionsTransport.js` covering openai/azure/custom, and
    `ollamaGenerateTransport.js` carrying the retry, `format`, `num_ctx`/`num_predict` and
-   `done_reason` truncation check moved verbatim from `ollamaService.js:811-922`. Cover with
-   `llm-transport-request-mapping`.
+   `done_reason` truncation check moved verbatim from `ollamaService.js`. Cover with
+   `llm-transport-request-mapping`. Landed in two commits, because `generateText()` brings
+   streaming, abort, three reasoning dialects and per-provider defaults the analysis call has
+   none of:
+   - ~~**5a — the analysis request.**~~ **Done.** Each service owns a `transport`; `client`
+     (and Ollama's `apiUrl`/`model`) are accessors onto it, so the many tests that replace
+     `service.client` keep working unchanged. The chat transport takes a `createClient()`
+     factory — OpenAI's still carries the unreachable branches of row h, removed in step 8 — and
+     an optional `describeMissingContent()`, which is how Custom keeps its detailed
+     "Invalid API response structure: missing choices[0]." (355ac29). Per-provider temperature
+     stays with the provider: OpenAI passes `undefined` for o3-mini, which the transport omits;
+     Azure still passes 0.3 and is pinned as such until step 6 (row a). The Ollama transport gets
+     the axios client handed in, because `ollama-temperature-wiring` mocks axios for the service
+     module; `_callOllamaAPI()`, `_assertNotTruncated()`, `_extractOllamaMetrics()` and
+     `_buildRequestHeaders()` are one-line delegations, the first and third because tests call
+     them. `num_ctx` is still computed by the service and handed in; moving it behind the
+     transport is step 6. `llm-transport-request-mapping` (31 cases) pins both transports with
+     fake clients and every provider's analysis request; six mutations each fail a case.
+   - **5b — `generateText()`** through the same transports. `checkStatus()` (row g) moves with it.
 6. **Introduce `documentAnalyzer.js`** and the four composition roots. Reduce
    `services/{openai,azure,custom,ollama}Service.js` to re-export shims. Add `llm-provider-parity`.
    Fix Finding 7a here (Azure temperature) with its test.
diff --git a/services/azureService.js b/services/azureService.js
index 175c05b7..60d7c919 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -27,6 +27,9 @@ const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
 const tiktokenBudget = require('./llm/tokenBudget/tiktokenBudget');
+const {
+  ChatCompletionsTransport,
+} = require('./llm/transports/chatCompletionsTransport');
 const {
   buildSuccess,
   buildFailure,
@@ -73,18 +76,31 @@ function supportsStreamOptions(apiVersion) {
 
 class AzureOpenAIService {
   constructor() {
-    this.client = null;
+    this.transport = new ChatCompletionsTransport({
+      label: 'AzureOpenAI',
+      createClient: () =>
+        config.aiProvider === 'azure'
+          ? new AzureOpenAI({
+              apiKey: config.azure.apiKey,
+              endpoint: config.azure.endpoint,
+              deploymentName: config.azure.deploymentName,
+              apiVersion: config.azure.apiVersion,
+            })
+          : null,
+    });
+  }
+
+  /** The SDK client. It lives on the transport; tests replace it here. */
+  get client() {
+    return this.transport.client;
+  }
+
+  set client(client) {
+    this.transport.client = client;
   }
 
   initialize() {
-    if (!this.client && config.aiProvider === 'azure') {
-      this.client = new AzureOpenAI({
-        apiKey: config.azure.apiKey,
-        endpoint: config.azure.endpoint,
-        deploymentName: config.azure.deploymentName,
-        apiVersion: config.azure.apiVersion,
-      });
-    }
+    this.transport.initialize();
   }
 
   async analyzeDocument(
@@ -98,11 +114,6 @@ class AzureOpenAIService {
   ) {
     try {
       this.initialize();
-      const now = new Date();
-      const timestamp = now.toLocaleString('de-DE', {
-        dateStyle: 'short',
-        timeStyle: 'short',
-      });
 
       if (!this.client) {
         throw new Error('AzureOpenAI client not initialized');
@@ -164,43 +175,14 @@ class AzureOpenAIService {
 
       await writePromptToFile(systemPrompt, truncatedContent);
 
-      const response = await this.client.chat.completions.create({
-        model: model,
-        messages: [
-          {
-            role: 'system',
-            content: systemPrompt,
-          },
-          {
-            role: 'user',
-            content: truncatedContent,
-          },
-        ],
+      const { payload, metrics: mappedUsage } = await this.transport.complete({
+        systemPrompt,
+        userContent: truncatedContent,
+        model,
+        // Hardcoded unlike the other two (Finding 7a); fixed in step 6.
         temperature: 0.3,
       });
-
-      assertCompletionNotTruncated(
-        response,
-        'AzureOpenAI',
-        'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.'
-      );
-      const message = response?.choices?.[0]?.message;
-      let jsonContent = extractChatMessageContent(message, 'AzureOpenAI');
-      if (!jsonContent) {
-        throw new Error('Invalid API response structure');
-      }
-
-      console.log(`[DEBUG] [${timestamp}] AzureOpenAI request sent`);
-      console.log(
-        `[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}`
-      );
-
-      const usage = response.usage;
-      const mappedUsage = {
-        promptTokens: usage.prompt_tokens,
-        completionTokens: usage.completion_tokens,
-        totalTokens: usage.total_tokens,
-      };
+      let jsonContent = payload;
 
       let parsedResponse;
       try {
diff --git a/services/customService.js b/services/customService.js
index 0e3ef633..0b8e4159 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -27,6 +27,9 @@ const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
 const tiktokenBudget = require('./llm/tokenBudget/tiktokenBudget');
+const {
+  ChatCompletionsTransport,
+} = require('./llm/transports/chatCompletionsTransport');
 const {
   buildSuccess,
   buildFailure,
@@ -56,18 +59,48 @@ const QWEN_MODEL_PATTERN = /qwen/i;
 
 class CustomOpenAIService {
   constructor() {
-    this.client = null;
+    this.transport = new ChatCompletionsTransport({
+      label: 'Custom OpenAI',
+      createClient: () =>
+        config.aiProvider === 'custom'
+          ? new OpenAI({
+              baseURL: config.custom.apiUrl,
+              // OpenAI-compatible SDKs require an apiKey value even when auth is disabled.
+              apiKey: config.custom.apiKey || CUSTOM_PROVIDER_FALLBACK_API_KEY,
+            })
+          : null,
+      // Every message keeps the "Invalid API response structure" prefix:
+      // shouldQueueForOcrOnAiError() matches on that phrase, and without it
+      // the document no longer reaches the OCR fallback.
+      describeMissingContent: (response, message) => {
+        const missing = !response
+          ? 'the response is empty'
+          : !response.choices
+            ? 'missing choices'
+            : !response.choices[0]
+              ? 'missing choices[0]'
+              : !response.choices[0].message
+                ? 'missing choices[0].message'
+                : !response.choices[0].message.content
+                  ? 'missing choices[0].message.content'
+                  : `unusable content ${message?.content}`;
+        return `Invalid API response structure: ${missing}.`;
+      },
+    });
     this.tokenizer = null;
   }
 
+  /** The SDK client. It lives on the transport; tests replace it here. */
+  get client() {
+    return this.transport.client;
+  }
+
+  set client(client) {
+    this.transport.client = client;
+  }
+
   initialize() {
-    if (!this.client && config.aiProvider === 'custom') {
-      this.client = new OpenAI({
-        baseURL: config.custom.apiUrl,
-        // OpenAI-compatible SDKs require an apiKey value even when auth is disabled.
-        apiKey: config.custom.apiKey || CUSTOM_PROVIDER_FALLBACK_API_KEY,
-      });
-    }
+    this.transport.initialize();
   }
 
   async analyzeDocument(
@@ -81,11 +114,6 @@ class CustomOpenAIService {
   ) {
     try {
       this.initialize();
-      const now = new Date();
-      const timestamp = now.toLocaleString('de-DE', {
-        dateStyle: 'short',
-        timeStyle: 'short',
-      });
 
       if (!this.client) {
         throw new Error('Custom OpenAI client not initialized');
@@ -154,60 +182,13 @@ class CustomOpenAIService {
       // console.log(`[DEBUG] External API data: ${validatedExternalApiData}`);
       // console.log('######################################################################');
 
-      const response = await this.client.chat.completions.create({
-        model: model,
-        messages: [
-          {
-            role: 'system',
-            content: systemPrompt,
-          },
-          {
-            role: 'user',
-            content: truncatedContent,
-          },
-        ],
+      const { payload, metrics: mappedUsage } = await this.transport.complete({
+        systemPrompt,
+        userContent: truncatedContent,
+        model,
         temperature: config.aiTemperatureAnalysis,
       });
-
-      // Handle response
-      //console.log(`MESSAGE: ${response?.choices?.[0]?.message?.content}`);
-      assertCompletionNotTruncated(
-        response,
-        'Custom OpenAI',
-        'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.'
-      );
-      const message = response?.choices?.[0]?.message;
-      let jsonContent = extractChatMessageContent(message, 'Custom OpenAI');
-      if (!jsonContent) {
-        // Every message keeps the "Invalid API response structure" prefix:
-        // shouldQueueForOcrOnAiError() matches on that phrase, and without it
-        // the document no longer reaches the OCR fallback.
-        const missing = !response
-          ? 'the response is empty'
-          : !response.choices
-            ? 'missing choices'
-            : !response.choices[0]
-              ? 'missing choices[0]'
-              : !response.choices[0].message
-                ? 'missing choices[0].message'
-                : !response.choices[0].message.content
-                  ? 'missing choices[0].message.content'
-                  : `unusable content ${message?.content}`;
-        throw new Error(`Invalid API response structure: ${missing}.`);
-      }
-
-      // Log token usage
-      console.log(`[DEBUG] [${timestamp}] Custom OpenAI request sent`);
-      console.log(
-        `[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}`
-      );
-
-      const usage = response.usage;
-      const mappedUsage = {
-        promptTokens: usage.prompt_tokens,
-        completionTokens: usage.completion_tokens,
-        totalTokens: usage.total_tokens,
-      };
+      let jsonContent = payload;
 
       let parsedResponse;
       try {
diff --git a/services/llm/transports/chatCompletionsTransport.js b/services/llm/transports/chatCompletionsTransport.js
new file mode 100644
index 00000000..c0b759bd
--- /dev/null
+++ b/services/llm/transports/chatCompletionsTransport.js
@@ -0,0 +1,110 @@
+'use strict';
+
+/**
+ * Talks to one chat completions API: OpenAI, Azure OpenAI, or any
+ * OpenAI-compatible server behind a custom base URL.
+ *
+ * A transport owns the client and the request/response mapping, and nothing
+ * else. It throws on a transport failure and on a provider-reported cut-off
+ * (error.code 'ai_response_truncated'); it never shapes an analysis result —
+ * that is analysisResult.js.
+ */
+
+const {
+  assertCompletionNotTruncated,
+  extractChatMessageContent,
+} = require('../../serviceUtils');
+
+/** What an operator can do about an analysis answer that hit the cap. */
+const ANALYSIS_TRUNCATION_REMEDY =
+  'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.';
+
+class ChatCompletionsTransport {
+  /**
+   * @param {Object} options
+   * @param {string} options.label - Provider name used in logs and errors
+   * @param {() => Object|null} options.createClient - Builds the SDK client;
+   *   null when the provider is not configured, which leaves it not ready
+   * @param {(response:Object, message:Object) => string} [options.describeMissingContent]
+   *   The error message for an answer without usable text. It must keep the
+   *   "Invalid API response structure" phrase the OCR fallback matches on.
+   */
+  constructor({ label, createClient, describeMissingContent }) {
+    this.label = label;
+    this.client = null;
+    this._createClient = createClient;
+    this._describeMissingContent =
+      describeMissingContent || (() => 'Invalid API response structure');
+  }
+
+  /** Build the client on first use; a client set from outside is kept. */
+  initialize() {
+    if (!this.client) {
+      this.client = this._createClient() || null;
+    }
+  }
+
+  isReady() {
+    return Boolean(this.client);
+  }
+
+  /**
+   * Send one analysis request.
+   *
+   * @param {Object} request
+   * @param {string} request.systemPrompt
+   * @param {string} request.userContent
+   * @param {string} request.model
+   * @param {number} [request.temperature] - Not sent when undefined
+   * @returns {Promise<{payload:string, metrics:{promptTokens:number, completionTokens:number, totalTokens:number}}>}
+   */
+  async complete({ systemPrompt, userContent, model, temperature }) {
+    const timestamp = new Date().toLocaleString('de-DE', {
+      dateStyle: 'short',
+      timeStyle: 'short',
+    });
+
+    const response = await this.client.chat.completions.create({
+      model: model,
+      messages: [
+        {
+          role: 'system',
+          content: systemPrompt,
+        },
+        {
+          role: 'user',
+          content: userContent,
+        },
+      ],
+      ...(temperature !== undefined && { temperature }),
+    });
+
+    assertCompletionNotTruncated(
+      response,
+      this.label,
+      ANALYSIS_TRUNCATION_REMEDY
+    );
+    const message = response?.choices?.[0]?.message;
+    const payload = extractChatMessageContent(message, this.label);
+    if (!payload) {
+      throw new Error(this._describeMissingContent(response, message));
+    }
+
+    console.log(`[DEBUG] [${timestamp}] ${this.label} request sent`);
+    console.log(
+      `[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}`
+    );
+
+    const usage = response.usage;
+    return {
+      payload,
+      metrics: {
+        promptTokens: usage.prompt_tokens,
+        completionTokens: usage.completion_tokens,
+        totalTokens: usage.total_tokens,
+      },
+    };
+  }
+}
+
+module.exports = { ChatCompletionsTransport, ANALYSIS_TRUNCATION_REMEDY };
diff --git a/services/llm/transports/ollamaGenerateTransport.js b/services/llm/transports/ollamaGenerateTransport.js
new file mode 100644
index 00000000..adc06b70
--- /dev/null
+++ b/services/llm/transports/ollamaGenerateTransport.js
@@ -0,0 +1,278 @@
+'use strict';
+
+/**
+ * Talks to Ollama's native /api/generate.
+ *
+ * Owns what makes Ollama different from the chat completions APIs: a prompt
+ * and a system string instead of messages, a `format` schema for structured
+ * output, num_ctx/num_predict, the `think` switch, one bounded retry for
+ * transient failures, and the done_reason cut-off check. Like every
+ * transport it throws on failure and never shapes an analysis result.
+ *
+ * The axios client is handed in rather than created here: the service builds
+ * it, and tests replace it there.
+ */
+
+const config = require('../../../config/config');
+
+/**
+ * One bounded retry for transient failures (backend EOF, 5xx). The field
+ * report behind this: an MLX runner closed its connection once under memory
+ * pressure; the immediate second attempt succeeded.
+ */
+const MAX_ATTEMPTS = 2;
+
+class OllamaGenerateTransport {
+  /**
+   * @param {Object} options
+   * @param {Object} options.client - axios instance
+   * @param {string} options.apiUrl - Ollama base URL
+   * @param {string} options.model - Default model
+   * @param {number} [options.retryDelayMs=1000] - Pause before the retry
+   */
+  constructor({ client, apiUrl, model, retryDelayMs = 1000 }) {
+    this.label = 'Ollama';
+    this.client = client;
+    this.apiUrl = apiUrl;
+    this.model = model;
+    this.retryDelayMs = retryDelayMs;
+  }
+
+  initialize() {}
+
+  isReady() {
+    return Boolean(this.client);
+  }
+
+  /**
+   * Build request headers, adding bearer auth when an API key is set.
+   * The key is read at request time, so runtime config changes apply
+   * without re-instantiating the singleton.
+   * @returns {Object} Headers object
+   */
+  buildHeaders() {
+    const headers = { 'Content-Type': 'application/json' };
+    const apiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || '';
+    if (apiKey) {
+      headers.Authorization = `Bearer ${apiKey}`;
+    }
+    return headers;
+  }
+
+  /**
+   * Extract the cause an Ollama (or its inference backend) reported in a
+   * non-2xx response body. Ollama wraps backend failures — its runner closing
+   * the connection on an oversized prompt, for example — into
+   * `{ error: 'Post "http://…/v1/completions": EOF' }`; without this the
+   * operator only ever sees "Request failed with status code 500" and cannot
+   * tell an Ollama problem from a model-backend problem.
+   *
+   * @param {Object} axiosError
+   * @returns {string|null}
+   */
+  extractUpstreamErrorMessage(axiosError) {
+    const data = axiosError?.response?.data;
+    if (!data) return null;
+    if (typeof data === 'string') {
+      const trimmed = data.trim();
+      try {
+        return JSON.parse(trimmed)?.error || trimmed || null;
+      } catch {
+        return trimmed || null;
+      }
+    }
+    return data.error || data.message || null;
+  }
+
+  /**
+   * True for failures worth one more attempt: the connection dropped or the
+   * server answered 5xx. A 4xx is a request problem and will fail again.
+   *
+   * @param {Object} axiosError
+   * @returns {boolean}
+   */
+  isTransientFailure(axiosError) {
+    if (!axiosError.response) {
+      // No response at all: connection refused/reset/timeout.
+      return true;
+    }
+    return axiosError.response.status >= 500;
+  }
+
+  /**
+   * Wrap an axios failure with the upstream detail attached.
+   *
+   * @param {Object} axiosError - The axios error being wrapped
+   * @param {number} attempt - Which attempt failed (1-based)
+   * @returns {Error}
+   */
+  wrapRequestError(axiosError, attempt) {
+    const upstream = this.extractUpstreamErrorMessage(axiosError);
+    const status = axiosError?.response?.status;
+    const detail = upstream ? `: ${upstream}` : '';
+    const wrapped = new Error(
+      `Ollama request failed (HTTP ${status || 'no response'}, attempt ${attempt})${detail}`,
+      { cause: axiosError }
+    );
+    // Keep the axios code so the scan-loop classification (OCR fallback,
+    // failure reasons) keeps working unchanged.
+    wrapped.code = axiosError.code;
+    return wrapped;
+  }
+
+  /**
+   * Send one analysis request.
+   *
+   * @param {Object} request
+   * @param {string} request.systemPrompt - Sent as `system`
+   * @param {string} request.userContent - Sent as `prompt`
+   * @param {string} [request.model] - Defaults to the transport's model
+   * @param {number} request.temperature
+   * @param {number} request.maxResponseTokens - Sent as num_predict
+   * @param {number} request.numCtx - Context window to request
+   * @param {Object|null} request.schema - Sent as `format`
+   * @returns {Promise<{payload:string|Object, metrics:Object, raw:Object}>}
+   */
+  async complete({
+    systemPrompt,
+    userContent,
+    model,
+    temperature,
+    maxResponseTokens,
+    numCtx,
+    schema,
+  }) {
+    const numPredict = maxResponseTokens;
+    const requestBody = {
+      model: model || this.model,
+      prompt: userContent,
+      system: systemPrompt,
+      stream: false,
+      format: schema,
+      options: {
+        temperature,
+        top_p: 0.9,
+        repeat_penalty: 1.1,
+        top_k: 7,
+        num_predict: numPredict,
+        num_ctx: numCtx,
+      },
+    };
+
+    // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
+    // unless explicitly enabled via OLLAMA_THINK=true
+    if (!config.ollama.think) {
+      requestBody.think = false;
+    }
+
+    let response;
+    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
+      try {
+        response = await this.client.post(
+          `${this.apiUrl}/api/generate`,
+          requestBody,
+          {
+            headers: this.buildHeaders(),
+          }
+        );
+        break;
+      } catch (error) {
+        const wrapped = this.wrapRequestError(error, attempt);
+        console.error(`[ERROR] ${wrapped.message}`);
+        if (attempt >= MAX_ATTEMPTS || !this.isTransientFailure(error)) {
+          throw wrapped;
+        }
+        await new Promise((resolve) => setTimeout(resolve, this.retryDelayMs));
+      }
+    }
+
+    if (!response?.data) {
+      throw new Error('Invalid response from Ollama API');
+    }
+
+    this.assertNotTruncated(response.data, numPredict, numCtx);
+    return {
+      payload: response.data.response,
+      metrics: this.extractMetrics(response.data),
+      raw: response.data,
+    };
+  }
+
+  /**
+   * A generation Ollama had to cut short comes back with done_reason "length".
+   * What it returns is a prefix, and for structured output that means JSON
+   * without its closing braces — which no amount of sanitizing recovers, so
+   * there is nothing to do with it but say so.
+   *
+   * Two different limits produce the same done_reason and want opposite fixes:
+   * num_predict is the response budget, but the context window can leave less
+   * room than that, and then raising Response Tokens changes nothing. The
+   * message names whichever one actually bit.
+   *
+   * @param {Object} responseData - Ollama /api/generate response
+   * @param {number} numPredict - response token limit that was sent
+   * @param {number} numCtx - context window that was sent
+   * @param {Object} [extra] - what the caller salvaged from the cut-off answer
+   * @param {string} [extra.partialText] - the text that did arrive, reasoning
+   *   stripped; put on the error as `partialText` the way the OpenAI-compatible
+   *   providers do, so a caller can keep it instead of asking twice
+   */
+  assertNotTruncated(responseData, numPredict, numCtx, extra) {
+    const evalCount = Number(responseData?.eval_count) || 0;
+    const promptEvalCount = Number(responseData?.prompt_eval_count) || 0;
+
+    // done_reason is the direct signal. Reaching the limit exactly is the
+    // fallback for an Ollama old enough not to send one.
+    const stoppedOnLength =
+      responseData?.done_reason === 'length' ||
+      (!responseData?.done_reason && numPredict > 0 && evalCount >= numPredict);
+
+    if (!stoppedOnLength) {
+      return;
+    }
+
+    /* Which limit bit is read off the count itself rather than guessed from
+       the context arithmetic: reaching num_predict exactly is the response
+       budget, stopping short of it means the context filled up first. The
+       arithmetic route was tried and got it backwards — the window is sized
+       from an estimate of the prompt, so comparing it against the real
+       prompt_eval_count blames the context for a run that generated its full
+       budget. */
+    const error =
+      evalCount >= numPredict
+        ? new Error(
+            `Ollama stopped generating after ${evalCount} tokens, which is the configured response limit. Raise Response Tokens (RESPONSE_TOKENS) above ${numPredict}.`
+          )
+        : new Error(
+            `Ollama stopped generating after ${evalCount} of ${numPredict} response tokens: the ${numCtx}-token context window was full, ${promptEvalCount} of it prompt. Raise Token Limit (TOKEN_LIMIT) or shorten the document.`
+          );
+
+    /* Carried as a code rather than matched out of the message later. The
+       phrase-matching in serviceUtils classifies errors that arrive from
+       elsewhere and have no better handle; this one is raised right here, so
+       rewording it should not silently reclassify the failure. */
+    error.code = 'ai_response_truncated';
+    if (typeof extra?.partialText === 'string') {
+      error.partialText = extra.partialText;
+    }
+    throw error;
+  }
+
+  /**
+   * Token usage of an /api/generate response.
+   * @param {Object} responseData - Ollama API response data
+   * @returns {{promptTokens:number, completionTokens:number, totalTokens:number}}
+   */
+  extractMetrics(responseData) {
+    const promptTokens = Number(responseData?.prompt_eval_count) || 0;
+    const completionTokens = Number(responseData?.eval_count) || 0;
+
+    return {
+      promptTokens,
+      completionTokens,
+      totalTokens: promptTokens + completionTokens,
+    };
+  }
+}
+
+module.exports = { OllamaGenerateTransport };
diff --git a/services/ollamaService.js b/services/ollamaService.js
index 0e3a1aa1..bd42082a 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -17,6 +17,9 @@ const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
 const { parseAnalysisResponse } = require('./llm/responseParser');
 const charBudget = require('./llm/tokenBudget/charBudget');
+const {
+  OllamaGenerateTransport,
+} = require('./llm/transports/ollamaGenerateTransport');
 const {
   buildSuccess,
   buildFailure,
@@ -32,10 +35,12 @@ class OllamaService {
    * Initialize the Ollama service
    */
   constructor() {
-    this.apiUrl = config.ollama.apiUrl;
-    this.model = config.ollama.model;
-    this.client = axios.create({
-      timeout: 1800000, // 30 minutes timeout
+    this.transport = new OllamaGenerateTransport({
+      client: axios.create({
+        timeout: 1800000, // 30 minutes timeout
+      }),
+      apiUrl: config.ollama.apiUrl,
+      model: config.ollama.model,
     });
 
     // JSON schema for document analysis output
@@ -67,6 +72,31 @@ class OllamaService {
     };
   }
 
+  /** The axios client. It lives on the transport; tests replace it here. */
+  get client() {
+    return this.transport.client;
+  }
+
+  set client(client) {
+    this.transport.client = client;
+  }
+
+  get apiUrl() {
+    return this.transport.apiUrl;
+  }
+
+  set apiUrl(apiUrl) {
+    this.transport.apiUrl = apiUrl;
+  }
+
+  get model() {
+    return this.transport.model;
+  }
+
+  set model(model) {
+    this.transport.model = model;
+  }
+
   /**
    * Analyzes a document to extract metadata and tags using an AI model.
    * @param {string} content - The content of the document to analyze.
@@ -378,208 +408,38 @@ class OllamaService {
     return numCtx;
   }
 
-  /**
-   * Build request headers, adding bearer auth when an API key is set.
-   * The key is read at request time, so runtime config changes apply
-   * without re-instantiating the singleton.
-   * @returns {Object} Headers object
-   */
+  /** Request headers, with bearer auth when an API key is set. */
   _buildRequestHeaders() {
-    const headers = { 'Content-Type': 'application/json' };
-    const apiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || '';
-    if (apiKey) {
-      headers.Authorization = `Bearer ${apiKey}`;
-    }
-    return headers;
+    return this.transport.buildHeaders();
   }
 
   /**
-   * Call Ollama API
+   * Send the analysis request.
    * @param {string} prompt - Prompt text
    * @param {string} systemPrompt - System prompt
    * @param {number} numCtx - Context window size
    * @param {Object} schema - Response schema
-   * @returns {Object} Ollama API response
-   */
-  /**
-   * Extract the cause an Ollama (or its inference backend) reported in a
-   * non-2xx response body. Ollama wraps backend failures — its runner closing
-   * the connection on an oversized prompt, for example — into
-   * `{ error: 'Post "http://…/v1/completions": EOF' }`; without this the
-   * operator only ever sees "Request failed with status code 500" and cannot
-   * tell an Ollama problem from a model-backend problem.
-   *
-   * @param {Object} axiosError
-   * @returns {string|null}
-   */
-  _extractUpstreamErrorMessage(axiosError) {
-    const data = axiosError?.response?.data;
-    if (!data) return null;
-    if (typeof data === 'string') {
-      const trimmed = data.trim();
-      try {
-        return JSON.parse(trimmed)?.error || trimmed || null;
-      } catch {
-        return trimmed || null;
-      }
-    }
-    return data.error || data.message || null;
-  }
-
-  /**
-   * True for failures worth one more attempt: the connection dropped or the
-   * server answered 5xx. A 4xx is a request problem and will fail again.
-   *
-   * @param {Object} axiosError
-   * @returns {boolean}
-   */
-  _isTransientOllamaFailure(axiosError) {
-    if (!axiosError.response) {
-      // No response at all: connection refused/reset/timeout.
-      return true;
-    }
-    return axiosError.response.status >= 500;
-  }
-
-  /**
-   * Wrap an axios failure with the upstream detail attached.
-   *
-   * @param {Object} axiosError - The axios error being wrapped
-   * @param {number} attempt - Which attempt failed (1-based)
-   * @returns {Error}
+   * @returns {Promise<Object>} The raw /api/generate response data
    */
-  _wrapOllamaRequestError(axiosError, attempt) {
-    const upstream = this._extractUpstreamErrorMessage(axiosError);
-    const status = axiosError?.response?.status;
-    const detail = upstream ? `: ${upstream}` : '';
-    const wrapped = new Error(
-      `Ollama request failed (HTTP ${status || 'no response'}, attempt ${attempt})${detail}`,
-      { cause: axiosError }
-    );
-    // Keep the axios code so the scan-loop classification (OCR fallback,
-    // failure reasons) keeps working unchanged.
-    wrapped.code = axiosError.code;
-    return wrapped;
-  }
-
   async _callOllamaAPI(prompt, systemPrompt, numCtx, schema) {
-    // The same number _calculateNumCtx() already reserved room for. It used to
-    // be a hardcoded 256 while the reservation followed the setting, so the
-    // context held space for an answer the model was then forbidden to write.
-    const numPredict = Number(config.responseTokens);
-    const requestBody = {
-      model: this.model,
-      prompt: prompt,
-      system: systemPrompt,
-      stream: false,
-      format: schema,
-      options: {
-        temperature: config.aiTemperatureAnalysis,
-        top_p: 0.9,
-        repeat_penalty: 1.1,
-        top_k: 7,
-        num_predict: numPredict,
-        num_ctx: numCtx,
-      },
-    };
-
-    // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
-    // unless explicitly enabled via OLLAMA_THINK=true
-    if (!config.ollama.think) {
-      requestBody.think = false;
-    }
-
-    // One bounded retry for transient failures (backend EOF, 5xx). The
-    // field report behind this: an MLX runner closed its connection once
-    // under memory pressure; the immediate second attempt succeeded.
-    const MAX_ATTEMPTS = 2;
-    let response;
-    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
-      try {
-        response = await this.client.post(
-          `${this.apiUrl}/api/generate`,
-          requestBody,
-          {
-            headers: this._buildRequestHeaders(),
-          }
-        );
-        break;
-      } catch (error) {
-        const wrapped = this._wrapOllamaRequestError(error, attempt);
-        console.error(`[ERROR] ${wrapped.message}`);
-        if (attempt >= MAX_ATTEMPTS || !this._isTransientOllamaFailure(error)) {
-          throw wrapped;
-        }
-        await new Promise((resolve) => setTimeout(resolve, 1000));
-      }
-    }
-
-    if (!response?.data) {
-      throw new Error('Invalid response from Ollama API');
-    }
-
-    this._assertNotTruncated(response.data, numPredict, numCtx);
-    return response.data;
+    const { raw } = await this.transport.complete({
+      systemPrompt,
+      userContent: prompt,
+      temperature: config.aiTemperatureAnalysis,
+      // The same number _calculateNumCtx() already reserved room for. It used
+      // to be a hardcoded 256 while the reservation followed the setting, so
+      // the context held space for an answer the model was then forbidden to
+      // write.
+      maxResponseTokens: Number(config.responseTokens),
+      numCtx,
+      schema,
+    });
+    return raw;
   }
 
-  /**
-   * A generation Ollama had to cut short comes back with done_reason "length".
-   * What it returns is a prefix, and for structured output that means JSON
-   * without its closing braces — which no amount of sanitizing recovers, so
-   * there is nothing to do with it but say so.
-   *
-   * Two different limits produce the same done_reason and want opposite fixes:
-   * num_predict is the response budget, but the context window can leave less
-   * room than that, and then raising Response Tokens changes nothing. The
-   * message names whichever one actually bit.
-   *
-   * @param {Object} responseData - Ollama /api/generate response
-   * @param {number} numPredict - response token limit that was sent
-   * @param {number} numCtx - context window that was sent
-   * @param {Object} [extra] - what the caller salvaged from the cut-off answer
-   * @param {string} [extra.partialText] - the text that did arrive, reasoning
-   *   stripped; put on the error as `partialText` the way the OpenAI-compatible
-   *   providers do, so a caller can keep it instead of asking twice
-   */
+  /** See OllamaGenerateTransport.assertNotTruncated(). */
   _assertNotTruncated(responseData, numPredict, numCtx, extra) {
-    const evalCount = Number(responseData?.eval_count) || 0;
-    const promptEvalCount = Number(responseData?.prompt_eval_count) || 0;
-
-    // done_reason is the direct signal. Reaching the limit exactly is the
-    // fallback for an Ollama old enough not to send one.
-    const stoppedOnLength =
-      responseData?.done_reason === 'length' ||
-      (!responseData?.done_reason && numPredict > 0 && evalCount >= numPredict);
-
-    if (!stoppedOnLength) {
-      return;
-    }
-
-    /* Which limit bit is read off the count itself rather than guessed from
-       the context arithmetic: reaching num_predict exactly is the response
-       budget, stopping short of it means the context filled up first. The
-       arithmetic route was tried and got it backwards — _calculateNumCtx sizes
-       the window from an estimate of the prompt (length / 4), so comparing it
-       against the real prompt_eval_count blames the context for a run that
-       generated its full budget. */
-    const error =
-      evalCount >= numPredict
-        ? new Error(
-            `Ollama stopped generating after ${evalCount} tokens, which is the configured response limit. Raise Response Tokens (RESPONSE_TOKENS) above ${numPredict}.`
-          )
-        : new Error(
-            `Ollama stopped generating after ${evalCount} of ${numPredict} response tokens: the ${numCtx}-token context window was full, ${promptEvalCount} of it prompt. Raise Token Limit (TOKEN_LIMIT) or shorten the document.`
-          );
-
-    /* Carried as a code rather than matched out of the message later. The
-       phrase-matching in serviceUtils classifies errors that arrive from
-       elsewhere and have no better handle; this one is raised right here, so
-       rewording it should not silently reclassify the failure. */
-    error.code = 'ai_response_truncated';
-    if (typeof extra?.partialText === 'string') {
-      error.partialText = extra.partialText;
-    }
-    throw error;
+    this.transport.assertNotTruncated(responseData, numPredict, numCtx, extra);
   }
 
   /**
@@ -614,14 +474,7 @@ class OllamaService {
    * @returns {{promptTokens:number, completionTokens:number, totalTokens:number}}
    */
   _extractOllamaMetrics(responseData) {
-    const promptTokens = Number(responseData?.prompt_eval_count) || 0;
-    const completionTokens = Number(responseData?.eval_count) || 0;
-
-    return {
-      promptTokens,
-      completionTokens,
-      totalTokens: promptTokens + completionTokens,
-    };
+    return this.transport.extractMetrics(responseData);
   }
 
   /**
diff --git a/services/openaiService.js b/services/openaiService.js
index b30f90e5..e128c580 100644
--- a/services/openaiService.js
+++ b/services/openaiService.js
@@ -26,6 +26,9 @@ const path = require('path');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
 const tiktokenBudget = require('./llm/tokenBudget/tiktokenBudget');
+const {
+  ChatCompletionsTransport,
+} = require('./llm/transports/chatCompletionsTransport');
 const {
   buildSuccess,
   buildFailure,
@@ -48,29 +51,44 @@ const TRUNCATION_REMEDY =
 
 class OpenAIService {
   constructor() {
-    this.client = null;
+    this.transport = new ChatCompletionsTransport({
+      label: 'OpenAI',
+      createClient: () => {
+        if (config.aiProvider === 'ollama') {
+          return new OpenAI({
+            baseURL: config.ollama.apiUrl + '/v1',
+            // Ollama ignores the key unless the endpoint enforces auth (e.g. a
+            // reverse proxy); the OpenAI SDK requires a non-empty value.
+            apiKey: config.ollama.apiKey || 'ollama',
+          });
+        }
+        if (config.aiProvider === 'custom') {
+          return new OpenAI({
+            baseURL: config.custom.apiUrl,
+            apiKey: config.custom.apiKey,
+          });
+        }
+        if (config.aiProvider === 'openai' && config.openai.apiKey) {
+          return new OpenAI({
+            apiKey: config.openai.apiKey,
+          });
+        }
+        return null;
+      },
+    });
+  }
+
+  /** The SDK client. It lives on the transport; tests replace it here. */
+  get client() {
+    return this.transport.client;
+  }
+
+  set client(client) {
+    this.transport.client = client;
   }
 
   initialize() {
-    if (!this.client && config.aiProvider === 'ollama') {
-      this.client = new OpenAI({
-        baseURL: config.ollama.apiUrl + '/v1',
-        // Ollama ignores the key unless the endpoint enforces auth (e.g. a
-        // reverse proxy); the OpenAI SDK requires a non-empty value.
-        apiKey: config.ollama.apiKey || 'ollama',
-      });
-    } else if (!this.client && config.aiProvider === 'custom') {
-      this.client = new OpenAI({
-        baseURL: config.custom.apiUrl,
-        apiKey: config.custom.apiKey,
-      });
-    } else if (!this.client && config.aiProvider === 'openai') {
-      if (!this.client && config.openai.apiKey) {
-        this.client = new OpenAI({
-          apiKey: config.openai.apiKey,
-        });
-      }
-    }
+    this.transport.initialize();
   }
 
   async analyzeDocument(
@@ -84,11 +102,6 @@ class OpenAIService {
   ) {
     try {
       this.initialize();
-      const now = new Date();
-      const timestamp = now.toLocaleString('de-DE', {
-        dateStyle: 'short',
-        timeStyle: 'short',
-      });
 
       if (!this.client) {
         throw new Error('OpenAI client not initialized');
@@ -150,45 +163,15 @@ class OpenAIService {
 
       await writePromptToFile(systemPrompt, truncatedContent);
 
-      const response = await this.client.chat.completions.create({
-        model: model,
-        messages: [
-          {
-            role: 'system',
-            content: systemPrompt,
-          },
-          {
-            role: 'user',
-            content: truncatedContent,
-          },
-        ],
-        ...(model !== 'o3-mini' && {
-          temperature: config.aiTemperatureAnalysis,
-        }),
+      const { payload, metrics: mappedUsage } = await this.transport.complete({
+        systemPrompt,
+        userContent: truncatedContent,
+        model,
+        // o3-mini rejects the parameter.
+        temperature:
+          model !== 'o3-mini' ? config.aiTemperatureAnalysis : undefined,
       });
-
-      assertCompletionNotTruncated(
-        response,
-        'OpenAI',
-        'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.'
-      );
-      const message = response?.choices?.[0]?.message;
-      let jsonContent = extractChatMessageContent(message, 'OpenAI');
-      if (!jsonContent) {
-        throw new Error('Invalid API response structure');
-      }
-
-      console.log(`[DEBUG] [${timestamp}] OpenAI request sent`);
-      console.log(
-        `[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}`
-      );
-
-      const usage = response.usage;
-      const mappedUsage = {
-        promptTokens: usage.prompt_tokens,
-        completionTokens: usage.completion_tokens,
-        totalTokens: usage.total_tokens,
-      };
+      let jsonContent = payload;
 
       let parsedResponse;
       try {
diff --git a/tests/test-llm-transport-request-mapping.js b/tests/test-llm-transport-request-mapping.js
new file mode 100644
index 00000000..8daebde9
--- /dev/null
+++ b/tests/test-llm-transport-request-mapping.js
@@ -0,0 +1,580 @@
+/**
+ * The transports: what each API family is sent, and what comes back.
+ *
+ * Step 5 of the LLM service refactoring (services/ManualServiceRefactoring.md).
+ * A transport owns exactly one thing — how to talk to one API family — and
+ * never shapes an analysis result:
+ *
+ *   services/llm/transports/chatCompletionsTransport.js  openai, azure, custom
+ *   services/llm/transports/ollamaGenerateTransport.js   Ollama /api/generate
+ *
+ * The first half pins each transport on its own with a fake client. The
+ * second half checks that every provider's analysis request goes through its
+ * transport and is mapped the way it was before: OpenAI omits the
+ * temperature for o3-mini, Azure still hardcodes 0.3 (Finding 7a, fixed in
+ * step 6), Ollama sends format, num_ctx, num_predict and think:false unless
+ * OLLAMA_THINK is set.
+ */
+
+const assert = require('assert');
+const fs = require('fs').promises;
+
+process.env.AI_PROVIDER = 'openai';
+process.env.OPENAI_API_KEY = 'test-key';
+process.env.OPENAI_MODEL = 'gpt-4';
+process.env.AZURE_API_KEY = 'test-key';
+process.env.AZURE_ENDPOINT = 'https://example.invalid';
+process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment';
+process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1';
+process.env.CUSTOM_API_KEY = 'test-key';
+process.env.CUSTOM_MODEL = 'test-model';
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+process.env.USE_PROMPT_TAGS = 'no';
+delete process.env.CUSTOM_FIELDS;
+delete process.env.CONTENT_MAX_LENGTH;
+delete process.env.OLLAMA_API_KEY;
+
+const config = require('../config/config');
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('../services/thumbnailCachePaths');
+const {
+  ChatCompletionsTransport,
+} = require('../services/llm/transports/chatCompletionsTransport');
+const {
+  OllamaGenerateTransport,
+} = require('../services/llm/transports/ollamaGenerateTransport');
+const openaiService = require('../services/openaiService');
+const azureService = require('../services/azureService');
+const customService = require('../services/customService');
+const ollamaService = require('../services/ollamaService');
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+/** Own id so the seeded thumbnail cannot collide with a real cached document. */
+const DOCUMENT_ID = 'testLlmTransportMapping';
+
+const ANSWER = JSON.stringify({
+  title: 'Telekom invoice July',
+  correspondent: 'Telekom Deutschland GmbH',
+  tags: ['invoice'],
+  document_type: 'Rechnung',
+  document_date: '2026-09-03',
+  language: 'de',
+});
+
+function completion(content = ANSWER, finishReason = 'stop') {
+  return {
+    choices: [{ message: { content }, finish_reason: finishReason }],
+    usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 },
+  };
+}
+
+function fakeChatClient(requests, respond = async () => completion()) {
+  return {
+    chat: {
+      completions: {
+        create: async (request) => {
+          requests.push(request);
+          return respond(request);
+        },
+      },
+    },
+    models: { list: async () => ({ data: [] }) },
+  };
+}
+
+function ollamaData(overrides = {}) {
+  return {
+    response: ANSWER,
+    done_reason: 'stop',
+    eval_count: 3,
+    prompt_eval_count: 9,
+    ...overrides,
+  };
+}
+
+/** An axios-shaped client; `answers` are returned or thrown in turn. */
+function fakeOllamaClient(answers, posts) {
+  let next = 0;
+  return {
+    post: async (url, body, requestConfig) => {
+      posts.push({ url, body, requestConfig });
+      const answer = answers[Math.min(next, answers.length - 1)];
+      next += 1;
+      if (answer instanceof Error) throw answer;
+      return answer;
+    },
+    get: async () => ({ status: 200, data: { models: [] } }),
+  };
+}
+
+function axiosFailure(status, data) {
+  const error = new Error(
+    status ? `Request failed with status code ${status}` : 'socket hang up'
+  );
+  error.code = status ? 'ERR_BAD_RESPONSE' : 'ECONNRESET';
+  if (status) error.response = { status, data };
+  return error;
+}
+
+const ANALYSIS_REQUEST = {
+  systemPrompt: 'You analyse documents.',
+  userContent: 'Invoice from Telekom.',
+  model: 'gpt-4',
+  temperature: 0.2,
+};
+
+(async () => {
+  console.log('\n=== LLM transport request mapping ===');
+
+  /* --- ChatCompletionsTransport -------------------------------------------- */
+
+  await check('chat: the client is created once, on first use', () => {
+    let created = 0;
+    const transport = new ChatCompletionsTransport({
+      label: 'Fake',
+      createClient: () => {
+        created += 1;
+        return fakeChatClient([]);
+      },
+    });
+    assert.strictEqual(transport.isReady(), false);
+    transport.initialize();
+    transport.initialize();
+    assert.strictEqual(created, 1);
+    assert.strictEqual(transport.isReady(), true);
+  });
+
+  await check('chat: a factory without credentials leaves it not ready', () => {
+    const transport = new ChatCompletionsTransport({
+      label: 'Fake',
+      createClient: () => null,
+    });
+    transport.initialize();
+    assert.strictEqual(transport.isReady(), false);
+    assert.strictEqual(transport.client, null);
+  });
+
+  await check('chat: sends a system and a user message', async () => {
+    const requests = [];
+    const transport = new ChatCompletionsTransport({
+      label: 'Fake',
+      createClient: () => fakeChatClient(requests),
+    });
+    transport.initialize();
+    await transport.complete(ANALYSIS_REQUEST);
+    assert.deepStrictEqual(requests, [
+      {
+        model: 'gpt-4',
+        messages: [
+          { role: 'system', content: 'You analyse documents.' },
+          { role: 'user', content: 'Invoice from Telekom.' },
+        ],
+        temperature: 0.2,
+      },
+    ]);
+  });
+
+  await check('chat: an undefined temperature is not sent at all', async () => {
+    const requests = [];
+    const transport = new ChatCompletionsTransport({
+      label: 'Fake',
+      createClient: () => fakeChatClient(requests),
+    });
+    transport.initialize();
+    await transport.complete({ ...ANALYSIS_REQUEST, temperature: undefined });
+    assert.ok(!('temperature' in requests[0]), JSON.stringify(requests[0]));
+  });
+
+  await check('chat: returns the text and the usage as metrics', async () => {
+    const transport = new ChatCompletionsTransport({
+      label: 'Fake',
+      createClient: () => fakeChatClient([]),
+    });
+    transport.initialize();
+    const result = await transport.complete(ANALYSIS_REQUEST);
+    assert.deepStrictEqual(result, {
+      payload: ANSWER,
+      metrics: { promptTokens: 9, completionTokens: 3, totalTokens: 12 },
+    });
+  });
+
+  await check('chat: a cut-off answer is ai_response_truncated', async () => {
+    const transport = new ChatCompletionsTransport({
+      label: 'Fake',
+      createClient: () =>
+        fakeChatClient([], async () => completion('{"tags": [', 'length')),
+    });
+    transport.initialize();
+    await assert.rejects(
+      () => transport.complete(ANALYSIS_REQUEST),
+      (error) => error.code === 'ai_response_truncated'
+    );
+  });
+
+  await check(
+    'chat: no content is "Invalid API response structure"',
+    async () => {
+      const transport = new ChatCompletionsTransport({
+        label: 'Fake',
+        createClient: () => fakeChatClient([], async () => ({ choices: [] })),
+      });
+      transport.initialize();
+      await assert.rejects(
+        () => transport.complete(ANALYSIS_REQUEST),
+        (error) => error.message === 'Invalid API response structure'
+      );
+    }
+  );
+
+  await check(
+    'chat: a provider may describe the missing content itself',
+    async () => {
+      const transport = new ChatCompletionsTransport({
+        label: 'Fake',
+        createClient: () => fakeChatClient([], async () => ({ choices: [] })),
+        describeMissingContent: (response) =>
+          `Invalid API response structure: ${response.choices.length} choices.`,
+      });
+      transport.initialize();
+      await assert.rejects(
+        () => transport.complete(ANALYSIS_REQUEST),
+        (error) =>
+          error.message === 'Invalid API response structure: 0 choices.'
+      );
+    }
+  );
+
+  /* --- OllamaGenerateTransport --------------------------------------------- */
+
+  const ollamaTransport = (answers, posts, options = {}) =>
+    new OllamaGenerateTransport({
+      client: fakeOllamaClient(answers, posts),
+      apiUrl: 'http://ollama.invalid:11434',
+      model: 'llama3',
+      retryDelayMs: 1,
+      ...options,
+    });
+
+  const OLLAMA_REQUEST = {
+    systemPrompt: 'system',
+    userContent: 'prompt',
+    temperature: 0.2,
+    maxResponseTokens: 700,
+    numCtx: 4096,
+    schema: { type: 'object' },
+  };
+
+  await check('ollama: maps the request onto /api/generate', async () => {
+    const posts = [];
+    const previousThink = config.ollama.think;
+    config.ollama.think = false;
+    try {
+      await ollamaTransport([{ data: ollamaData() }], posts).complete(
+        OLLAMA_REQUEST
+      );
+    } finally {
+      config.ollama.think = previousThink;
+    }
+    assert.strictEqual(posts.length, 1);
+    assert.strictEqual(
+      posts[0].url,
+      'http://ollama.invalid:11434/api/generate'
+    );
+    assert.deepStrictEqual(posts[0].body, {
+      model: 'llama3',
+      prompt: 'prompt',
+      system: 'system',
+      stream: false,
+      format: { type: 'object' },
+      options: {
+        temperature: 0.2,
+        top_p: 0.9,
+        repeat_penalty: 1.1,
+        top_k: 7,
+        num_predict: 700,
+        num_ctx: 4096,
+      },
+      think: false,
+    });
+    assert.deepStrictEqual(posts[0].requestConfig, {
+      headers: { 'Content-Type': 'application/json' },
+    });
+  });
+
+  await check('ollama: OLLAMA_THINK=true sends no think field', async () => {
+    const posts = [];
+    const previousThink = config.ollama.think;
+    config.ollama.think = true;
+    try {
+      await ollamaTransport([{ data: ollamaData() }], posts).complete(
+        OLLAMA_REQUEST
+      );
+    } finally {
+      config.ollama.think = previousThink;
+    }
+    assert.ok(!('think' in posts[0].body), JSON.stringify(posts[0].body));
+  });
+
+  await check('ollama: an API key becomes a bearer header', async () => {
+    const posts = [];
+    process.env.OLLAMA_API_KEY = 'secret';
+    try {
+      await ollamaTransport([{ data: ollamaData() }], posts).complete(
+        OLLAMA_REQUEST
+      );
+    } finally {
+      delete process.env.OLLAMA_API_KEY;
+    }
+    assert.strictEqual(
+      posts[0].requestConfig.headers.Authorization,
+      'Bearer secret'
+    );
+  });
+
+  await check(
+    'ollama: returns the answer, its metrics and the raw data',
+    async () => {
+      const data = ollamaData();
+      const result = await ollamaTransport([{ data }], []).complete(
+        OLLAMA_REQUEST
+      );
+      assert.strictEqual(result.payload, ANSWER);
+      assert.deepStrictEqual(result.metrics, {
+        promptTokens: 9,
+        completionTokens: 3,
+        totalTokens: 12,
+      });
+      assert.strictEqual(result.raw, data);
+    }
+  );
+
+  await check(
+    'ollama: a 5xx is tried once more, and the second answer wins',
+    async () => {
+      const posts = [];
+      const result = await ollamaTransport(
+        [axiosFailure(500, { error: 'EOF' }), { data: ollamaData() }],
+        posts
+      ).complete(OLLAMA_REQUEST);
+      assert.strictEqual(posts.length, 2);
+      assert.strictEqual(result.payload, ANSWER);
+    }
+  );
+
+  await check(
+    'ollama: a 4xx is not retried and names the upstream cause',
+    async () => {
+      const posts = [];
+      await assert.rejects(
+        () =>
+          ollamaTransport(
+            [axiosFailure(404, { error: 'model "llama3" not found' })],
+            posts
+          ).complete(OLLAMA_REQUEST),
+        (error) =>
+          error.message ===
+            'Ollama request failed (HTTP 404, attempt 1): model "llama3" not found' &&
+          error.code === 'ERR_BAD_RESPONSE'
+      );
+      assert.strictEqual(posts.length, 1);
+    }
+  );
+
+  await check(
+    'ollama: no response at all is retried, then reported',
+    async () => {
+      const posts = [];
+      await assert.rejects(
+        () =>
+          ollamaTransport([axiosFailure(null)], posts).complete(OLLAMA_REQUEST),
+        (error) =>
+          error.message ===
+            'Ollama request failed (HTTP no response, attempt 2)' &&
+          error.code === 'ECONNRESET'
+      );
+      assert.strictEqual(posts.length, 2);
+    }
+  );
+
+  await check(
+    'ollama: done_reason "length" is ai_response_truncated',
+    async () => {
+      await assert.rejects(
+        () =>
+          ollamaTransport(
+            [{ data: ollamaData({ done_reason: 'length', eval_count: 700 }) }],
+            []
+          ).complete(OLLAMA_REQUEST),
+        (error) =>
+          error.code === 'ai_response_truncated' &&
+          /Raise Response Tokens \(RESPONSE_TOKENS\) above 700/.test(
+            error.message
+          )
+      );
+    }
+  );
+
+  await check(
+    'ollama: an empty HTTP body is "Invalid response from Ollama API"',
+    async () => {
+      await assert.rejects(
+        () => ollamaTransport([{ data: null }], []).complete(OLLAMA_REQUEST),
+        (error) => error.message === 'Invalid response from Ollama API'
+      );
+    }
+  );
+
+  /* --- every provider's analysis goes through its transport ---------------- */
+
+  await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+  await fs.writeFile(getThumbnailCachePath(DOCUMENT_ID), 'thumbnail');
+  Object.assign(config, { tokenLimit: 128000, responseTokens: 1000 });
+
+  const analyse = (service) =>
+    service.analyzeDocument('Invoice from Telekom.', [], [], [], DOCUMENT_ID);
+
+  for (const [label, service, transportLabel] of [
+    ['OpenAI', openaiService, 'OpenAI'],
+    ['Azure', azureService, 'AzureOpenAI'],
+    ['Custom', customService, 'Custom OpenAI'],
+  ]) {
+    await check(`${label}: owns a chat completions transport`, () => {
+      assert.ok(service.transport instanceof ChatCompletionsTransport);
+      assert.strictEqual(service.transport.label, transportLabel);
+    });
+
+    await check(`${label}: the client it is given is the transport's`, () => {
+      const client = fakeChatClient([]);
+      service.client = client;
+      assert.strictEqual(service.transport.client, client);
+      assert.strictEqual(service.client, client);
+    });
+  }
+
+  await check('OpenAI: analysis sends AI_TEMPERATURE_ANALYSIS', async () => {
+    const requests = [];
+    openaiService.client = fakeChatClient(requests);
+    const analysis = await analyse(openaiService);
+    assert.strictEqual(analysis.error, undefined, analysis.error);
+    assert.strictEqual(requests[0].model, 'gpt-4');
+    assert.strictEqual(requests[0].temperature, config.aiTemperatureAnalysis);
+    assert.deepStrictEqual(
+      requests[0].messages.map((m) => m.role),
+      ['system', 'user']
+    );
+  });
+
+  await check(
+    'OpenAI: analysis with o3-mini sends no temperature',
+    async () => {
+      const requests = [];
+      openaiService.client = fakeChatClient(requests);
+      process.env.OPENAI_MODEL = 'o3-mini';
+      try {
+        await analyse(openaiService);
+      } finally {
+        process.env.OPENAI_MODEL = 'gpt-4';
+      }
+      assert.strictEqual(requests[0].model, 'o3-mini');
+      assert.ok(!('temperature' in requests[0]), JSON.stringify(requests[0]));
+    }
+  );
+
+  await check(
+    'Azure: analysis still sends 0.3 (Finding 7a, step 6)',
+    async () => {
+      const requests = [];
+      azureService.client = fakeChatClient(requests);
+      await analyse(azureService);
+      assert.strictEqual(requests[0].model, 'test-deployment');
+      assert.strictEqual(requests[0].temperature, 0.3);
+    }
+  );
+
+  await check('Custom: analysis sends AI_TEMPERATURE_ANALYSIS', async () => {
+    const requests = [];
+    customService.client = fakeChatClient(requests);
+    await analyse(customService);
+    assert.strictEqual(requests[0].model, config.custom.model);
+    assert.strictEqual(requests[0].temperature, config.aiTemperatureAnalysis);
+  });
+
+  await check(
+    'Custom: keeps its detailed missing-content message',
+    async () => {
+      customService.client = fakeChatClient([], async () => ({ choices: [] }));
+      const analysis = await analyse(customService);
+      assert.strictEqual(
+        analysis.error,
+        'Invalid API response structure: missing choices[0].'
+      );
+    }
+  );
+
+  await check('Ollama: owns a generate transport', () => {
+    assert.ok(ollamaService.transport instanceof OllamaGenerateTransport);
+  });
+
+  await check("Ollama: client, apiUrl and model are the transport's", () => {
+    const client = fakeOllamaClient([{ data: ollamaData() }], []);
+    const previous = {
+      apiUrl: ollamaService.apiUrl,
+      model: ollamaService.model,
+    };
+    ollamaService.client = client;
+    ollamaService.apiUrl = 'http://elsewhere.invalid';
+    ollamaService.model = 'other-model';
+    try {
+      assert.strictEqual(ollamaService.transport.client, client);
+      assert.strictEqual(
+        ollamaService.transport.apiUrl,
+        'http://elsewhere.invalid'
+      );
+      assert.strictEqual(ollamaService.transport.model, 'other-model');
+    } finally {
+      Object.assign(ollamaService, previous);
+    }
+  });
+
+  await check(
+    'Ollama: analysis sends format, num_ctx and num_predict',
+    async () => {
+      const posts = [];
+      ollamaService.client = fakeOllamaClient([{ data: ollamaData() }], posts);
+      const analysis = await analyse(ollamaService);
+      assert.strictEqual(analysis.error, undefined, analysis.error);
+      const { body } = posts[0];
+      assert.deepStrictEqual(body.format, ollamaService.documentAnalysisSchema);
+      assert.strictEqual(body.model, ollamaService.model);
+      assert.strictEqual(body.options.num_predict, 1000);
+      assert.strictEqual(
+        body.options.num_ctx,
+        Math.min(Math.ceil(body.prompt.length / 2) + 1000, 128000)
+      );
+      assert.strictEqual(
+        body.options.temperature,
+        config.aiTemperatureAnalysis
+      );
+    }
+  );
+
+  await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
+
+  if (failed > 0) {
+    console.error(`\n${failed} transport request mapping case(s) failed`);
+    process.exit(1);
+  }
+  console.log('\nAll transport request mapping cases passed');
+})();

From fdd4e5ae4dbef380292d52302be829625e6200fe Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 11 Oct 2026 21:26:24 +0200
Subject: [PATCH 38/53] Send generateText() through the transports as well

Background
Step 5b of the LLM service refactoring (services/ManualServiceRefactoring.md).
After 5a the analysis request went through one transport per API family,
but generateText() - used by the Duplicates and "Simplify tags" features -
was still written out four times, each with its own streaming, abort,
usage and cut-off handling.

Changes
- chatCompletionsTransport.generate(): abort and progress handling, the
  streamed and the plain call, lastGenerateTextUsage, reasoning stripping,
  the cut-off check with partialText and the abort normalization, moved
  verbatim from the three services. Hooks: generation.buildRequest(),
  notReadyMessage, remedy, streamUsage() and describeEmptyText().
- openai/azure/custom: generateText() delegates; each keeps its dialect in
  _buildGenerationRequest(), lifted verbatim (OpenAI caps max_tokens only on
  request; Azure always caps at RESPONSE_TOKENS, defaults to 0.7 and gates
  stream_options on its API version; Custom clamps to the context window and
  sends /no_think, chat_template_kwargs and reasoning_effort).
- ollamaGenerateTransport.generate() and its NDJSON stream reader, moved
  verbatim from ollamaService. The service still sizes num_predict/num_ctx
  through _calculatePromptTokenCount()/_calculateNumCtx() and hands them in.
- lastGenerateTextUsage is an accessor onto the transport on all four
  services; the AI review features keep reading it from the service.
- tests/test-llm-transport-request-mapping.js: 48 cases now.
- Plan: 5b marked done. New finding: checkStatus() has no caller anywhere
  in the repository; left untouched, decided in step 6/8.

Testing
Locally, better-sqlite3 was rebuilt for Node v24.19.0 (approved by the
user), so entity-match-ai-service, tag-simplify-service and the other
DB-backed guards of generateText() run again.
Mutation-checked on the generation path: dropping stream_options, the
abort normalization, the usage record, Azure's stream gate, Custom's
/no_think, Custom's empty-answer message, Ollama's reasoning override and
Ollama's abort signal each fail a test; the last three were not pinned
before and are now.
node scripts/run-tests.js --all: 105 passed, 7 skipped, 15 failed; the
same 15 fail on the parent commit. 14 of them also fail on 56b278f, before
the refactoring steps; icon-sizing fails only on CRLF line endings in the
local working copy of public/css. ESLint and Prettier clean on all changed
files.

Impact
None intended at runtime: the same requests, streams, usage reports,
errors and log lines. One edge: an exception while sizing Ollama's
num_ctx (e.g. a non-string prompt) now propagates without the
"Error generating text with Ollama" log line. No API or OpenAPI change.

Upstream Status
Local only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/ManualServiceRefactoring.md          |  22 +-
 services/azureService.js                      | 166 ++++------
 services/customService.js                     | 241 ++++++--------
 .../transports/chatCompletionsTransport.js    | 118 ++++++-
 .../llm/transports/ollamaGenerateTransport.js | 255 +++++++++++++++
 services/ollamaService.js                     | 264 ++-------------
 services/openaiService.js                     | 165 ++++------
 tests/test-llm-transport-request-mapping.js   | 303 ++++++++++++++++++
 8 files changed, 914 insertions(+), 620 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 1a47c18a..8b74381d 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -556,7 +556,27 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
      them. `num_ctx` is still computed by the service and handed in; moving it behind the
      transport is step 6. `llm-transport-request-mapping` (31 cases) pins both transports with
      fake clients and every provider's analysis request; six mutations each fail a case.
-   - **5b — `generateText()`** through the same transports. `checkStatus()` (row g) moves with it.
+   - ~~**5b — `generateText()`**~~ **Done.** The chat transport's `generate()` holds what the
+     three `generateText()` copies shared: abort and progress handling, the streamed and the plain
+     call, `lastGenerateTextUsage`, reasoning stripping, the cut-off check with `partialText`, and
+     the abort normalization. What differs stays with the provider, as one
+     `_buildGenerationRequest()` each — moved verbatim: OpenAI caps `max_tokens` only on request;
+     Azure always caps at RESPONSE_TOKENS and defaults to temperature 0.7; Custom clamps to the
+     context window and speaks three reasoning dialects (Qwen's `/no_think`,
+     `chat_template_kwargs`, `reasoning_effort`). Hooks: `streamUsage()` (Azure's API-version
+     gate for `stream_options`) and `describeEmptyText()` (Custom's "the answer carried no
+     text …"). Ollama's `generate()` and its NDJSON stream reader moved into
+     `ollamaGenerateTransport.js` verbatim; the service still sizes `num_predict`/`num_ctx`
+     through `_calculatePromptTokenCount()`/`_calculateNumCtx()` and hands them in.
+     `lastGenerateTextUsage` is an accessor onto the transport on all four services.
+     `llm-transport-request-mapping` grew to 48 cases. Of eight mutations on the generation path,
+     three survived the existing suite — Custom's empty-answer message, Ollama's `reasoning`
+     override of OLLAMA_THINK, and Ollama's abort signal — and are pinned now.
+
+     **Not moved: `checkStatus()`** (row g). Nothing in the repository calls it — no route, no
+     view, no service, no test. It stays in the four services untouched; step 6 decides whether the
+     composition roots expose it at all or whether it is dropped with the dead branches of step 8.
+
 6. **Introduce `documentAnalyzer.js`** and the four composition roots. Reduce
    `services/{openai,azure,custom,ollama}Service.js` to re-export shims. Add `llm-provider-parity`.
    Fix Finding 7a here (Azure temperature) with its test.
diff --git a/services/azureService.js b/services/azureService.js
index 60d7c919..1ebcb94b 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -2,22 +2,13 @@ const {
   calculateTokens,
   truncateToTokenLimit,
   writePromptToFile,
-  extractChatMessageContent,
-  assertCompletionNotTruncated,
 } = require('./serviceUtils');
 const {
-  abortSignal,
-  abortedGenerationError,
   hasNumber,
   hasSystemPrompt,
-  isAbortError,
   modelOverride,
-  progressHandler,
-  readCompletionUsage,
   reasoningEffortForOpenAi,
   reasoningEnabled,
-  runChatCompletionStream,
-  stripReasoningText,
 } = require('./aiGenerateOptions');
 const axios = require('axios');
 const AzureOpenAI = require('openai').AzureOpenAI;
@@ -78,6 +69,15 @@ class AzureOpenAIService {
   constructor() {
     this.transport = new ChatCompletionsTransport({
       label: 'AzureOpenAI',
+      generation: {
+        notReadyMessage: 'AzureOpenAI client not initialized - missing API key',
+        remedy: TRUNCATION_REMEDY,
+        buildRequest: (prompt, options) =>
+          this._buildGenerationRequest(prompt, options),
+        // Only where the configured API version takes it; an older one would
+        // answer 400 instead of streaming, so that deployment estimates.
+        streamUsage: () => supportsStreamOptions(config.azure.apiVersion),
+      },
       createClient: () =>
         config.aiProvider === 'azure'
           ? new AzureOpenAI({
@@ -99,6 +99,15 @@ class AzureOpenAIService {
     this.transport.client = client;
   }
 
+  /** Usage of the last generateText() call; read by the AI review features. */
+  get lastGenerateTextUsage() {
+    return this.transport.lastGenerateTextUsage;
+  }
+
+  set lastGenerateTextUsage(usage) {
+    this.transport.lastGenerateTextUsage = usage;
+  }
+
   initialize() {
     this.transport.initialize();
   }
@@ -269,110 +278,45 @@ class AzureOpenAIService {
    * @returns {Promise<string>} - The generated text
    */
   async generateText(prompt, options = {}) {
-    // Read before the try so the catch can tell a stopped request from a
-    // failed one.
-    const signal = abortSignal(options);
-    const onProgress = progressHandler(options);
-
-    try {
-      this.initialize();
-
-      if (!this.client) {
-        throw new Error('AzureOpenAI client not initialized - missing API key');
-      }
-
-      // Azure addresses a deployment rather than a model, so an override
-      // names the deployment to send this one request to.
-      const model = modelOverride(options) || process.env.AZURE_DEPLOYMENT_NAME;
-      const messages = [];
-      if (hasSystemPrompt(options)) {
-        messages.push({ role: 'system', content: options.systemPrompt });
-      }
-      messages.push({ role: 'user', content: prompt });
-
-      const request = {
-        model: model,
-        messages,
-        temperature: hasNumber(options.temperature) ? options.temperature : 0.7,
-        // Was a hardcoded 1000, which quarrelled with the setting the same way
-        // Ollama's num_predict did — the reservation followed the operator,
-        // the limit did not.
-        max_tokens:
-          hasNumber(options.maxTokens) && options.maxTokens > 0
-            ? Math.floor(options.maxTokens)
-            : Number(config.responseTokens),
-      };
-      // As on OpenAI: the reasoning deployments take reasoning_effort, every
-      // other one answers 400 for it, so it is sent for those names only and
-      // only when the caller asked for no thinking.
-      if (reasoningEnabled(options) === false) {
-        const effort = reasoningEffortForOpenAi(model);
-        if (effort) request.reasoning_effort = effort;
-      }
-
-      this.lastGenerateTextUsage = null;
-      // The second argument is the SDK's request option bag; a caller that
-      // brought no signal gets the call it always got.
-      const requestOptions = signal ? { signal } : null;
-
-      if (onProgress) {
-        const streamRequest = { ...request, stream: true };
-        // Only where the configured API version takes it; an older one would
-        // answer 400 instead of streaming, so that deployment estimates.
-        if (supportsStreamOptions(config.azure.apiVersion)) {
-          streamRequest.stream_options = { include_usage: true };
-        }
-        return await runChatCompletionStream({
-          client: this.client,
-          request: streamRequest,
-          requestOptions,
-          signal,
-          onProgress,
-          provider: 'AzureOpenAI',
-          remedy: TRUNCATION_REMEDY,
-          onUsage: (usage) => {
-            this.lastGenerateTextUsage = usage;
-          },
-        });
-      }
-
-      const response = requestOptions
-        ? await this.client.chat.completions.create(request, requestOptions)
-        : await this.client.chat.completions.create(request);
-      this.lastGenerateTextUsage = readCompletionUsage(response);
-
-      const message = response?.choices?.[0]?.message;
-      // Reasoning is no part of the answer, and a block the cut-off left
-      // open is no part of it either.
-      const generatedText = extractChatMessageContent(
-        { ...message, content: stripReasoningText(message?.content) },
-        'AzureOpenAI'
-      );
-
-      // After the text is in hand: a cut-off answer is still a failure, but
-      // the caller can keep the prefix instead of paying for it twice.
-      assertCompletionNotTruncated(response, 'AzureOpenAI', TRUNCATION_REMEDY, {
-        partialText: generatedText,
-      });
-
-      if (!generatedText) {
-        throw new Error('Invalid API response structure');
-      }
+    return this.transport.generate(prompt, options);
+  }
 
-      return generatedText;
-    } catch (error) {
-      // One name for a stopped request, whatever the client called it; an
-      // abort this service raised already carries its partial text.
-      const thrown =
-        isAbortError(error, signal) && error?.name !== 'AbortError'
-          ? abortedGenerationError(error, '')
-          : error;
-      console.error(
-        `Error generating text with AzureOpenAI: ${thrown.message}`
-      );
-      console.debug(thrown);
-      throw thrown;
+  /**
+   * This provider's generateText() request. The transport sends it, streams
+   * it and reads the answer; what differs between the providers is here.
+   * @param {string} prompt
+   * @param {object} options - see generateText()
+   * @returns {Object} The chat completions request
+   */
+  _buildGenerationRequest(prompt, options) {
+    const model = modelOverride(options) || process.env.AZURE_DEPLOYMENT_NAME;
+    const messages = [];
+    if (hasSystemPrompt(options)) {
+      messages.push({ role: 'system', content: options.systemPrompt });
     }
+    messages.push({ role: 'user', content: prompt });
+
+    const request = {
+      model: model,
+      messages,
+      temperature: hasNumber(options.temperature) ? options.temperature : 0.7,
+      // Was a hardcoded 1000, which quarrelled with the setting the same way
+      // Ollama's num_predict did — the reservation followed the operator,
+      // the limit did not.
+      max_tokens:
+        hasNumber(options.maxTokens) && options.maxTokens > 0
+          ? Math.floor(options.maxTokens)
+          : Number(config.responseTokens),
+    };
+    // As on OpenAI: the reasoning deployments take reasoning_effort, every
+    // other one answers 400 for it, so it is sent for those names only and
+    // only when the caller asked for no thinking.
+    if (reasoningEnabled(options) === false) {
+      const effort = reasoningEffortForOpenAi(model);
+      if (effort) request.reasoning_effort = effort;
+    }
+
+    return request;
   }
 
   async checkStatus() {
diff --git a/services/customService.js b/services/customService.js
index 0b8e4159..fad687e8 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -1,24 +1,15 @@
 const {
   calculateTokens,
   truncateToTokenLimit,
-  extractChatMessageContent,
-  assertCompletionNotTruncated,
   isTimeoutError,
   buildTimeoutErrorMessage,
 } = require('./serviceUtils');
 const {
-  abortSignal,
-  abortedGenerationError,
   hasNumber,
   hasSystemPrompt,
-  isAbortError,
   modelOverride,
-  progressHandler,
-  readCompletionUsage,
   reasoningEffortForCompatible,
   reasoningEnabled,
-  runChatCompletionStream,
-  stripReasoningText,
 } = require('./aiGenerateOptions');
 const OpenAI = require('openai');
 const config = require('../config/config');
@@ -61,6 +52,22 @@ class CustomOpenAIService {
   constructor() {
     this.transport = new ChatCompletionsTransport({
       label: 'Custom OpenAI',
+      generation: {
+        notReadyMessage:
+          'Custom OpenAI client not initialized - missing API key',
+        remedy: TRUNCATION_REMEDY,
+        buildRequest: (prompt, options) =>
+          this._buildGenerationRequest(prompt, options),
+        // Say what came back: a gateway that timed out behind a 200, a
+        // model that stopped after its reasoning without an answer, or an
+        // empty choices array all end here and need telling apart.
+        describeEmptyText: (response) => {
+          const finishReason =
+            response?.choices?.[0]?.finish_reason ?? 'unknown';
+          const head = JSON.stringify(response ?? null).slice(0, 200);
+          return `Invalid API response structure: the answer carried no text (finish_reason ${finishReason}; the response begins ${head})`;
+        },
+      },
       createClient: () =>
         config.aiProvider === 'custom'
           ? new OpenAI({
@@ -99,6 +106,15 @@ class CustomOpenAIService {
     this.transport.client = client;
   }
 
+  /** Usage of the last generateText() call; read by the AI review features. */
+  get lastGenerateTextUsage() {
+    return this.transport.lastGenerateTextUsage;
+  }
+
+  set lastGenerateTextUsage(usage) {
+    this.transport.lastGenerateTextUsage = usage;
+  }
+
   initialize() {
     this.transport.initialize();
   }
@@ -282,152 +298,77 @@ class CustomOpenAIService {
    * @returns {Promise<string>} - The generated text
    */
   async generateText(prompt, options = {}) {
-    // Read before the try so the catch can tell a stopped request from a
-    // failed one.
-    const signal = abortSignal(options);
-    const onProgress = progressHandler(options);
-
-    try {
-      this.initialize();
-
-      if (!this.client) {
-        throw new Error(
-          'Custom OpenAI client not initialized - missing API key'
-        );
-      }
-
-      const model = modelOverride(options) || config.custom.model;
-      const maxContextTokens = Number(config.tokenLimit) || 128000;
-      const desiredCompletionTokens =
-        hasNumber(options.maxTokens) && options.maxTokens > 0
-          ? Math.floor(options.maxTokens)
-          : Number(config.responseTokens) || 1000;
-      const promptTokens = await calculateTokens(prompt, model);
-      const availableCompletionTokens = Math.max(
-        1,
-        maxContextTokens - promptTokens - 64
-      );
-      const maxCompletionTokens = Math.max(
-        1,
-        Math.min(desiredCompletionTokens, availableCompletionTokens)
-      );
-
-      if (maxCompletionTokens < desiredCompletionTokens) {
-        console.log(
-          `[DEBUG] Clamped max_tokens for custom generateText from ${desiredCompletionTokens} to ${maxCompletionTokens} ` +
-            `(context limit: ${maxContextTokens}, prompt tokens: ${promptTokens})`
-        );
-      }
-
-      const reasoning = reasoningEnabled(options);
-      const messages = [];
-      if (hasSystemPrompt(options)) {
-        messages.push({ role: 'system', content: options.systemPrompt });
-      }
-      messages.push({
-        role: 'user',
-        content:
-          reasoning === false && QWEN_MODEL_PATTERN.test(model)
-            ? `${prompt}\n${QWEN_NO_THINK_SWITCH}`
-            : prompt,
-      });
-
-      const request = {
-        model: model,
-        messages,
-        temperature: hasNumber(options.temperature)
-          ? options.temperature
-          : config.aiTemperatureGeneration,
-        max_tokens: maxCompletionTokens,
-      };
-      // The key every OpenAI-compatible server that applies a chat template
-      // understands; one that does not know it passes it through and the
-      // model decides for itself, as it did before.
-      if (reasoning !== null) {
-        request.chat_template_kwargs = { enable_thinking: reasoning };
-      }
-      // A hosted reasoning model reads neither of the two above and thinks
-      // anyway; the families that take reasoning_effort are told in their own
-      // dialect. Only when the caller asked for no thinking, and only for
-      // those names — another model answers 400 for the field.
-      if (reasoning === false) {
-        const effort = reasoningEffortForCompatible(model);
-        if (effort) request.reasoning_effort = effort;
-      }
-
-      this.lastGenerateTextUsage = null;
-      // The second argument is the SDK's request option bag; a caller that
-      // brought no signal gets the call it always got.
-      const requestOptions = signal ? { signal } : null;
-
-      if (onProgress) {
-        // include_usage buys the exact completion count at the end of the
-        // stream; without it the report is an estimate all the way.
-        return await runChatCompletionStream({
-          client: this.client,
-          request: {
-            ...request,
-            stream: true,
-            stream_options: { include_usage: true },
-          },
-          requestOptions,
-          signal,
-          onProgress,
-          provider: 'Custom OpenAI',
-          remedy: TRUNCATION_REMEDY,
-          onUsage: (usage) => {
-            this.lastGenerateTextUsage = usage;
-          },
-        });
-      }
-
-      const response = requestOptions
-        ? await this.client.chat.completions.create(request, requestOptions)
-        : await this.client.chat.completions.create(request);
-      this.lastGenerateTextUsage = readCompletionUsage(response);
-
-      const message = response?.choices?.[0]?.message;
-      // Reasoning is no part of the answer, and a block the cut-off left
-      // open is no part of it either.
-      const generatedText = extractChatMessageContent(
-        { ...message, content: stripReasoningText(message?.content) },
-        'Custom OpenAI'
-      );
+    return this.transport.generate(prompt, options);
+  }
 
-      // After the text is in hand: a cut-off answer is still a failure, but
-      // the caller can keep the prefix instead of paying for it twice.
-      assertCompletionNotTruncated(
-        response,
-        'Custom OpenAI',
-        TRUNCATION_REMEDY,
-        { partialText: generatedText }
+  /**
+   * This provider's generateText() request. The transport sends it, streams
+   * it and reads the answer; what differs between the providers is here.
+   * @param {string} prompt
+   * @param {object} options - see generateText()
+   * @returns {Promise<Object>} The chat completions request
+   */
+  async _buildGenerationRequest(prompt, options) {
+    const model = modelOverride(options) || config.custom.model;
+    const maxContextTokens = Number(config.tokenLimit) || 128000;
+    const desiredCompletionTokens =
+      hasNumber(options.maxTokens) && options.maxTokens > 0
+        ? Math.floor(options.maxTokens)
+        : Number(config.responseTokens) || 1000;
+    const promptTokens = await calculateTokens(prompt, model);
+    const availableCompletionTokens = Math.max(
+      1,
+      maxContextTokens - promptTokens - 64
+    );
+    const maxCompletionTokens = Math.max(
+      1,
+      Math.min(desiredCompletionTokens, availableCompletionTokens)
+    );
+
+    if (maxCompletionTokens < desiredCompletionTokens) {
+      console.log(
+        `[DEBUG] Clamped max_tokens for custom generateText from ${desiredCompletionTokens} to ${maxCompletionTokens} ` +
+          `(context limit: ${maxContextTokens}, prompt tokens: ${promptTokens})`
       );
+    }
 
-      if (!generatedText) {
-        // Say what came back: a gateway that timed out behind a 200, a
-        // model that stopped after its reasoning without an answer, or an
-        // empty choices array all end here and need telling apart.
-        const finishReason = response?.choices?.[0]?.finish_reason ?? 'unknown';
-        const head = JSON.stringify(response ?? null).slice(0, 200);
-        throw new Error(
-          `Invalid API response structure: the answer carried no text (finish_reason ${finishReason}; the response begins ${head})`
-        );
-      }
+    const reasoning = reasoningEnabled(options);
+    const messages = [];
+    if (hasSystemPrompt(options)) {
+      messages.push({ role: 'system', content: options.systemPrompt });
+    }
+    messages.push({
+      role: 'user',
+      content:
+        reasoning === false && QWEN_MODEL_PATTERN.test(model)
+          ? `${prompt}\n${QWEN_NO_THINK_SWITCH}`
+          : prompt,
+    });
 
-      return generatedText;
-    } catch (error) {
-      // One name for a stopped request, whatever the client called it; an
-      // abort this service raised already carries its partial text.
-      const thrown =
-        isAbortError(error, signal) && error?.name !== 'AbortError'
-          ? abortedGenerationError(error, '')
-          : error;
-      console.error(
-        `Error generating text with Custom OpenAI: ${thrown.message}`
-      );
-      console.debug(thrown);
-      throw thrown;
+    const request = {
+      model: model,
+      messages,
+      temperature: hasNumber(options.temperature)
+        ? options.temperature
+        : config.aiTemperatureGeneration,
+      max_tokens: maxCompletionTokens,
+    };
+    // The key every OpenAI-compatible server that applies a chat template
+    // understands; one that does not know it passes it through and the
+    // model decides for itself, as it did before.
+    if (reasoning !== null) {
+      request.chat_template_kwargs = { enable_thinking: reasoning };
+    }
+    // A hosted reasoning model reads neither of the two above and thinks
+    // anyway; the families that take reasoning_effort are told in their own
+    // dialect. Only when the caller asked for no thinking, and only for
+    // those names — another model answers 400 for the field.
+    if (reasoning === false) {
+      const effort = reasoningEffortForCompatible(model);
+      if (effort) request.reasoning_effort = effort;
     }
+
+    return request;
   }
 
   async checkStatus() {
diff --git a/services/llm/transports/chatCompletionsTransport.js b/services/llm/transports/chatCompletionsTransport.js
index c0b759bd..64f673a6 100644
--- a/services/llm/transports/chatCompletionsTransport.js
+++ b/services/llm/transports/chatCompletionsTransport.js
@@ -14,6 +14,15 @@ const {
   assertCompletionNotTruncated,
   extractChatMessageContent,
 } = require('../../serviceUtils');
+const {
+  abortSignal,
+  abortedGenerationError,
+  isAbortError,
+  progressHandler,
+  readCompletionUsage,
+  runChatCompletionStream,
+  stripReasoningText,
+} = require('../../aiGenerateOptions');
 
 /** What an operator can do about an analysis answer that hit the cap. */
 const ANALYSIS_TRUNCATION_REMEDY =
@@ -28,13 +37,31 @@ class ChatCompletionsTransport {
    * @param {(response:Object, message:Object) => string} [options.describeMissingContent]
    *   The error message for an answer without usable text. It must keep the
    *   "Invalid API response structure" phrase the OCR fallback matches on.
+   * @param {Object} [options.generation] - What generate() needs from the provider
+   * @param {string} options.generation.notReadyMessage - Thrown without a client
+   * @param {(prompt:string, options:Object) => Promise<Object>} options.generation.buildRequest
+   *   The provider's request: model, messages and its own knobs (temperature
+   *   default, max_tokens policy, reasoning fields). The dialects differ for
+   *   real, so they stay with the provider.
+   * @param {string} options.generation.remedy - Named when the answer is cut off
+   * @param {() => boolean} [options.generation.streamUsage] - Whether a
+   *   streamed request may ask for its usage (stream_options.include_usage)
+   * @param {(response:Object) => string} [options.generation.describeEmptyText]
+   *   The error message for an answer without text
    */
-  constructor({ label, createClient, describeMissingContent }) {
+  constructor({ label, createClient, describeMissingContent, generation }) {
     this.label = label;
     this.client = null;
+    /** Usage of the last generate() call; read by the AI review features. */
+    this.lastGenerateTextUsage = null;
     this._createClient = createClient;
     this._describeMissingContent =
       describeMissingContent || (() => 'Invalid API response structure');
+    this._generation = {
+      streamUsage: () => true,
+      describeEmptyText: () => 'Invalid API response structure',
+      ...generation,
+    };
   }
 
   /** Build the client on first use; a client set from outside is kept. */
@@ -105,6 +132,95 @@ class ChatCompletionsTransport {
       },
     };
   }
+
+  /**
+   * Send one generation request and return its text.
+   *
+   * @param {string} prompt
+   * @param {Object} [options] - As read by aiGenerateOptions: signal,
+   *   onProgress, and whatever the provider's buildRequest() reads
+   * @returns {Promise<string>} The answer, reasoning stripped
+   */
+  async generate(prompt, options = {}) {
+    // Read before the try so the catch can tell a stopped request from a
+    // failed one.
+    const signal = abortSignal(options);
+    const onProgress = progressHandler(options);
+    const generation = this._generation;
+
+    try {
+      this.initialize();
+
+      if (!this.client) {
+        throw new Error(generation.notReadyMessage);
+      }
+
+      const request = await generation.buildRequest(prompt, options);
+
+      this.lastGenerateTextUsage = null;
+      // The second argument is the SDK's request option bag; a caller that
+      // brought no signal gets the call it always got.
+      const requestOptions = signal ? { signal } : null;
+
+      if (onProgress) {
+        const streamRequest = { ...request, stream: true };
+        // include_usage buys the exact completion count at the end of the
+        // stream; without it the report is an estimate all the way.
+        if (generation.streamUsage()) {
+          streamRequest.stream_options = { include_usage: true };
+        }
+        return await runChatCompletionStream({
+          client: this.client,
+          request: streamRequest,
+          requestOptions,
+          signal,
+          onProgress,
+          provider: this.label,
+          remedy: generation.remedy,
+          onUsage: (usage) => {
+            this.lastGenerateTextUsage = usage;
+          },
+        });
+      }
+
+      const response = requestOptions
+        ? await this.client.chat.completions.create(request, requestOptions)
+        : await this.client.chat.completions.create(request);
+      this.lastGenerateTextUsage = readCompletionUsage(response);
+
+      const message = response?.choices?.[0]?.message;
+      // Reasoning is no part of the answer, and a block the cut-off left
+      // open is no part of it either.
+      const generatedText = extractChatMessageContent(
+        { ...message, content: stripReasoningText(message?.content) },
+        this.label
+      );
+
+      // After the text is in hand: a cut-off answer is still a failure, but
+      // the caller can keep the prefix instead of paying for it twice.
+      assertCompletionNotTruncated(response, this.label, generation.remedy, {
+        partialText: generatedText,
+      });
+
+      if (!generatedText) {
+        throw new Error(generation.describeEmptyText(response));
+      }
+
+      return generatedText;
+    } catch (error) {
+      // One name for a stopped request, whatever the client called it; an
+      // abort this transport raised already carries its partial text.
+      const thrown =
+        isAbortError(error, signal) && error?.name !== 'AbortError'
+          ? abortedGenerationError(error, '')
+          : error;
+      console.error(
+        `Error generating text with ${this.label}: ${thrown.message}`
+      );
+      console.debug(thrown);
+      throw thrown;
+    }
+  }
 }
 
 module.exports = { ChatCompletionsTransport, ANALYSIS_TRUNCATION_REMEDY };
diff --git a/services/llm/transports/ollamaGenerateTransport.js b/services/llm/transports/ollamaGenerateTransport.js
index adc06b70..610d6b93 100644
--- a/services/llm/transports/ollamaGenerateTransport.js
+++ b/services/llm/transports/ollamaGenerateTransport.js
@@ -14,6 +14,18 @@
  */
 
 const config = require('../../../config/config');
+const {
+  abortSignal,
+  abortedGenerationError,
+  createProgressCollector,
+  hasNumber,
+  hasSystemPrompt,
+  isAbortError,
+  modelOverride,
+  progressHandler,
+  reasoningEnabled,
+  stripReasoningText,
+} = require('../../aiGenerateOptions');
 
 /**
  * One bounded retry for transient failures (backend EOF, 5xx). The field
@@ -36,6 +48,8 @@ class OllamaGenerateTransport {
     this.apiUrl = apiUrl;
     this.model = model;
     this.retryDelayMs = retryDelayMs;
+    /** Usage of the last generate() call; read by the AI review features. */
+    this.lastGenerateTextUsage = null;
   }
 
   initialize() {}
@@ -198,6 +212,247 @@ class OllamaGenerateTransport {
     };
   }
 
+  /**
+   * Send one generation request and return its text.
+   *
+   * @param {string} prompt
+   * @param {object} [options] - As read by aiGenerateOptions: systemPrompt,
+   *   temperature, model, signal, onProgress, reasoning
+   * @param {Object} numbers - Sized by the caller
+   * @param {number} numbers.responseTokens - Sent as num_predict
+   * @param {number} numbers.numCtx - Context window to request
+   * @returns {Promise<string>} - The generated text
+   */
+  async generate(prompt, options = {}, { responseTokens, numCtx }) {
+    // Read before the try so the catch can tell a stopped request from a
+    // failed one, and so a cut-off request keeps what it measured.
+    const signal = abortSignal(options);
+    const onProgress = progressHandler(options);
+    let measuredUsage = null;
+
+    try {
+      // Simple system prompt for text generation; a caller that knows the job
+      // better (the Duplicates review) brings its own.
+      const systemPrompt = hasSystemPrompt(options)
+        ? options.systemPrompt
+        : `You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.`;
+
+      // Call Ollama API without enforcing a specific response format
+      const generateTextBody = {
+        model: modelOverride(options) || this.model,
+        prompt: prompt,
+        system: systemPrompt,
+        stream: false,
+        options: {
+          temperature: hasNumber(options.temperature)
+            ? options.temperature
+            : config.aiTemperatureGeneration,
+          top_p: 0.9,
+          // The line above reserves the same number in the context; a
+          // constant here would have the same quarrel with it that the
+          // analysis path had.
+          num_predict: responseTokens,
+          num_ctx: numCtx,
+        },
+      };
+
+      // A caller that says what it wants overrules the setting; without one
+      // OLLAMA_THINK decides, which is what it did before.
+      const reasoning = reasoningEnabled(options);
+      if (reasoning !== null) {
+        generateTextBody.think = reasoning;
+      } else if (!config.ollama.think) {
+        // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
+        // unless explicitly enabled via OLLAMA_THINK=true
+        generateTextBody.think = false;
+      }
+
+      // axios takes the signal in the request config; without one the config
+      // is the headers it always was.
+      const requestConfig = { headers: this.buildHeaders() };
+      if (signal) requestConfig.signal = signal;
+
+      if (onProgress) {
+        return await this._generateStreamed(
+          generateTextBody,
+          requestConfig,
+          { signal, onProgress, numPredict: responseTokens, numCtx },
+          (usage) => {
+            measuredUsage = usage;
+            this.lastGenerateTextUsage = usage;
+          }
+        );
+      }
+
+      const response = await this.client.post(
+        `${this.apiUrl}/api/generate`,
+        generateTextBody,
+        requestConfig
+      );
+
+      if (!response.data || !response.data.response) {
+        throw new Error('Invalid response from Ollama API');
+      }
+
+      // Ollama reports its token counts on the answer itself; the callers
+      // that care (the Duplicates review) read them off the service after
+      // the call, so the return value stays a plain string.
+      const promptTokens = Number(response.data.prompt_eval_count);
+      const completionTokens = Number(response.data.eval_count);
+      measuredUsage =
+        Number.isFinite(promptTokens) || Number.isFinite(completionTokens)
+          ? {
+              promptTokens: Number.isFinite(promptTokens) ? promptTokens : null,
+              completionTokens: Number.isFinite(completionTokens)
+                ? completionTokens
+                : null,
+              totalTokens:
+                (Number.isFinite(promptTokens) ? promptTokens : 0) +
+                (Number.isFinite(completionTokens) ? completionTokens : 0),
+            }
+          : null;
+      this.lastGenerateTextUsage = measuredUsage;
+
+      // A model that writes its reasoning into the answer wrote it for
+      // itself; only what it said afterwards is the answer. Anything that is
+      // not a string is left alone — it is a structured answer, not text.
+      const generatedText =
+        typeof response.data.response === 'string'
+          ? stripReasoningText(response.data.response).trim()
+          : response.data.response;
+
+      // The cut-off check the analysis path has always run, now on this path
+      // too, and carrying the prefix the caller can still use.
+      this.assertNotTruncated(response.data, responseTokens, numCtx, {
+        partialText: typeof generatedText === 'string' ? generatedText : '',
+      });
+
+      return generatedText;
+    } catch (error) {
+      // Whatever was measured before the failure stays readable; a failure
+      // that measured nothing still leaves null behind, as it always did.
+      this.lastGenerateTextUsage = measuredUsage;
+      // One name for a stopped request, whatever axios called it; an abort
+      // this transport raised already carries its partial text.
+      const thrown =
+        isAbortError(error, signal) && error?.name !== 'AbortError'
+          ? abortedGenerationError(error, '')
+          : error;
+      console.error(`Error generating text with Ollama: ${thrown.message}`);
+      console.debug(thrown);
+      throw thrown;
+    }
+  }
+
+  /**
+   * The streamed half of generate(): the same request with `stream: true`,
+   * read as the NDJSON lines Ollama answers with.
+   *
+   * Ollama does not speak server-sent events — every line is a complete JSON
+   * object with the next piece of the answer in `response`, the next piece of
+   * the model's thinking in `thinking`, and on the last line the counts and
+   * the reason it stopped. Lines arrive split across socket reads, so the tail
+   * of a read is kept until its newline turns up.
+   *
+   * @param {Object} body - the request body, `stream` still to be set
+   * @param {Object} requestConfig - headers, and the signal if there is one
+   * @param {Object} context
+   * @param {AbortSignal|null} context.signal
+   * @param {Function} context.onProgress
+   * @param {number} context.numPredict - response budget that was sent
+   * @param {number} context.numCtx - context window that was sent
+   * @param {Function} onUsage - told before a cut-off throws
+   * @returns {Promise<string>} the answer, reasoning stripped
+   */
+  async _generateStreamed(body, requestConfig, context, onUsage) {
+    const { signal, onProgress, numPredict, numCtx } = context;
+    const collector = createProgressCollector({ onProgress });
+    let last = {};
+
+    try {
+      const response = await this.client.post(
+        `${this.apiUrl}/api/generate`,
+        { ...body, stream: true },
+        { ...requestConfig, responseType: 'stream' }
+      );
+
+      const stream = response.data;
+      // An abort tears the socket down, and a socket error that arrives after
+      // the loop has left would be an unhandled 'error' event — which ends
+      // the process rather than the request.
+      stream.on('error', () => {});
+      let pending = '';
+      const takeLine = (line) => {
+        if (line.trim() === '') return;
+        let parsed;
+        try {
+          parsed = JSON.parse(line);
+        } catch {
+          // A line that is not JSON is a line this client cannot use; the
+          // stream goes on, and the answer is whatever the rest carries.
+          return;
+        }
+        collector.pushContent(parsed.response);
+        collector.pushReasoning(parsed.thinking);
+        if (parsed.done === true) last = parsed;
+        collector.report();
+      };
+
+      for await (const piece of stream) {
+        if (signal?.aborted) {
+          // Nothing more is wanted, so nothing more is read: the socket goes
+          // now rather than after the model finished talking to itself.
+          stream.destroy();
+          break;
+        }
+        pending += piece.toString('utf8');
+        let at = pending.indexOf('\n');
+        while (at !== -1) {
+          takeLine(pending.slice(0, at));
+          pending = pending.slice(at + 1);
+          at = pending.indexOf('\n');
+        }
+      }
+      if (!signal?.aborted) takeLine(pending);
+
+      if (signal?.aborted) {
+        collector.finish();
+        throw abortedGenerationError(null, collector.text());
+      }
+    } catch (error) {
+      collector.finish();
+      if (error?.name === 'AbortError') throw error;
+      if (isAbortError(error, signal)) {
+        throw abortedGenerationError(error, collector.text());
+      }
+      throw error;
+    }
+
+    collector.finish();
+
+    const promptTokens = Number(last.prompt_eval_count);
+    const completionTokens = Number(last.eval_count);
+    const measured =
+      Number.isFinite(promptTokens) || Number.isFinite(completionTokens)
+        ? {
+            promptTokens: Number.isFinite(promptTokens) ? promptTokens : null,
+            completionTokens: Number.isFinite(completionTokens)
+              ? completionTokens
+              : null,
+            totalTokens:
+              (Number.isFinite(promptTokens) ? promptTokens : 0) +
+              (Number.isFinite(completionTokens) ? completionTokens : 0),
+          }
+        : collector.usageSummary();
+    if (typeof onUsage === 'function') onUsage(measured);
+
+    this.assertNotTruncated(last, numPredict, numCtx, {
+      partialText: collector.text(),
+    });
+
+    return collector.text();
+  }
+
   /**
    * A generation Ollama had to cut short comes back with done_reason "length".
    * What it returns is a prefix, and for structured output that means JSON
diff --git a/services/ollamaService.js b/services/ollamaService.js
index bd42082a..f5717ac7 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -1,16 +1,5 @@
 const { writePromptToFile } = require('./serviceUtils');
-const {
-  abortSignal,
-  abortedGenerationError,
-  createProgressCollector,
-  hasNumber,
-  hasSystemPrompt,
-  isAbortError,
-  modelOverride,
-  progressHandler,
-  reasoningEnabled,
-  stripReasoningText,
-} = require('./aiGenerateOptions');
+const { hasNumber } = require('./aiGenerateOptions');
 const axios = require('axios');
 const config = require('../config/config');
 const { cacheThumbnail } = require('./thumbnailCache');
@@ -97,6 +86,15 @@ class OllamaService {
     this.transport.model = model;
   }
 
+  /** Usage of the last generateText() call; read by the AI review features. */
+  get lastGenerateTextUsage() {
+    return this.transport.lastGenerateTextUsage;
+  }
+
+  set lastGenerateTextUsage(usage) {
+    this.transport.lastGenerateTextUsage = usage;
+  }
+
   /**
    * Analyzes a document to extract metadata and tags using an AI model.
    * @param {string} content - The content of the document to analyze.
@@ -513,242 +511,16 @@ class OllamaService {
    * @returns {Promise<string>} - The generated text
    */
   async generateText(prompt, options = {}) {
-    // Read before the try so the catch can tell a stopped request from a
-    // failed one, and so a cut-off request keeps what it measured.
-    const signal = abortSignal(options);
-    const onProgress = progressHandler(options);
-    let measuredUsage = null;
-
-    try {
-      const responseTokens =
-        hasNumber(options.maxTokens) && options.maxTokens > 0
-          ? Math.floor(options.maxTokens)
-          : Number(config.responseTokens);
-
-      // Calculate context window size based on prompt length
-      const promptTokenCount = this._calculatePromptTokenCount(prompt);
-      const numCtx = this._calculateNumCtx(promptTokenCount, responseTokens);
-
-      // Simple system prompt for text generation; a caller that knows the job
-      // better (the Duplicates review) brings its own.
-      const systemPrompt = hasSystemPrompt(options)
-        ? options.systemPrompt
-        : `You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.`;
-
-      // Call Ollama API without enforcing a specific response format
-      const generateTextBody = {
-        model: modelOverride(options) || this.model,
-        prompt: prompt,
-        system: systemPrompt,
-        stream: false,
-        options: {
-          temperature: hasNumber(options.temperature)
-            ? options.temperature
-            : config.aiTemperatureGeneration,
-          top_p: 0.9,
-          // The line above reserves the same number in the context; a
-          // constant here would have the same quarrel with it that the
-          // analysis path had.
-          num_predict: responseTokens,
-          num_ctx: numCtx,
-        },
-      };
-
-      // A caller that says what it wants overrules the setting; without one
-      // OLLAMA_THINK decides, which is what it did before.
-      const reasoning = reasoningEnabled(options);
-      if (reasoning !== null) {
-        generateTextBody.think = reasoning;
-      } else if (!config.ollama.think) {
-        // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1)
-        // unless explicitly enabled via OLLAMA_THINK=true
-        generateTextBody.think = false;
-      }
-
-      // axios takes the signal in the request config; without one the config
-      // is the headers it always was.
-      const requestConfig = { headers: this._buildRequestHeaders() };
-      if (signal) requestConfig.signal = signal;
-
-      if (onProgress) {
-        return await this._generateTextStreamed(
-          generateTextBody,
-          requestConfig,
-          { signal, onProgress, numPredict: responseTokens, numCtx },
-          (usage) => {
-            measuredUsage = usage;
-            this.lastGenerateTextUsage = usage;
-          }
-        );
-      }
-
-      const response = await this.client.post(
-        `${this.apiUrl}/api/generate`,
-        generateTextBody,
-        requestConfig
-      );
-
-      if (!response.data || !response.data.response) {
-        throw new Error('Invalid response from Ollama API');
-      }
-
-      // Ollama reports its token counts on the answer itself; the callers
-      // that care (the Duplicates review) read them off the service after
-      // the call, so the return value stays a plain string.
-      const promptTokens = Number(response.data.prompt_eval_count);
-      const completionTokens = Number(response.data.eval_count);
-      measuredUsage =
-        Number.isFinite(promptTokens) || Number.isFinite(completionTokens)
-          ? {
-              promptTokens: Number.isFinite(promptTokens) ? promptTokens : null,
-              completionTokens: Number.isFinite(completionTokens)
-                ? completionTokens
-                : null,
-              totalTokens:
-                (Number.isFinite(promptTokens) ? promptTokens : 0) +
-                (Number.isFinite(completionTokens) ? completionTokens : 0),
-            }
-          : null;
-      this.lastGenerateTextUsage = measuredUsage;
-
-      // A model that writes its reasoning into the answer wrote it for
-      // itself; only what it said afterwards is the answer. Anything that is
-      // not a string is left alone — it is a structured answer, not text.
-      const generatedText =
-        typeof response.data.response === 'string'
-          ? stripReasoningText(response.data.response).trim()
-          : response.data.response;
-
-      // The cut-off check the analysis path has always run, now on this path
-      // too, and carrying the prefix the caller can still use.
-      this._assertNotTruncated(response.data, responseTokens, numCtx, {
-        partialText: typeof generatedText === 'string' ? generatedText : '',
-      });
-
-      return generatedText;
-    } catch (error) {
-      // Whatever was measured before the failure stays readable; a failure
-      // that measured nothing still leaves null behind, as it always did.
-      this.lastGenerateTextUsage = measuredUsage;
-      // One name for a stopped request, whatever axios called it; an abort
-      // this service raised already carries its partial text.
-      const thrown =
-        isAbortError(error, signal) && error?.name !== 'AbortError'
-          ? abortedGenerationError(error, '')
-          : error;
-      console.error(`Error generating text with Ollama: ${thrown.message}`);
-      console.debug(thrown);
-      throw thrown;
-    }
-  }
-
-  /**
-   * The streamed half of generateText(): the same request with `stream: true`,
-   * read as the NDJSON lines Ollama answers with.
-   *
-   * Ollama does not speak server-sent events — every line is a complete JSON
-   * object with the next piece of the answer in `response`, the next piece of
-   * the model's thinking in `thinking`, and on the last line the counts and
-   * the reason it stopped. Lines arrive split across socket reads, so the tail
-   * of a read is kept until its newline turns up.
-   *
-   * @param {Object} body - the request body, `stream` still to be set
-   * @param {Object} requestConfig - headers, and the signal if there is one
-   * @param {Object} context
-   * @param {AbortSignal|null} context.signal
-   * @param {Function} context.onProgress
-   * @param {number} context.numPredict - response budget that was sent
-   * @param {number} context.numCtx - context window that was sent
-   * @param {Function} onUsage - told before a cut-off throws
-   * @returns {Promise<string>} the answer, reasoning stripped
-   */
-  async _generateTextStreamed(body, requestConfig, context, onUsage) {
-    const { signal, onProgress, numPredict, numCtx } = context;
-    const collector = createProgressCollector({ onProgress });
-    let last = {};
-
-    try {
-      const response = await this.client.post(
-        `${this.apiUrl}/api/generate`,
-        { ...body, stream: true },
-        { ...requestConfig, responseType: 'stream' }
-      );
+    const responseTokens =
+      hasNumber(options.maxTokens) && options.maxTokens > 0
+        ? Math.floor(options.maxTokens)
+        : Number(config.responseTokens);
 
-      const stream = response.data;
-      // An abort tears the socket down, and a socket error that arrives after
-      // the loop has left would be an unhandled 'error' event — which ends
-      // the process rather than the request.
-      stream.on('error', () => {});
-      let pending = '';
-      const takeLine = (line) => {
-        if (line.trim() === '') return;
-        let parsed;
-        try {
-          parsed = JSON.parse(line);
-        } catch {
-          // A line that is not JSON is a line this client cannot use; the
-          // stream goes on, and the answer is whatever the rest carries.
-          return;
-        }
-        collector.pushContent(parsed.response);
-        collector.pushReasoning(parsed.thinking);
-        if (parsed.done === true) last = parsed;
-        collector.report();
-      };
-
-      for await (const piece of stream) {
-        if (signal?.aborted) {
-          // Nothing more is wanted, so nothing more is read: the socket goes
-          // now rather than after the model finished talking to itself.
-          stream.destroy();
-          break;
-        }
-        pending += piece.toString('utf8');
-        let at = pending.indexOf('\n');
-        while (at !== -1) {
-          takeLine(pending.slice(0, at));
-          pending = pending.slice(at + 1);
-          at = pending.indexOf('\n');
-        }
-      }
-      if (!signal?.aborted) takeLine(pending);
-
-      if (signal?.aborted) {
-        collector.finish();
-        throw abortedGenerationError(null, collector.text());
-      }
-    } catch (error) {
-      collector.finish();
-      if (error?.name === 'AbortError') throw error;
-      if (isAbortError(error, signal)) {
-        throw abortedGenerationError(error, collector.text());
-      }
-      throw error;
-    }
-
-    collector.finish();
-
-    const promptTokens = Number(last.prompt_eval_count);
-    const completionTokens = Number(last.eval_count);
-    const measured =
-      Number.isFinite(promptTokens) || Number.isFinite(completionTokens)
-        ? {
-            promptTokens: Number.isFinite(promptTokens) ? promptTokens : null,
-            completionTokens: Number.isFinite(completionTokens)
-              ? completionTokens
-              : null,
-            totalTokens:
-              (Number.isFinite(promptTokens) ? promptTokens : 0) +
-              (Number.isFinite(completionTokens) ? completionTokens : 0),
-          }
-        : collector.usageSummary();
-    if (typeof onUsage === 'function') onUsage(measured);
-
-    this._assertNotTruncated(last, numPredict, numCtx, {
-      partialText: collector.text(),
-    });
+    // Calculate context window size based on prompt length
+    const promptTokenCount = this._calculatePromptTokenCount(prompt);
+    const numCtx = this._calculateNumCtx(promptTokenCount, responseTokens);
 
-    return collector.text();
+    return this.transport.generate(prompt, options, { responseTokens, numCtx });
   }
 
   /**
diff --git a/services/openaiService.js b/services/openaiService.js
index e128c580..4234fe80 100644
--- a/services/openaiService.js
+++ b/services/openaiService.js
@@ -2,22 +2,13 @@ const {
   calculateTokens,
   truncateToTokenLimit,
   writePromptToFile,
-  extractChatMessageContent,
-  assertCompletionNotTruncated,
 } = require('./serviceUtils');
 const {
-  abortSignal,
-  abortedGenerationError,
   hasNumber,
   hasSystemPrompt,
-  isAbortError,
   modelOverride,
-  progressHandler,
-  readCompletionUsage,
   reasoningEffortForOpenAi,
   reasoningEnabled,
-  runChatCompletionStream,
-  stripReasoningText,
 } = require('./aiGenerateOptions');
 const OpenAI = require('openai');
 const config = require('../config/config');
@@ -53,6 +44,12 @@ class OpenAIService {
   constructor() {
     this.transport = new ChatCompletionsTransport({
       label: 'OpenAI',
+      generation: {
+        notReadyMessage: 'OpenAI client not initialized - missing API key',
+        remedy: TRUNCATION_REMEDY,
+        buildRequest: (prompt, options) =>
+          this._buildGenerationRequest(prompt, options),
+      },
       createClient: () => {
         if (config.aiProvider === 'ollama') {
           return new OpenAI({
@@ -87,6 +84,15 @@ class OpenAIService {
     this.transport.client = client;
   }
 
+  /** Usage of the last generateText() call; read by the AI review features. */
+  get lastGenerateTextUsage() {
+    return this.transport.lastGenerateTextUsage;
+  }
+
+  set lastGenerateTextUsage(usage) {
+    this.transport.lastGenerateTextUsage = usage;
+  }
+
   initialize() {
     this.transport.initialize();
   }
@@ -270,110 +276,47 @@ class OpenAIService {
    * @returns {Promise<string>} - The generated text
    */
   async generateText(prompt, options = {}) {
-    // Read before the try so the catch can tell a stopped request from a
-    // failed one.
-    const signal = abortSignal(options);
-    const onProgress = progressHandler(options);
-
-    try {
-      this.initialize();
-
-      if (!this.client) {
-        throw new Error('OpenAI client not initialized - missing API key');
-      }
-
-      const model =
-        modelOverride(options) ||
-        process.env.OPENAI_MODEL ||
-        config.openai.model;
-      const messages = [];
-      if (hasSystemPrompt(options)) {
-        messages.push({ role: 'system', content: options.systemPrompt });
-      }
-      messages.push({ role: 'user', content: prompt });
-
-      const request = {
-        model: model,
-        messages,
-        temperature: hasNumber(options.temperature)
-          ? options.temperature
-          : config.aiTemperatureGeneration,
-      };
-      // Without a cap the model answers with whatever the deployment allows,
-      // which is what every caller before the AI review wanted.
-      if (hasNumber(options.maxTokens) && options.maxTokens > 0) {
-        request.max_tokens = Math.floor(options.maxTokens);
-      }
-      // The one thinking switch this API has, and it exists on the reasoning
-      // families only: an ordinary model answers 400 for the field, so it is
-      // sent for those names and for no others, and only when the caller
-      // asked for no thinking.
-      if (reasoningEnabled(options) === false) {
-        const effort = reasoningEffortForOpenAi(model);
-        if (effort) request.reasoning_effort = effort;
-      }
-
-      this.lastGenerateTextUsage = null;
-      // The second argument is the SDK's request option bag; a caller that
-      // brought no signal gets the call it always got.
-      const requestOptions = signal ? { signal } : null;
-
-      if (onProgress) {
-        // include_usage buys the exact completion count at the end of the
-        // stream; without it the report is an estimate all the way.
-        return await runChatCompletionStream({
-          client: this.client,
-          request: {
-            ...request,
-            stream: true,
-            stream_options: { include_usage: true },
-          },
-          requestOptions,
-          signal,
-          onProgress,
-          provider: 'OpenAI',
-          remedy: TRUNCATION_REMEDY,
-          onUsage: (usage) => {
-            this.lastGenerateTextUsage = usage;
-          },
-        });
-      }
-
-      const response = requestOptions
-        ? await this.client.chat.completions.create(request, requestOptions)
-        : await this.client.chat.completions.create(request);
-      this.lastGenerateTextUsage = readCompletionUsage(response);
-
-      const message = response?.choices?.[0]?.message;
-      // Reasoning is no part of the answer, and a block the cut-off left
-      // open is no part of it either.
-      const generatedText = extractChatMessageContent(
-        { ...message, content: stripReasoningText(message?.content) },
-        'OpenAI'
-      );
-
-      // After the text is in hand: a cut-off answer is still a failure, but
-      // the caller can keep the prefix instead of paying for it twice.
-      assertCompletionNotTruncated(response, 'OpenAI', TRUNCATION_REMEDY, {
-        partialText: generatedText,
-      });
-
-      if (!generatedText) {
-        throw new Error('Invalid API response structure');
-      }
+    return this.transport.generate(prompt, options);
+  }
 
-      return generatedText;
-    } catch (error) {
-      // One name for a stopped request, whatever the client called it; an
-      // abort this service raised already carries its partial text.
-      const thrown =
-        isAbortError(error, signal) && error?.name !== 'AbortError'
-          ? abortedGenerationError(error, '')
-          : error;
-      console.error(`Error generating text with OpenAI: ${thrown.message}`);
-      console.debug(thrown);
-      throw thrown;
+  /**
+   * This provider's generateText() request. The transport sends it, streams
+   * it and reads the answer; what differs between the providers is here.
+   * @param {string} prompt
+   * @param {object} options - see generateText()
+   * @returns {Object} The chat completions request
+   */
+  _buildGenerationRequest(prompt, options) {
+    const model =
+      modelOverride(options) || process.env.OPENAI_MODEL || config.openai.model;
+    const messages = [];
+    if (hasSystemPrompt(options)) {
+      messages.push({ role: 'system', content: options.systemPrompt });
     }
+    messages.push({ role: 'user', content: prompt });
+
+    const request = {
+      model: model,
+      messages,
+      temperature: hasNumber(options.temperature)
+        ? options.temperature
+        : config.aiTemperatureGeneration,
+    };
+    // Without a cap the model answers with whatever the deployment allows,
+    // which is what every caller before the AI review wanted.
+    if (hasNumber(options.maxTokens) && options.maxTokens > 0) {
+      request.max_tokens = Math.floor(options.maxTokens);
+    }
+    // The one thinking switch this API has, and it exists on the reasoning
+    // families only: an ordinary model answers 400 for the field, so it is
+    // sent for those names and for no others, and only when the caller
+    // asked for no thinking.
+    if (reasoningEnabled(options) === false) {
+      const effort = reasoningEffortForOpenAi(model);
+      if (effort) request.reasoning_effort = effort;
+    }
+
+    return request;
   }
 
   async checkStatus() {
diff --git a/tests/test-llm-transport-request-mapping.js b/tests/test-llm-transport-request-mapping.js
index 8daebde9..352c4120 100644
--- a/tests/test-llm-transport-request-mapping.js
+++ b/tests/test-llm-transport-request-mapping.js
@@ -570,6 +570,309 @@ const ANALYSIS_REQUEST = {
     }
   );
 
+  /* --- generateText(): the shared half in the transport (step 5b) ---------- */
+
+  const generationTransport = (respond, generation = {}) => {
+    const requests = [];
+    const calls = [];
+    const transport = new ChatCompletionsTransport({
+      label: 'Fake',
+      createClient: () => ({
+        chat: {
+          completions: {
+            create: async (request, requestOptions) => {
+              requests.push(request);
+              calls.push(requestOptions);
+              return respond(request);
+            },
+          },
+        },
+      }),
+      generation: {
+        notReadyMessage: 'Fake client not initialized - missing API key',
+        remedy: 'Raise Response Tokens (RESPONSE_TOKENS).',
+        buildRequest: async (prompt) => ({
+          model: 'fake-model',
+          messages: [{ role: 'user', content: prompt }],
+          temperature: 0.7,
+        }),
+        ...generation,
+      },
+    });
+    return { transport, requests, calls };
+  };
+
+  await check('chat generate: sends what the provider built', async () => {
+    const { transport, requests, calls } = generationTransport(async () =>
+      completion('<think>hm</think>The answer.')
+    );
+    const text = await transport.generate('hello');
+    assert.strictEqual(text, 'The answer.');
+    assert.deepStrictEqual(requests, [
+      {
+        model: 'fake-model',
+        messages: [{ role: 'user', content: 'hello' }],
+        temperature: 0.7,
+      },
+    ]);
+    assert.deepStrictEqual(calls, [undefined], 'no request options');
+    assert.deepStrictEqual(transport.lastGenerateTextUsage, {
+      promptTokens: 9,
+      completionTokens: 3,
+      totalTokens: 12,
+      reasoningTokens: null,
+    });
+  });
+
+  await check(
+    'chat generate: a signal rides in the request options',
+    async () => {
+      const { transport, calls } = generationTransport(async () =>
+        completion('ok')
+      );
+      const controller = new AbortController();
+      await transport.generate('hello', { signal: controller.signal });
+      assert.deepStrictEqual(calls, [{ signal: controller.signal }]);
+    }
+  );
+
+  await check('chat generate: not ready names the provider', async () => {
+    const transport = new ChatCompletionsTransport({
+      label: 'Fake',
+      createClient: () => null,
+      generation: {
+        notReadyMessage: 'Fake client not initialized - missing API key',
+        buildRequest: async () => ({}),
+      },
+    });
+    await assert.rejects(
+      () => transport.generate('hello'),
+      (error) =>
+        error.message === 'Fake client not initialized - missing API key'
+    );
+  });
+
+  await check(
+    'chat generate: no text is "Invalid API response structure"',
+    async () => {
+      const { transport } = generationTransport(async () => completion(''));
+      await assert.rejects(
+        () => transport.generate('hello'),
+        (error) => error.message === 'Invalid API response structure'
+      );
+    }
+  );
+
+  await check(
+    'chat generate: a provider may describe the empty answer',
+    async () => {
+      const { transport } = generationTransport(async () => completion(''), {
+        describeEmptyText: () => 'Invalid API response structure: empty.',
+      });
+      await assert.rejects(
+        () => transport.generate('hello'),
+        (error) => error.message === 'Invalid API response structure: empty.'
+      );
+    }
+  );
+
+  await check('chat generate: a cut-off answer keeps its prefix', async () => {
+    const { transport } = generationTransport(async () =>
+      completion('Half an ans', 'length')
+    );
+    await assert.rejects(
+      () => transport.generate('hello'),
+      (error) =>
+        error.code === 'ai_response_truncated' &&
+        error.partialText === 'Half an ans' &&
+        /Raise Response Tokens/.test(error.message)
+    );
+  });
+
+  await check(
+    'ollama generate: maps the request onto /api/generate',
+    async () => {
+      const posts = [];
+      const previousThink = config.ollama.think;
+      config.ollama.think = false;
+      let text;
+      try {
+        text = await ollamaTransport(
+          [{ data: ollamaData({ response: '<think>x</think> Hi there. ' }) }],
+          posts
+        ).generate('hello', {}, { responseTokens: 300, numCtx: 2048 });
+      } finally {
+        config.ollama.think = previousThink;
+      }
+      assert.strictEqual(text, 'Hi there.');
+      assert.deepStrictEqual(posts[0].body, {
+        model: 'llama3',
+        prompt: 'hello',
+        system:
+          "You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.",
+        stream: false,
+        options: {
+          temperature: config.aiTemperatureGeneration,
+          top_p: 0.9,
+          num_predict: 300,
+          num_ctx: 2048,
+        },
+        think: false,
+      });
+    }
+  );
+
+  await check(
+    'ollama generate: options override model, system, temperature, think',
+    async () => {
+      const posts = [];
+      await ollamaTransport(
+        [{ data: ollamaData({ response: 'ok' }) }],
+        posts
+      ).generate(
+        'hello',
+        {
+          model: 'judge',
+          systemPrompt: 'Be brief.',
+          temperature: 0,
+          reasoning: true,
+        },
+        { responseTokens: 300, numCtx: 2048 }
+      );
+      const { body } = posts[0];
+      assert.strictEqual(body.model, 'judge');
+      assert.strictEqual(body.system, 'Be brief.');
+      assert.strictEqual(body.options.temperature, 0);
+      assert.strictEqual(body.think, true);
+    }
+  );
+
+  await check('ollama generate: records the usage it was told', async () => {
+    const transport = ollamaTransport(
+      [{ data: ollamaData({ response: 'ok' }) }],
+      []
+    );
+    await transport.generate(
+      'hello',
+      {},
+      { responseTokens: 300, numCtx: 2048 }
+    );
+    assert.deepStrictEqual(transport.lastGenerateTextUsage, {
+      promptTokens: 9,
+      completionTokens: 3,
+      totalTokens: 12,
+    });
+  });
+
+  await check(
+    'ollama generate: a signal rides in the axios config',
+    async () => {
+      const posts = [];
+      const controller = new AbortController();
+      await ollamaTransport(
+        [{ data: ollamaData({ response: 'ok' }) }],
+        posts
+      ).generate(
+        'hello',
+        { signal: controller.signal },
+        { responseTokens: 300, numCtx: 2048 }
+      );
+      assert.strictEqual(posts[0].requestConfig.signal, controller.signal);
+    }
+  );
+
+  await check(
+    'ollama generate: reasoning false overrules OLLAMA_THINK',
+    async () => {
+      const posts = [];
+      const previousThink = config.ollama.think;
+      config.ollama.think = true;
+      try {
+        await ollamaTransport(
+          [{ data: ollamaData({ response: 'ok' }) }],
+          posts
+        ).generate(
+          'hello',
+          { reasoning: false },
+          { responseTokens: 300, numCtx: 2048 }
+        );
+      } finally {
+        config.ollama.think = previousThink;
+      }
+      assert.strictEqual(posts[0].body.think, false);
+    }
+  );
+
+  await check(
+    'Custom: an empty generated answer says what came back',
+    async () => {
+      customService.client = fakeChatClient([], async () =>
+        completion('', 'stop')
+      );
+      await assert.rejects(
+        () => customService.generateText('hello'),
+        (error) =>
+          error.message.startsWith(
+            'Invalid API response structure: the answer carried no text (finish_reason stop; the response begins {"choices"'
+          )
+      );
+    }
+  );
+
+  /* --- every provider's generateText() goes through its transport ---------- */
+
+  for (const [label, service] of [
+    ['OpenAI', openaiService],
+    ['Azure', azureService],
+    ['Custom', customService],
+    ['Ollama', ollamaService],
+  ]) {
+    await check(
+      `${label}: generateText() is the transport's generate()`,
+      async () => {
+        const original = service.transport.generate;
+        const seen = [];
+        service.transport.generate = async function (...args) {
+          seen.push(args);
+          this.lastGenerateTextUsage = { totalTokens: 42 };
+          return 'from the transport';
+        };
+        try {
+          const text = await service.generateText('hello', { temperature: 0 });
+          assert.strictEqual(text, 'from the transport');
+          assert.strictEqual(seen.length, 1);
+          assert.strictEqual(seen[0][0], 'hello');
+          assert.deepStrictEqual(seen[0][1], { temperature: 0 });
+          assert.deepStrictEqual(service.lastGenerateTextUsage, {
+            totalTokens: 42,
+          });
+        } finally {
+          service.transport.generate = original;
+        }
+      }
+    );
+  }
+
+  await check(
+    "Ollama: generateText() hands the service's numbers on",
+    async () => {
+      const original = ollamaService.transport.generate;
+      let budget;
+      ollamaService.transport.generate = async (prompt, options, numbers) => {
+        budget = numbers;
+        return 'ok';
+      };
+      try {
+        Object.assign(config, { tokenLimit: 1000, responseTokens: 400 });
+        await ollamaService.generateText('x'.repeat(2000), { maxTokens: 250 });
+      } finally {
+        ollamaService.transport.generate = original;
+        Object.assign(config, { tokenLimit: 128000, responseTokens: 1000 });
+      }
+      assert.deepStrictEqual(budget, { responseTokens: 250, numCtx: 1000 });
+    }
+  );
+
   await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
 
   if (failed > 0) {

From 8908cd3fffa427d19d7b36c3cf0737edd97021a4 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Sun, 11 Oct 2026 21:48:30 +0200
Subject: [PATCH 39/53] Record the step 6 decisions and a handoff for whoever
 continues

Background
Steps 1-5 of the LLM service refactoring are done. Step 6 is the first
that changes behaviour on purpose, and it may be carried out by another
agent; the decisions it needs were taken by the maintainer up front so
nobody has to guess them.

Changes
- Step 6 lists the decisions: 7a Azure temperature follows the config on
  analysis and generateText(); 7k an answer without assignable metadata is
  a failure on all four, checked before the field defaults; field defaults
  for all four; 7c refusal detection for all four; row g checkStatus()
  delegated unchanged. Each names the existing test cases that flip.
- New section "Handoff notes for step 6 onwards": what to read, the test
  seams on the four singletons, the rules, how to verify against the
  parent commit, and what "done" means.

Testing
Documentation only; Prettier clean. The test cases the decisions refer to
were checked against tests/.

Impact
None at runtime.

Upstream Status
Local only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/ManualServiceRefactoring.md | 61 ++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 8b74381d..d8a1721e 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -580,6 +580,38 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
 6. **Introduce `documentAnalyzer.js`** and the four composition roots. Reduce
    `services/{openai,azure,custom,ollama}Service.js` to re-export shims. Add `llm-provider-parity`.
    Fix Finding 7a here (Azure temperature) with its test.
+
+   **Decisions, taken by the maintainer on 2026-10-11** — these are the behaviour changes of this
+   step; each needs its own test that fails before and passes after:
+   - **7a — Azure temperature follows the config on both paths.** Analysis sends
+     `config.aiTemperatureAnalysis` (was a hardcoded 0.3); `generateText()` defaults to
+     `config.aiTemperatureGeneration` (was a hardcoded 0.7). Planned test:
+     `llm-azure-temperature-wiring`. Update the case in `llm-transport-request-mapping` that pins
+     0.3 ("Azure: analysis still sends 0.3").
+   - **7k — an answer without assignable metadata is a failure on all four.** Ollama adopts
+     `assertAssignableMetadata()` and so the OCR fallback. Order matters: the check runs on the
+     **parsed answer, before** `withDocumentDefaults()` — the defaults turn a missing `tags` into
+     `[]` and would make every answer pass. Note what the check really is: `tags` must be an array
+     and `correspondent` a string or null; `{ tags: [], correspondent: null }` passes on all four,
+     as it does on the chat providers today. The pinned drift case in
+     `llm-analysis-result-contract` ("NOT reported as a failure (known drift)") flips to the
+     failure assertion and Ollama joins the loop above it; in `llm-ocr-fallback-phrases` Ollama joins the "an answer without metadata routes to OCR" loop it is filtered out of today.
+   - **Field defaults for all four.** `withDocumentDefaults()` shapes every success: the seven
+     fields (`tags` an array, the rest a value or null), unknown fields dropped. Safe for the
+     write-back: `documentProcessingService.buildUpdateData()` falls back to the document's own
+     title/date and skips a null correspondent, type, language and custom fields. The chat cases in
+     `llm-analysis-result-contract`, `llm-response-parser` (`sameDocument()`) and
+     `llm-token-budget` that compare the document exactly need the defaults in their expectation.
+   - **7c — refusal detection for all four.** OpenAI's "i'm sorry" / "i cannot" / "insufficient"
+     check after a failed parse, with `buildInsufficientContent()`, moves into the analyzer. The OCR
+     reason for those documents changes from the invalid-JSON reason to "insufficient content" on
+     azure, custom and Ollama. Check that `llm-response-parser`'s unreadable answers contain none
+     of the three phrases (they do not today), and extend the OpenAI-only refusal case of `llm-ocr-fallback-phrases` (reason `ai_insufficient_content`) to all four.
+   - **Row g — `checkStatus()`** has no caller. The composition roots delegate it unchanged; whether
+     it goes is decided with step 8.
+   - **Not decided, keep as is:** row e/m (timeout normalization only on Custom), row l ("No
+     response data from Ollama API" matches no OCR marker), Ollama's `CONTENT_MAX_LENGTH` cut.
+
 7. **Delete `services/manualService.js`** and repoint `POST /manual/analyze`
    (`routes/setup.js:7477`) at `AIServiceFactory.getService()`, collapsing its four-way switch.
    Clean up what used to trigger Finding 7j while here: all four branches pass `id || []`
@@ -600,6 +632,35 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
 
 ---
 
+## Handoff notes for step 6 onwards
+
+Written when steps 1-5 were done (tag `refactor-step-5-done`), for whoever continues — human or
+agent.
+
+- **Read first:** `/CLAUDE.md`, this plan (Findings, Target Architecture, the done notes of steps
+  1-5), `services/llm/` (`promptBuilder`, `responseParser`, `analysisResult`, `tokenBudget/`,
+  `transports/`) and one provider service, e.g. `services/azureService.js`.
+- **Seams the tests rely on — keep them on the four exported singletons:** `service.client` is
+  replaced directly by many tests; `lastGenerateTextUsage` is read and set; Ollama's `apiUrl`,
+  `model`, `documentAnalysisSchema`, `_buildPrompt()`, `_callOllamaAPI(prompt, system, numCtx,
+schema)`, `_extractOllamaMetrics()`, `_calculatePromptTokenCount()` and `_calculateNumCtx()`
+  are called or stubbed. `ollama-temperature-wiring` mocks axios through `require.cache` for
+  `services/ollamaService.js`, so the axios client must still be created by that module.
+  `ollama-upstream-error` patches the global `setTimeout` for the retry delay.
+- **Rules:** TDD; new tests are plain node scripts in `tests/`, registered in `TESTS` and an
+  `AREAS` entry of `scripts/run-tests.js`; ESLint and Prettier clean on every touched file; one
+  green commit per sub-step, commit body with Background / Changes / Testing / Impact / Upstream
+  Status; English only in the repo.
+- **Verify like this:** `node scripts/run-tests.js --all` takes about five minutes. Some tests
+  fail for reasons unrelated to the code, depending on the machine (native modules, file
+  permissions, line endings in the working copy, a changelog/version mismatch inherited from
+  upstream). Never judge a run by its count: run the suite on the parent commit as well and compare
+  the lists of failed tests — the change is fine when the lists are identical apart from tests it
+  added. Back every new test with a mutation check: break the code it guards, see it fail, restore.
+- **Done means:** the four services are re-export shims of `services/llm/providers/*`; the tests
+  above pass unmodified except where a decision of step 6 says otherwise; `llm-provider-parity`
+  shows byte-identical `document` objects for the same payload on all four.
+
 ## Out of Scope / Follow-ups
 
 - **Relocating tests into `tests/services/`.** Needs `findUnregisteredTests()` made recursive first

From 5afb774d14e776a275835dc432f2938ca2d82aea Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 18:27:54 +0200
Subject: [PATCH 40/53] Let Azure follow the configured temperature on both
 paths

Background
Step 6, decision 7a of services/ManualServiceRefactoring.md: Azure was the
only provider that ignored the temperature settings. Analysis sent a
hardcoded 0.3, generateText() defaulted to a hardcoded 0.7, so
AI_TEMPERATURE_ANALYSIS and AI_TEMPERATURE_GENERATION had no effect on an
Azure deployment.

Changes
- services/azureService.js: analysis sends config.aiTemperatureAnalysis;
  generateText() defaults to config.aiTemperatureGeneration, an explicit
  options.temperature still wins for one call.
- tests/test-llm-azure-temperature-wiring.js (new, area processing): runs the
  singleton with a fake client and test values deliberately different from
  the defaults (0.2 / 1.1), so a hardcoded value cannot pass. Taken from the
  refactor-step-6 branch.
- tests/test-entity-match-ai-service.js: the Azure generation request expects
  config.aiTemperatureGeneration instead of the literal 0.7.
- tests/test-llm-transport-request-mapping.js: the case pinning "Azure:
  analysis still sends 0.3" now expects the configured value.

Testing
llm-azure-temperature-wiring failed 4 of 6 cases before the fix and passes
after it; llm-transport-request-mapping, entity-match-ai-service,
ai-temperature-config and ai-provider-streaming pass. ESLint and Prettier clean on the touched files.

Impact
An Azure deployment now uses the operator's temperatures. With the defaults
(0.3 analysis, 0.7 generation) nothing changes.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 scripts/run-tests.js                        |   2 +
 services/azureService.js                    |   9 +-
 tests/test-entity-match-ai-service.js       |   5 +-
 tests/test-llm-azure-temperature-wiring.js  | 178 ++++++++++++++++++++
 tests/test-llm-transport-request-mapping.js |  10 +-
 5 files changed, 194 insertions(+), 10 deletions(-)
 create mode 100644 tests/test-llm-azure-temperature-wiring.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index fa4cfc20..1fb93278 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -103,6 +103,7 @@ const TESTS = {
   'llm-response-parser': 'test-llm-response-parser.js',
   'llm-token-budget': 'test-llm-token-budget.js',
   'llm-transport-request-mapping': 'test-llm-transport-request-mapping.js',
+  'llm-azure-temperature-wiring': 'test-llm-azure-temperature-wiring.js',
   'custom-field-longtext': 'test-custom-field-longtext.js',
   'document-processing-service': 'test-document-processing-service.js',
   'settings-custom-field-type-conflict':
@@ -261,6 +262,7 @@ const AREAS = {
     'llm-ocr-fallback-phrases',
     'llm-response-parser',
     'llm-transport-request-mapping',
+    'llm-azure-temperature-wiring',
     'rescan-fetch-before-delete',
   ],
   prompts: [
diff --git a/services/azureService.js b/services/azureService.js
index 1ebcb94b..d528bf66 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -188,8 +188,7 @@ class AzureOpenAIService {
         systemPrompt,
         userContent: truncatedContent,
         model,
-        // Hardcoded unlike the other two (Finding 7a); fixed in step 6.
-        temperature: 0.3,
+        temperature: config.aiTemperatureAnalysis,
       });
       let jsonContent = payload;
 
@@ -267,7 +266,7 @@ class AzureOpenAIService {
    * @param {string} prompt - The prompt to generate text from
    * @param {object} [options] - Optional overrides for this one call
    * @param {string} [options.systemPrompt] - Sent as the system message
-   * @param {number} [options.temperature] - Overrides the default of 0.7
+   * @param {number} [options.temperature] - Overrides AI_TEMPERATURE_GENERATION
    * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS
    * @param {AbortSignal} [options.signal] - Cancels the request in flight
    * @param {Function} [options.onProgress] - Streams the answer and reports it
@@ -299,7 +298,9 @@ class AzureOpenAIService {
     const request = {
       model: model,
       messages,
-      temperature: hasNumber(options.temperature) ? options.temperature : 0.7,
+      temperature: hasNumber(options.temperature)
+        ? options.temperature
+        : config.aiTemperatureGeneration,
       // Was a hardcoded 1000, which quarrelled with the setting the same way
       // Ollama's num_predict did — the reservation followed the operator,
       // the limit did not.
diff --git a/tests/test-entity-match-ai-service.js b/tests/test-entity-match-ai-service.js
index c3ef2324..7a1b8f6c 100644
--- a/tests/test-entity-match-ai-service.js
+++ b/tests/test-entity-match-ai-service.js
@@ -878,7 +878,10 @@ async function main() {
           stub.sent.openai.temperature,
           config.aiTemperatureGeneration
         );
-        assert.strictEqual(stub.sent.azure.temperature, 0.7);
+        assert.strictEqual(
+          stub.sent.azure.temperature,
+          config.aiTemperatureGeneration
+        );
         assert.strictEqual(
           stub.sent.azure.max_tokens,
           Number(config.responseTokens)
diff --git a/tests/test-llm-azure-temperature-wiring.js b/tests/test-llm-azure-temperature-wiring.js
new file mode 100644
index 00000000..4bd591eb
--- /dev/null
+++ b/tests/test-llm-azure-temperature-wiring.js
@@ -0,0 +1,178 @@
+/**
+ * Azure carries the configured temperature on both its paths.
+ *
+ * Step 6 decision 7a of the LLM service refactoring
+ * (services/ManualServiceRefactoring.md): analysis sends
+ * config.aiTemperatureAnalysis (was a hardcoded 0.3) and generateText()
+ * defaults to config.aiTemperatureGeneration (was a hardcoded 0.7). The rest
+ * of the request — model, messages, max_tokens, the absence of
+ * reasoning_effort on a plain deployment — is behaviour that OpenAI and
+ * Azure share, so those stay pinned exactly as before and only the
+ * temperature follows the config.
+ *
+ * Like the other provider wiring tests, this runs against the singleton
+ * with a fake client — no network, no live Azure deployment.
+ */
+
+const assert = require('assert');
+
+process.env.AI_PROVIDER = 'azure';
+process.env.OPENAI_API_KEY = 'test-key';
+process.env.OPENAI_MODEL = 'gpt-4';
+process.env.AZURE_API_KEY = 'test-key';
+process.env.AZURE_ENDPOINT = 'https://example.invalid';
+process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment';
+process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1';
+process.env.CUSTOM_API_KEY = 'test-key';
+process.env.CUSTOM_MODEL = 'test-model';
+// Deliberately different from the parseTemperature() defaults (0.3 / 0.7):
+// a hardcoded value that passes here is not actually reading the config.
+process.env.AI_TEMPERATURE_ANALYSIS = '0.2';
+process.env.AI_TEMPERATURE_GENERATION = '1.1';
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+process.env.USE_PROMPT_TAGS = 'no';
+process.env.RESPONSE_TOKENS = '1000';
+delete process.env.CUSTOM_FIELDS;
+delete process.env.CONTENT_MAX_LENGTH;
+delete process.env.OLLAMA_API_KEY;
+
+const config = require('../config/config');
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+const ANSWER = JSON.stringify({
+  title: 'Telekom invoice July',
+  correspondent: 'Telekom Deutschland GmbH',
+  tags: ['invoice'],
+  document_type: 'Rechnung',
+  document_date: '2026-09-03',
+  language: 'de',
+});
+
+function completion(content = ANSWER, finishReason = 'stop') {
+  return {
+    choices: [{ message: { content }, finish_reason: finishReason }],
+    usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 },
+  };
+}
+
+/** Captures the request the SDK is handed; `request` is the mapping target. */
+function fakeChatClient(requests, respond = async () => completion()) {
+  return {
+    chat: {
+      completions: {
+        create: async (request) => {
+          requests.push(request);
+          return respond(request);
+        },
+      },
+    },
+    models: { list: async () => ({ data: [] }) },
+  };
+}
+
+(async () => {
+  console.log('\n=== LLM Azure temperature wiring (Finding 7a, step 6) ===');
+
+  await check('the config really carries the test values', () => {
+    assert.strictEqual(config.aiTemperatureAnalysis, 0.2);
+    assert.strictEqual(config.aiTemperatureGeneration, 1.1);
+  });
+
+  const azureService = require('../services/azureService');
+
+  await check('analysis sends AI_TEMPERATURE_ANALYSIS, not 0.3', async () => {
+    const requests = [];
+    azureService.client = fakeChatClient(requests);
+    const result = await azureService.analyzeDocument(
+      'Invoice from Telekom.',
+      [],
+      [],
+      [],
+      'doc-1'
+    );
+    assert.strictEqual(result.error, undefined, result.error);
+    assert.strictEqual(requests.length, 1, JSON.stringify(requests));
+    const request = requests[0];
+    assert.strictEqual(request.model, 'test-deployment');
+    assert.strictEqual(request.temperature, 0.2);
+    assert.deepStrictEqual(
+      request.messages.map((m) => [m.role, typeof m.content]),
+      [
+        ['system', 'string'],
+        ['user', 'string'],
+      ],
+      JSON.stringify(request.messages)
+    );
+  });
+
+  await check(
+    'generateText() defaults to AI_TEMPERATURE_GENERATION, not 0.7',
+    async () => {
+      const requests = [];
+      azureService.client = fakeChatClient(requests, async () =>
+        completion('The answer.')
+      );
+      const text = await azureService.generateText('hello');
+      assert.strictEqual(text, 'The answer.');
+      assert.strictEqual(requests.length, 1, JSON.stringify(requests));
+      const request = requests[0];
+      assert.strictEqual(request.model, 'test-deployment');
+      assert.strictEqual(request.temperature, 1.1);
+      assert.deepStrictEqual(request.messages, [
+        { role: 'user', content: 'hello' },
+      ]);
+      assert.strictEqual(request.max_tokens, Number(config.responseTokens));
+    }
+  );
+
+  await check('the options override still wins for one call', async () => {
+    const requests = [];
+    azureService.client = fakeChatClient(requests, async () =>
+      completion('The answer.')
+    );
+    await azureService.generateText('hello', {
+      systemPrompt: 'You summarise.',
+      temperature: 0,
+      maxTokens: 42,
+    });
+    assert.strictEqual(requests.length, 1, JSON.stringify(requests));
+    const request = requests[0];
+    assert.strictEqual(request.temperature, 0);
+    assert.strictEqual(request.max_tokens, 42);
+    assert.deepStrictEqual(
+      request.messages.map((m) => m.role),
+      ['system', 'user']
+    );
+  });
+
+  await check(
+    'no reasoning_effort on a plain deployment even when thinking was switched off',
+    async () => {
+      const requests = [];
+      azureService.client = fakeChatClient(requests, async () =>
+        completion('The answer.')
+      );
+      await azureService.generateText('hello', { reasoning: false });
+      assert.strictEqual(requests.length, 1, JSON.stringify(requests));
+      const request = requests[0];
+      assert.ok(!('reasoning_effort' in request), JSON.stringify(request));
+      assert.strictEqual(request.temperature, 1.1);
+    }
+  );
+
+  if (failed > 0) {
+    console.error(`\n${failed} Azure temperature wiring case(s) failed`);
+    process.exit(1);
+  }
+  console.log('\nAll Azure temperature wiring cases passed');
+})();
diff --git a/tests/test-llm-transport-request-mapping.js b/tests/test-llm-transport-request-mapping.js
index 352c4120..f1f32a19 100644
--- a/tests/test-llm-transport-request-mapping.js
+++ b/tests/test-llm-transport-request-mapping.js
@@ -11,9 +11,9 @@
  * The first half pins each transport on its own with a fake client. The
  * second half checks that every provider's analysis request goes through its
  * transport and is mapped the way it was before: OpenAI omits the
- * temperature for o3-mini, Azure still hardcodes 0.3 (Finding 7a, fixed in
- * step 6), Ollama sends format, num_ctx, num_predict and think:false unless
- * OLLAMA_THINK is set.
+ * temperature for o3-mini, Azure sends the configured analysis
+ * temperature (Finding 7a, step 6), Ollama sends format, num_ctx,
+ * num_predict and think:false unless OLLAMA_THINK is set.
  */
 
 const assert = require('assert');
@@ -493,13 +493,13 @@ const ANALYSIS_REQUEST = {
   );
 
   await check(
-    'Azure: analysis still sends 0.3 (Finding 7a, step 6)',
+    'Azure: analysis sends AI_TEMPERATURE_ANALYSIS (7a)',
     async () => {
       const requests = [];
       azureService.client = fakeChatClient(requests);
       await analyse(azureService);
       assert.strictEqual(requests[0].model, 'test-deployment');
-      assert.strictEqual(requests[0].temperature, 0.3);
+      assert.strictEqual(requests[0].temperature, config.aiTemperatureAnalysis);
     }
   );
 

From fa2d516eb1ee2987c6e8c57c3497cb99ae58e1ec Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 18:35:59 +0200
Subject: [PATCH 41/53] Fail an answer without metadata and shape every success
 on all four providers

Background
Step 6, decision 7k and "field defaults for all four" of
services/ManualServiceRefactoring.md. Ollama logged an answer with neither
tags nor a correspondent and returned an all-null document as a success, so
the scan loop wrote nulls back to Paperless and never fell back to OCR; the
three chat providers failed it. The opposite held for the shape of a
success: Ollama filled every document field with a value or null, the chat
providers passed the parsed answer on as it came, missing and invented
fields included.

Changes
- services/ollamaService.js: _processOllamaResponse() runs
  assertAssignableMetadata() on the parsed answer, before
  withDocumentDefaults() would turn a missing `tags` into [] and let every
  answer pass. The "No tags or correspondent found" warning is gone with the
  case it described.
- services/{openai,azure,custom}Service.js: the success document goes
  through withDocumentDefaults(): the seven fields, `tags` an array, the
  rest a value or null, unknown fields dropped.
- services/llm/analysisResult.js: the withDocumentDefaults() comment says
  who uses it now.
- Tests, taken from the refactor-step-6 branch: llm-analysis-result-contract
  flips the pinned drift case ("NOT reported as a failure (known drift)")
  into the failure assertion with Ollama in the loop, and adds a partial
  answer case per provider; llm-ocr-fallback-phrases lets Ollama join "an
  answer without metadata routes to OCR"; llm-response-parser compares every
  provider's document with the defaults (sameDocument()) and adds the 7k and
  partial-answer cases; llm-token-budget expects the default-shaped
  document. Comments written for the failing state now describe the decided
  one.

Testing
Before the change: 4 cases failed in llm-analysis-result-contract, 1 in
llm-ocr-fallback-phrases, 4 in llm-response-parser, 6 in llm-token-budget;
after it all four pass. The 46 provider-related tests of run-tests.js: 43
pass, the other 3 (settings-custom-field-type-conflict and
ocr-queue-add-endpoint crash with exit 3221226505, ai-run-cost) fail the
same way on the merge commit. ESLint and Prettier clean on the touched files.

Impact
An Ollama answer without assignable metadata now reaches the OCR queue like
on the other providers. { tags: [], correspondent: null } still passes on all
four. A chat provider's success carries all seven fields; the write-back is
unaffected, documentProcessingService.buildUpdateData() falls back to the
document's own title/date and skips a null correspondent, type, language and
custom fields.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/azureService.js                   |   3 +-
 services/customService.js                  |   3 +-
 services/llm/analysisResult.js             |   3 +-
 services/ollamaService.js                  |  14 +--
 services/openaiService.js                  |   3 +-
 tests/test-llm-analysis-result-contract.js |  90 +++++++++--------
 tests/test-llm-ocr-fallback-phrases.js     |  11 +--
 tests/test-llm-response-parser.js          | 106 +++++++++++++++++++--
 tests/test-llm-token-budget.js             |  22 +++++
 9 files changed, 187 insertions(+), 68 deletions(-)

diff --git a/services/azureService.js b/services/azureService.js
index d528bf66..e9fd4b8c 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -25,6 +25,7 @@ const {
   buildSuccess,
   buildFailure,
   assertAssignableMetadata,
+  withDocumentDefaults,
 } = require('./llm/analysisResult');
 const { parseAnalysisResponse } = require('./llm/responseParser');
 const responseLogPath = path.join(
@@ -213,7 +214,7 @@ class AzureOpenAIService {
       assertAssignableMetadata(parsedResponse);
 
       return buildSuccess(
-        parsedResponse,
+        withDocumentDefaults(parsedResponse),
         mappedUsage,
         truncatedContent.length < content.length
       );
diff --git a/services/customService.js b/services/customService.js
index fad687e8..863c2bda 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -25,6 +25,7 @@ const {
   buildSuccess,
   buildFailure,
   assertAssignableMetadata,
+  withDocumentDefaults,
 } = require('./llm/analysisResult');
 const { parseAnalysisResponse } = require('./llm/responseParser');
 const responseLogPath = path.join(
@@ -227,7 +228,7 @@ class CustomOpenAIService {
       assertAssignableMetadata(parsedResponse);
 
       return buildSuccess(
-        parsedResponse,
+        withDocumentDefaults(parsedResponse),
         mappedUsage,
         truncatedContent.length < content.length
       );
diff --git a/services/llm/analysisResult.js b/services/llm/analysisResult.js
index d95d6a47..d4a5b345 100644
--- a/services/llm/analysisResult.js
+++ b/services/llm/analysisResult.js
@@ -82,7 +82,8 @@ function assertAssignableMetadata(parsed) {
 
 /**
  * Every document field present: tags an array, everything else a value or
- * null. Ollama's shaping; the chat providers pass the answer on as parsed.
+ * null; fields the model invented are dropped. Shapes every provider's
+ * success, after assertAssignableMetadata() has seen the parsed answer.
  *
  * @param {Object} parsed - The parsed answer
  */
diff --git a/services/ollamaService.js b/services/ollamaService.js
index f5717ac7..a7d7b754 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -12,6 +12,7 @@ const {
 const {
   buildSuccess,
   buildFailure,
+  assertAssignableMetadata,
   withDocumentDefaults,
 } = require('./llm/analysisResult');
 const { buildCustomFieldsBlock } = require('./llm/customFieldsTemplate');
@@ -220,16 +221,6 @@ class OllamaService {
       const parsedResponse = this._processOllamaResponse(response);
       const metrics = this._extractOllamaMetrics(response);
 
-      // Check for missing data
-      if (
-        parsedResponse.tags.length === 0 &&
-        parsedResponse.correspondent === null
-      ) {
-        console.warn(
-          'No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.'
-        );
-      }
-
       // Log the prompt and response
       await this._logPromptAndResponse(systemPrompt, prompt, parsedResponse);
 
@@ -459,6 +450,9 @@ class OllamaService {
     }
     try {
       const { parsed } = parseAnalysisResponse(responseData.response);
+      // On the parsed answer: the defaults would turn a missing `tags` into
+      // [] and let every answer pass.
+      assertAssignableMetadata(parsed);
       return withDocumentDefaults(parsed);
     } catch (error) {
       console.error('Error parsing Ollama response:', error.message);
diff --git a/services/openaiService.js b/services/openaiService.js
index 4234fe80..c1b270d5 100644
--- a/services/openaiService.js
+++ b/services/openaiService.js
@@ -25,6 +25,7 @@ const {
   buildFailure,
   buildInsufficientContent,
   assertAssignableMetadata,
+  withDocumentDefaults,
 } = require('./llm/analysisResult');
 const {
   parseAnalysisResponse,
@@ -212,7 +213,7 @@ class OpenAIService {
       assertAssignableMetadata(parsedResponse);
 
       return buildSuccess(
-        parsedResponse,
+        withDocumentDefaults(parsedResponse),
         mappedUsage,
         truncatedContent.length < content.length
       );
diff --git a/tests/test-llm-analysis-result-contract.js b/tests/test-llm-analysis-result-contract.js
index c4fd0e93..bbe9c248 100644
--- a/tests/test-llm-analysis-result-contract.js
+++ b/tests/test-llm-analysis-result-contract.js
@@ -293,18 +293,17 @@ const assertFailureRecord = (analysis) => {
     );
   }
 
-  /* --- an answer with no usable metadata: the providers disagree --------- */
-
-  /* OpenAI, Azure and Custom raise "AI could not determine assignable
-     metadata: no tags or correspondent found", which is one of the phrases
-     shouldQueueForOcrOnAiError() matches, so the document reaches the OCR
-     queue. Ollama only logs and returns an all-null document as a *success*:
-     no error, metrics populated. The scan loop therefore writes nulls back to
-     Paperless and never falls back to OCR. */
-
-  for (const { label, service, arrangeAnswer } of PROVIDERS.filter(
-    (p) => p.label !== 'Ollama'
-  )) {
+  /* --- an answer with no usable metadata is a failure on all four -------- */
+
+  /* Finding 7k, decided 2026-10-11: all four providers raise "AI could not
+     determine assignable metadata: no tags or correspondent found", which is
+     one of the phrases shouldQueueForOcrOnAiError() matches, so the document
+     reaches the OCR queue. The check runs on the parsed answer, before
+     withDocumentDefaults() would turn a missing `tags` into [] and make
+     every answer pass. { tags: [], correspondent: null } still passes on
+     all four — a model that found nothing is not a broken answer. */
+
+  for (const { label, service, arrangeAnswer } of PROVIDERS) {
     await check(
       `${label}: an answer without usable metadata is a failure`,
       async () => {
@@ -320,34 +319,47 @@ const assertFailureRecord = (analysis) => {
     );
   }
 
-  /* Pinned as it is today so the divergence is visible rather than absent.
-     When the refactor unifies this path (Finding 7c/7k), this case will start
-     failing — that is the point. Replace it with the assertion above and drop
-     the Ollama exclusion from the loop overhead. */
-  await check(
-    'Ollama: an answer without usable metadata is NOT reported as a failure (known drift)',
-    async () => {
-      const ollama = PROVIDERS.find((p) => p.label === 'Ollama');
-      ollama.arrangeAnswer(JSON.stringify({ note: 'nothing usable here' }));
-      const analysis = await analyse(ollama.service);
+  /* --- field defaults shape every success -------------------------------- */
+
+  /* Field defaults for all four (step 6, decided 2026-10-11):
+     withDocumentDefaults() shapes every provider's success to the seven document fields — present values
+     survive, missing ones come back as null (tags as an empty array),
+     unknown fields are dropped. The answer is complete enough to pass
+     assertAssignableMetadata, so this pins only the shaping. The chat
+     providers used to return the parsed answer untouched, missing fields
+     and all. */
+
+  const PARTIAL_ANSWER = {
+    title: 'Telekom',
+    correspondent: 'Telekom Deutschland GmbH',
+    tags: ['invoice'],
+    unknown_field: 'dropped',
+  };
 
-      assert.strictEqual(
-        analysis.error,
-        undefined,
-        'Ollama has started reporting this as an error — unify it and update this case'
-      );
-      assert.deepStrictEqual(analysis.document, {
-        tags: [],
-        correspondent: null,
-        title: null,
-        document_date: null,
-        document_type: null,
-        language: null,
-        custom_fields: null,
-      });
-      assert.ok(analysis.metrics, 'the drift includes reporting metrics');
-    }
-  );
+  for (const { label, service, arrangeAnswer } of PROVIDERS) {
+    await check(
+      `${label}: a partial answer comes back default-shaped`,
+      async () => {
+        arrangeAnswer(JSON.stringify(PARTIAL_ANSWER));
+        const analysis = await analyse(service);
+
+        assert.strictEqual(
+          analysis.error,
+          undefined,
+          `unexpected error: ${analysis.error}`
+        );
+        assert.deepStrictEqual(analysis.document, {
+          tags: PARTIAL_ANSWER.tags,
+          correspondent: PARTIAL_ANSWER.correspondent,
+          title: PARTIAL_ANSWER.title,
+          document_date: null,
+          document_type: null,
+          language: null,
+          custom_fields: null,
+        });
+      }
+    );
+  }
 
   /* OpenAI alone (Finding 7c) answers a refusal with a placeholder document
      instead of the failure record, so the retry loop does not ask again. The
diff --git a/tests/test-llm-ocr-fallback-phrases.js b/tests/test-llm-ocr-fallback-phrases.js
index 5d5c15ad..af455eb2 100644
--- a/tests/test-llm-ocr-fallback-phrases.js
+++ b/tests/test-llm-ocr-fallback-phrases.js
@@ -275,13 +275,10 @@ const assertRoutesToOcr = (analysis, expectedReason) => {
     }
   );
 
-  /* The three OpenAI-compatible services raise "could not determine
-     assignable metadata"; Ollama returns an all-null document with no error
-     at all, so it never reaches OCR. See test-llm-analysis-result-contract.js,
-     where that drift is pinned. */
-  for (const { label, service, answer } of PROVIDERS.filter(
-    (p) => p.label !== 'Ollama'
-  )) {
+  /* All four providers raise "could not determine assignable metadata",
+     so every one of them routes the document to OCR as an invalid
+     response structure. */
+  for (const { label, service, answer } of PROVIDERS) {
     await check(
       `${label}: an answer without metadata routes to OCR`,
       async () => {
diff --git a/tests/test-llm-response-parser.js b/tests/test-llm-response-parser.js
index 321d7850..14c6c1af 100644
--- a/tests/test-llm-response-parser.js
+++ b/tests/test-llm-response-parser.js
@@ -95,11 +95,40 @@ const READABLE_ANSWERS = [
 
 /** The object a READABLE_ANSWERS entry stands for. */
 function expectedFor(name) {
+  if (name === 'partial answer') {
+    /* PARTIAL_ANSWER passed through withDocumentDefaults(): the three
+       assigned fields survive, the missing ones come back as null, and
+       the invented field is dropped. */
+    return {
+      tags: PARTIAL_ANSWER.tags,
+      correspondent: PARTIAL_ANSWER.correspondent,
+      title: PARTIAL_ANSWER.title,
+      document_date: null,
+      document_type: null,
+      language: null,
+      custom_fields: null,
+    };
+  }
+  /* The title ends in "}"; the object's real closing brace is the one
+     after the string. */
   return name === 'braces inside a string value'
     ? { ...RESULT, title: 'Invoice {July} }' }
     : RESULT;
 }
 
+/**
+ * A parseable answer the model leaves incomplete: it assigns tags and a
+ * correspondent, so it passes assertAssignableMetadata, but omits every
+ * other field. Field defaults for all four (step 6) must bring the missing
+ * ones back as null and drop whatever the model invented.
+ */
+const PARTIAL_ANSWER = {
+  tags: ['invoice'],
+  correspondent: 'Telekom Deutschland GmbH',
+  title: 'Telekom',
+  unknown_field: 'invented by the model',
+};
+
 /** Answers no provider can read; none of them may count as a success. */
 const UNREADABLE_ANSWERS = [
   ['no JSON at all', 'The document is a scanned letter.'],
@@ -155,15 +184,22 @@ async function analyze(service, answerWith, answer) {
   );
 }
 
-/** Ollama fills the fields a model left out; the chat providers pass them on. */
+/** Every provider shapes its success with withDocumentDefaults(). */
 function sameDocument(label, actual, expected) {
-  if (label !== 'Ollama') {
-    assert.deepStrictEqual(actual, expected);
-    return;
-  }
-  for (const [key, value] of Object.entries(expected)) {
-    assert.deepStrictEqual(actual[key], value, key);
-  }
+  label; // the signature keeps the call sites; the check is one for all four
+  assert.deepStrictEqual(
+    actual,
+    {
+      tags: expected.tags,
+      correspondent: expected.correspondent,
+      title: expected.title,
+      document_date: expected.document_date,
+      document_type: expected.document_type,
+      language: expected.language,
+      custom_fields: expected.custom_fields,
+    },
+    'every provider returns the seven default-shaped document fields'
+  );
 }
 
 (async () => {
@@ -261,6 +297,60 @@ function sameDocument(label, actual, expected) {
     }
   }
 
+  /* --- Finding 7k: an answer with no usable metadata -------------------- */
+
+  /* Decided 2026-10-11: all four providers fail the answer the same way.
+     Ollama used to log and return the defaults as a *success* (no tags, no
+     correspondent — the document landed in Paperless all null). The check
+     raises on the parsed answer, before withDocumentDefaults() would fill
+     in the missing fields and mask it. */
+  for (const [label, service, answerWith] of PROVIDERS) {
+    await check(
+      `${label}: an answer without tags or correspondent fails`,
+      async () => {
+        const analysis = await analyze(
+          service,
+          answerWith,
+          JSON.stringify({ note: 'nothing assignable here' })
+        );
+        assert.match(
+          analysis.error || '',
+          /could not determine assignable metadata/i
+        );
+        assert.ok(
+          shouldQueueForOcrOnAiError(analysis.error),
+          `the failure routes to OCR: ${analysis.error}`
+        );
+        assert.deepStrictEqual(analysis.document, {
+          tags: [],
+          correspondent: null,
+        });
+        assert.strictEqual(analysis.metrics, null);
+      }
+    );
+  }
+
+  /* --- field defaults: every success comes back default-shaped ---------- */
+
+  /* Decided 2026-10-11: withDocumentDefaults() shapes every provider's
+     success to the seven document fields, present values surviving and
+     the invented ones dropped. Before, only Ollama did this; the three
+     chat providers passed the parsed answer on untouched. */
+  for (const [label, service, answerWith] of PROVIDERS) {
+    await check(
+      `${label}: a partial answer comes back default-shaped`,
+      async () => {
+        const analysis = await analyze(
+          service,
+          answerWith,
+          JSON.stringify(PARTIAL_ANSWER)
+        );
+        assert.strictEqual(analysis.error, undefined, analysis.error);
+        sameDocument(label, analysis.document, expectedFor('partial answer'));
+      }
+    );
+  }
+
   await check('Ollama: an answer object is used as it is', async () => {
     const analysis = await analyze(ollamaService, ollamaClient, RESULT);
     assert.strictEqual(analysis.error, undefined, analysis.error);
diff --git a/tests/test-llm-token-budget.js b/tests/test-llm-token-budget.js
index add7e1c0..e6820bc0 100644
--- a/tests/test-llm-token-budget.js
+++ b/tests/test-llm-token-budget.js
@@ -74,6 +74,26 @@ const ANSWER = JSON.stringify({
   language: 'de',
 });
 
+/** The success document every provider shapes to the seven fields. */
+const DEFAULT_DOCUMENT = {
+  tags: ['invoice'],
+  correspondent: 'Telekom Deutschland GmbH',
+  title: 'Telekom invoice July',
+  document_date: '2026-09-03',
+  document_type: 'Rechnung',
+  language: 'de',
+  custom_fields: null,
+};
+
+/** Field defaults for all four (step 6): the success comes back default-shaped. */
+function assertDefaultDocument(analysis) {
+  assert.deepStrictEqual(
+    analysis.document,
+    DEFAULT_DOCUMENT,
+    'the success document carries the seven default-shaped fields'
+  );
+}
+
 /** Long enough that every limit below has to cut it. */
 const LONG_CONTENT = 'Invoice from Telekom Deutschland GmbH, July. '.repeat(
   400
@@ -297,6 +317,7 @@ async function analyzeRecorded(service, content, record) {
           )
         );
         assert.strictEqual(analysis.truncated, true);
+        assertDefaultDocument(analysis);
       }
     );
 
@@ -310,6 +331,7 @@ async function analyzeRecorded(service, content, record) {
       const user = requests[0].messages.find((m) => m.role === 'user');
       assert.strictEqual(user.content, 'Short invoice.');
       assert.strictEqual(analysis.truncated, false);
+      assertDefaultDocument(analysis);
     });
 
     await check(`${label}: no room left fails before any request`, async () => {

From 519044ae1c8e6403cb359d980b72fa92c35cb597 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 18:43:28 +0200
Subject: [PATCH 42/53] Recognise a refusal as insufficient content on all four
 providers

Background
Step 6, decision 7c of services/ManualServiceRefactoring.md. Only OpenAI
told a refusal ("I'm sorry", "I cannot", "insufficient") apart from broken
JSON: it returned the insufficient-content placeholder, so the document went
to OCR as ai_insufficient_content and the retry loop did not ask the model
again. Azure, Custom and Ollama reported the same answer as invalid JSON.

Changes
- services/llm/responseParser.js: isInsufficientContentAnswer(), the three
  phrases matched on the answer with reasoning and code fences stripped.
- services/openaiService.js: uses it instead of the inline check.
- services/{azure,custom}Service.js: after a failed parse, a refusal
  returns buildInsufficientContent() with the tokens the answer cost.
- services/ollamaService.js: the same after _processOllamaResponse() fails
  to parse; the metrics are read before the parse for it. A parsed answer
  without metadata is still the 7k failure.
- Tests: llm-analysis-result-contract and llm-ocr-fallback-phrases extend
  the OpenAI-only refusal cases to all four (taken from the refactor-step-6
  branch); llm-response-parser checks that its unreadable answers contain
  none of the phrases, as the decision asks, and pins the helper itself.

Testing
Before the change 3 cases failed in each of llm-analysis-result-contract and
llm-ocr-fallback-phrases, and llm-response-parser could not load the
helper; after it all three pass. The 46 provider-related tests: 43 pass, the
same 3 environmental failures as on the merge commit. ESLint and Prettier
clean on the touched files.

Impact
On azure, custom and Ollama a refusal is now recorded as insufficient
content: the document gets the placeholder metadata and the OCR reason
changes from invalid JSON to ai_insufficient_content.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/azureService.js                   | 11 +++-
 services/customService.js                  | 11 +++-
 services/llm/responseParser.js             | 23 ++++++++
 services/ollamaService.js                  | 20 ++++++-
 services/openaiService.js                  | 11 +---
 tests/test-llm-analysis-result-contract.js | 65 +++++++++++-----------
 tests/test-llm-ocr-fallback-phrases.js     | 27 ++++-----
 tests/test-llm-response-parser.js          | 24 ++++++++
 8 files changed, 133 insertions(+), 59 deletions(-)

diff --git a/services/azureService.js b/services/azureService.js
index e9fd4b8c..d7287ccc 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -24,10 +24,14 @@ const {
 const {
   buildSuccess,
   buildFailure,
+  buildInsufficientContent,
   assertAssignableMetadata,
   withDocumentDefaults,
 } = require('./llm/analysisResult');
-const { parseAnalysisResponse } = require('./llm/responseParser');
+const {
+  parseAnalysisResponse,
+  isInsufficientContentAnswer,
+} = require('./llm/responseParser');
 const responseLogPath = path.join(
   process.cwd(),
   'data',
@@ -201,6 +205,11 @@ class AzureOpenAIService {
       } catch (error) {
         console.error(`Failed to parse JSON response: ${error.message}`);
         console.debug(error);
+        if (isInsufficientContentAnswer(jsonContent)) {
+          console.warn(`Document ${id} has insufficient content for analysis`);
+          // Return a default structure instead of throwing to prevent retry loops
+          return buildInsufficientContent(id, mappedUsage);
+        }
         throw error;
       }
 
diff --git a/services/customService.js b/services/customService.js
index 863c2bda..49a5b707 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -24,10 +24,14 @@ const {
 const {
   buildSuccess,
   buildFailure,
+  buildInsufficientContent,
   assertAssignableMetadata,
   withDocumentDefaults,
 } = require('./llm/analysisResult');
-const { parseAnalysisResponse } = require('./llm/responseParser');
+const {
+  parseAnalysisResponse,
+  isInsufficientContentAnswer,
+} = require('./llm/responseParser');
 const responseLogPath = path.join(
   process.cwd(),
   'data',
@@ -215,6 +219,11 @@ class CustomOpenAIService {
       } catch (error) {
         console.log(`[ERROR] Failed to parse JSON response: ${error.message}`);
         console.debug(error);
+        if (isInsufficientContentAnswer(jsonContent)) {
+          console.warn(`Document ${id} has insufficient content for analysis`);
+          // Return a default structure instead of throwing to prevent retry loops
+          return buildInsufficientContent(id, mappedUsage);
+        }
         throw error;
       }
 
diff --git a/services/llm/responseParser.js b/services/llm/responseParser.js
index 0c29e7ac..72fcc647 100644
--- a/services/llm/responseParser.js
+++ b/services/llm/responseParser.js
@@ -23,6 +23,12 @@
 
 const INVALID_JSON = 'Invalid JSON response from API';
 
+/**
+ * What a model writes when it declines a document as too thin to analyse.
+ * Matched only on an answer that did not parse (Finding 7c).
+ */
+const INSUFFICIENT_CONTENT_MARKERS = ["i'm sorry", 'i cannot', 'insufficient'];
+
 /** Drop reasoning blocks and code fences around the answer. */
 function stripResponseWrapping(text) {
   return String(text ?? '')
@@ -32,6 +38,22 @@ function stripResponseWrapping(text) {
     .trim();
 }
 
+/**
+ * Whether an answer that did not parse is a refusal rather than broken JSON.
+ * Such a document is reported as insufficient content: it still reaches OCR,
+ * under that reason, and the retry loop does not ask the model again.
+ *
+ * @param {string} raw - The model's answer
+ * @returns {boolean}
+ */
+function isInsufficientContentAnswer(raw) {
+  const text = stripResponseWrapping(raw).toLowerCase();
+  return (
+    text !== '' &&
+    INSUFFICIENT_CONTENT_MARKERS.some((marker) => text.includes(marker))
+  );
+}
+
 /**
  * The end index of the balanced {...} that opens at `start`, or -1 when the
  * text ends first. Braces inside string literals do not count.
@@ -163,4 +185,5 @@ function parseAnalysisResponse(raw) {
 module.exports = {
   parseAnalysisResponse,
   stripResponseWrapping,
+  isInsufficientContentAnswer,
 };
diff --git a/services/ollamaService.js b/services/ollamaService.js
index a7d7b754..c9c108ed 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -4,7 +4,10 @@ const axios = require('axios');
 const config = require('../config/config');
 const { cacheThumbnail } = require('./thumbnailCache');
 const { buildAnalysisPrompt } = require('./llm/promptBuilder');
-const { parseAnalysisResponse } = require('./llm/responseParser');
+const {
+  parseAnalysisResponse,
+  isInsufficientContentAnswer,
+} = require('./llm/responseParser');
 const charBudget = require('./llm/tokenBudget/charBudget');
 const {
   OllamaGenerateTransport,
@@ -12,6 +15,7 @@ const {
 const {
   buildSuccess,
   buildFailure,
+  buildInsufficientContent,
   assertAssignableMetadata,
   withDocumentDefaults,
 } = require('./llm/analysisResult');
@@ -218,8 +222,20 @@ class OllamaService {
       );
 
       // Process response
-      const parsedResponse = this._processOllamaResponse(response);
       const metrics = this._extractOllamaMetrics(response);
+      let parsedResponse;
+      try {
+        parsedResponse = this._processOllamaResponse(response);
+      } catch (error) {
+        if (
+          error.message.startsWith('Invalid JSON response from API') &&
+          isInsufficientContentAnswer(response.response)
+        ) {
+          console.warn(`Document ${id} has insufficient content for analysis`);
+          return buildInsufficientContent(id, metrics);
+        }
+        throw error;
+      }
 
       // Log the prompt and response
       await this._logPromptAndResponse(systemPrompt, prompt, parsedResponse);
diff --git a/services/openaiService.js b/services/openaiService.js
index c1b270d5..e6a32b3a 100644
--- a/services/openaiService.js
+++ b/services/openaiService.js
@@ -29,7 +29,7 @@ const {
 } = require('./llm/analysisResult');
 const {
   parseAnalysisResponse,
-  stripResponseWrapping,
+  isInsufficientContentAnswer,
 } = require('./llm/responseParser');
 const responseLogPath = path.join(
   process.cwd(),
@@ -188,14 +188,7 @@ class OpenAIService {
       } catch (error) {
         console.error(`Failed to parse JSON response: ${error.message}`);
         console.debug(error);
-        // Check if the response indicates the content is too minimal or API can't process it
-        jsonContent = stripResponseWrapping(jsonContent);
-        if (
-          jsonContent &&
-          (jsonContent.toLowerCase().includes("i'm sorry") ||
-            jsonContent.toLowerCase().includes('i cannot') ||
-            jsonContent.toLowerCase().includes('insufficient'))
-        ) {
+        if (isInsufficientContentAnswer(jsonContent)) {
           console.warn(`Document ${id} has insufficient content for analysis`);
           // Return a default structure instead of throwing to prevent retry loops
           return buildInsufficientContent(id, mappedUsage);
diff --git a/tests/test-llm-analysis-result-contract.js b/tests/test-llm-analysis-result-contract.js
index bbe9c248..ca9d46ae 100644
--- a/tests/test-llm-analysis-result-contract.js
+++ b/tests/test-llm-analysis-result-contract.js
@@ -361,37 +361,40 @@ const assertFailureRecord = (analysis) => {
     );
   }
 
-  /* OpenAI alone (Finding 7c) answers a refusal with a placeholder document
-     instead of the failure record, so the retry loop does not ask again. The
-     error phrase still routes it to OCR; the document is what the write-back
-     would set, so it is pinned field by field. */
-  await check(
-    'OpenAI: a refusal returns the insufficient-content placeholder',
-    async () => {
-      const openai = PROVIDERS.find((p) => p.label === 'OpenAI');
-      openai.arrangeAnswer("I'm sorry, I cannot analyse this document.");
-      const analysis = await analyse(openai.service);
-
-      assert.strictEqual(
-        analysis.error,
-        'Insufficient content for AI analysis'
-      );
-      assert.deepStrictEqual(analysis.document, {
-        tags: [],
-        correspondent: 'Unknown',
-        title: `Document ${DOCUMENT_ID}`,
-        document_date: new Date().toISOString().split('T')[0],
-        document_type: 'Document',
-        language: 'und',
-      });
-      assert.deepStrictEqual(analysis.metrics, {
-        promptTokens: 900,
-        completionTokens: 120,
-        totalTokens: 1020,
-      });
-      assert.strictEqual(analysis.truncated, false);
-    }
-  );
+  /* Finding 7c: a refusal returns the insufficient-content placeholder on
+     all four providers, not the failure record, so the retry loop does not
+     ask again. The error phrase still routes it to OCR; the document is what
+     the write-back would set, so it is pinned field by field. Metrics are
+     the tokens the refusal cost — the stubs all report 900 prompt / 120
+     completion / 1020 total. */
+  for (const { label, service, arrangeAnswer } of PROVIDERS) {
+    await check(
+      `${label}: a refusal returns the insufficient-content placeholder`,
+      async () => {
+        arrangeAnswer("I'm sorry, I cannot analyse this document.");
+        const analysis = await analyse(service);
+
+        assert.strictEqual(
+          analysis.error,
+          'Insufficient content for AI analysis'
+        );
+        assert.deepStrictEqual(analysis.document, {
+          tags: [],
+          correspondent: 'Unknown',
+          title: `Document ${DOCUMENT_ID}`,
+          document_date: new Date().toISOString().split('T')[0],
+          document_type: 'Document',
+          language: 'und',
+        });
+        assert.deepStrictEqual(analysis.metrics, {
+          promptTokens: 900,
+          completionTokens: 120,
+          totalTokens: 1020,
+        });
+        assert.strictEqual(analysis.truncated, false);
+      }
+    );
+  }
 
   await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
 
diff --git a/tests/test-llm-ocr-fallback-phrases.js b/tests/test-llm-ocr-fallback-phrases.js
index af455eb2..181bf4e8 100644
--- a/tests/test-llm-ocr-fallback-phrases.js
+++ b/tests/test-llm-ocr-fallback-phrases.js
@@ -259,21 +259,18 @@ const assertRoutesToOcr = (analysis, expectedReason) => {
     }
   );
 
-  /* Only OpenAI implements the "insufficient content" early return today
-     (Finding 7c), and it triggers on the *answer*, not on the document: an
-     unparseable reply containing "i'm sorry" / "i cannot" / "insufficient".
-     The refactor is meant to extend it to the other three. */
-  await check(
-    'OpenAI: a refusal routes to OCR as insufficient content',
-    async () => {
-      const openai = PROVIDERS.find((p) => p.label === 'OpenAI');
-      openai.answer("I'm sorry, there is insufficient text to analyse.");
-      assertRoutesToOcr(
-        await analyse(openai.service),
-        'ai_insufficient_content'
-      );
-    }
-  );
+  /* Finding 7c: all four providers implement the "insufficient content"
+     early return — an unparseable answer containing "i'm sorry" / "i cannot"
+     / "insufficient" routes to OCR under that reason, not as invalid JSON. */
+  for (const { label, service, answer } of PROVIDERS) {
+    await check(
+      `${label}: a refusal routes to OCR as insufficient content`,
+      async () => {
+        answer("I'm sorry, there is insufficient text to analyse.");
+        assertRoutesToOcr(await analyse(service), 'ai_insufficient_content');
+      }
+    );
+  }
 
   /* All four providers raise "could not determine assignable metadata",
      so every one of them routes the document to OCR as an invalid
diff --git a/tests/test-llm-response-parser.js b/tests/test-llm-response-parser.js
index 14c6c1af..7af13839 100644
--- a/tests/test-llm-response-parser.js
+++ b/tests/test-llm-response-parser.js
@@ -39,6 +39,7 @@ const {
 const {
   parseAnalysisResponse,
   stripResponseWrapping,
+  isInsufficientContentAnswer,
 } = require('../services/llm/responseParser');
 const { shouldQueueForOcrOnAiError } = require('../services/serviceUtils');
 const openaiService = require('../services/openaiService');
@@ -246,6 +247,29 @@ function sameDocument(label, actual, expected) {
     }
   );
 
+  /* Finding 7c: an unreadable answer that reads like a refusal is reported
+     as insufficient content instead. The answers above must stay plain
+     failures, or the cases further down would test the refusal path. */
+  for (const [name, answer] of UNREADABLE_ANSWERS) {
+    await check(`${name} is not a refusal`, () => {
+      assert.strictEqual(isInsufficientContentAnswer(answer), false);
+    });
+  }
+
+  await check('a refusal is recognised, case and wrapping aside', () => {
+    for (const answer of [
+      "I'm sorry, I cannot help with this document.",
+      'I CANNOT analyse this.',
+      '<think>too short</think>The content is insufficient.',
+      '```\nInsufficient text.\n```',
+    ]) {
+      assert.strictEqual(isInsufficientContentAnswer(answer), true, answer);
+    }
+    for (const answer of ['', null, undefined, 'A scanned letter.']) {
+      assert.strictEqual(isInsufficientContentAnswer(answer), false, answer);
+    }
+  });
+
   for (const [name, answer] of UNREADABLE_ANSWERS) {
     await check(`fails on ${name} with the OCR fallback phrase`, () => {
       assert.throws(

From 36280876eaffa63d43007473d7671d096446a702 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 18:45:27 +0200
Subject: [PATCH 43/53] Pin byte-identical analysis results across the four
 providers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 6 is done when "llm-provider-parity shows byte-identical document
objects for the same payload on all four" (services/ManualServiceRefactoring.md,
handoff notes). With decisions 7a, 7k, the field defaults and 7c in place the
four providers now agree on every answer, so the parity can be pinned before
they are collapsed onto one DocumentAnalyzer; the structural commits that
follow must keep it green. The test of the same name on the refactor-step-6
branch compared method names and arities only and passed with Azure's
defaults removed; this one compares results.

Changes
- tests/test-llm-provider-parity.js (new, area processing): twelve answers —
  complete, partial, invented field, nothing found, no metadata, tags as a
  string, code fence, reasoning block, trailing comma, refusal, no JSON, cut
  off — sent to all four providers through stubbed clients that report the
  same token counts. The JSON of each analyzeDocument() result must be the
  same string on all four. Transport failures (rows e/m) and the empty
  payload (row l) are left out on purpose; step 6 keeps them as they are.
- scripts/run-tests.js: registered as llm-provider-parity.

Testing
Passes on all twelve cases. Mutation check: returning the parsed answer
without withDocumentDefaults() on Azure alone fails seven of them, the key
order included. ESLint and Prettier clean.

Impact
Tests only.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 scripts/run-tests.js              |   2 +
 tests/test-llm-provider-parity.js | 179 ++++++++++++++++++++++++++++++
 2 files changed, 181 insertions(+)
 create mode 100644 tests/test-llm-provider-parity.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index 1fb93278..2d73088d 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -104,6 +104,7 @@ const TESTS = {
   'llm-token-budget': 'test-llm-token-budget.js',
   'llm-transport-request-mapping': 'test-llm-transport-request-mapping.js',
   'llm-azure-temperature-wiring': 'test-llm-azure-temperature-wiring.js',
+  'llm-provider-parity': 'test-llm-provider-parity.js',
   'custom-field-longtext': 'test-custom-field-longtext.js',
   'document-processing-service': 'test-document-processing-service.js',
   'settings-custom-field-type-conflict':
@@ -263,6 +264,7 @@ const AREAS = {
     'llm-response-parser',
     'llm-transport-request-mapping',
     'llm-azure-temperature-wiring',
+    'llm-provider-parity',
     'rescan-fetch-before-delete',
   ],
   prompts: [
diff --git a/tests/test-llm-provider-parity.js b/tests/test-llm-provider-parity.js
new file mode 100644
index 00000000..3750f06b
--- /dev/null
+++ b/tests/test-llm-provider-parity.js
@@ -0,0 +1,179 @@
+/**
+ * The same answer gives the same result on all four providers.
+ *
+ * Step 6 of the LLM service refactoring (services/ManualServiceRefactoring.md,
+ * "Done means"): once the four providers share one DocumentAnalyzer, an
+ * answer must come back byte-identical from each of them — the document, the
+ * metrics, the truncation flag and, for a failure, the error and its code.
+ * Every stub reports the same token counts, so a difference anywhere in the
+ * result is a difference in the analysis itself.
+ *
+ * Deliberately left out, because step 6 keeps them as they are (the plan's
+ * "Not decided" list): transport failures, where only Custom normalizes a
+ * timeout (rows e/m), and an empty payload, which Ollama words "No response
+ * data from Ollama API" (row l).
+ */
+
+const assert = require('assert');
+const fs = require('fs').promises;
+
+process.env.AI_PROVIDER = 'openai';
+process.env.OPENAI_API_KEY = 'test-key';
+process.env.OPENAI_MODEL = 'gpt-4';
+process.env.AZURE_API_KEY = 'test-key';
+process.env.AZURE_ENDPOINT = 'https://example.invalid';
+process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment';
+process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1';
+process.env.CUSTOM_API_KEY = 'test-key';
+process.env.CUSTOM_MODEL = 'test-model';
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+process.env.USE_PROMPT_TAGS = 'no';
+delete process.env.CUSTOM_FIELDS;
+delete process.env.CONTENT_MAX_LENGTH;
+
+const {
+  THUMBNAIL_CACHE_DIR,
+  getThumbnailCachePath,
+} = require('../services/thumbnailCachePaths');
+
+const PROVIDERS = {
+  OpenAI: require('../services/openaiService'),
+  Azure: require('../services/azureService'),
+  Custom: require('../services/customService'),
+  Ollama: require('../services/ollamaService'),
+};
+
+/** Own id so the seeded thumbnail cannot collide with a real cached document. */
+const DOCUMENT_ID = 'testLlmProviderParity';
+
+/** Short enough that no provider cuts it. */
+const CONTENT = 'Invoice from Telekom Deutschland GmbH, July 2026.';
+
+/** The answer text every provider receives, by case. */
+const ANSWERS = [
+  [
+    'a complete answer',
+    JSON.stringify({
+      title: 'Telekom invoice July',
+      correspondent: 'Telekom Deutschland GmbH',
+      tags: ['invoice', 'telecom'],
+      document_type: 'Invoice',
+      document_date: '2026-07-31',
+      language: 'en',
+      custom_fields: [{ field_name: 'Amount', value: '49.99' }],
+    }),
+  ],
+  [
+    'a partial answer',
+    JSON.stringify({ tags: ['invoice'], correspondent: 'Telekom' }),
+  ],
+  [
+    'an answer with an invented field',
+    JSON.stringify({ tags: ['invoice'], correspondent: null, mood: 'calm' }),
+  ],
+  [
+    'an answer that found nothing',
+    JSON.stringify({ tags: [], correspondent: null }),
+  ],
+  ['an answer without metadata', JSON.stringify({ title: 'Telekom' })],
+  [
+    'tags given as a string',
+    JSON.stringify({ tags: 'invoice', correspondent: 'Telekom' }),
+  ],
+  [
+    'an answer in a code fence',
+    '```json\n' + JSON.stringify({ tags: ['a'], correspondent: 'C' }) + '\n```',
+  ],
+  [
+    'an answer after a reasoning block',
+    '<think>Looks like an invoice.</think>' +
+      JSON.stringify({ tags: ['a'], correspondent: 'C' }),
+  ],
+  ['an answer with a trailing comma', '{"tags": ["a"], "correspondent": "C",}'],
+  ['a refusal', "I'm sorry, I cannot analyse this document."],
+  ['no JSON at all', 'The document is a scanned letter.'],
+  ['an object cut off mid-way', '{"title": "Telekom", "tags": ["inv'],
+];
+
+/** Every provider answers `text` and reports 9 + 3 tokens. */
+function answerWith(label, text) {
+  if (label === 'Ollama') {
+    PROVIDERS.Ollama.client = {
+      post: async () => ({
+        data: {
+          response: text,
+          done_reason: 'stop',
+          prompt_eval_count: 9,
+          eval_count: 3,
+        },
+      }),
+      get: async () => ({ status: 200, data: { models: [] } }),
+    };
+    return;
+  }
+  PROVIDERS[label].client = {
+    chat: {
+      completions: {
+        create: async () => ({
+          choices: [{ message: { content: text }, finish_reason: 'stop' }],
+          usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 },
+        }),
+      },
+    },
+    models: { list: async () => ({ data: [] }) },
+  };
+}
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+(async () => {
+  console.log('\n=== LLM provider parity ===');
+
+  await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true });
+  await fs.writeFile(
+    getThumbnailCachePath(DOCUMENT_ID),
+    Buffer.from([0x89, 0x50, 0x4e, 0x47])
+  );
+
+  for (const [name, text] of ANSWERS) {
+    await check(`${name}: byte-identical on all four`, async () => {
+      const results = {};
+      for (const label of Object.keys(PROVIDERS)) {
+        answerWith(label, text);
+        results[label] = JSON.stringify(
+          await PROVIDERS[label].analyzeDocument(
+            CONTENT,
+            [],
+            [],
+            [],
+            DOCUMENT_ID
+          )
+        );
+      }
+      for (const label of Object.keys(PROVIDERS)) {
+        assert.strictEqual(
+          results[label],
+          results.OpenAI,
+          `${label} differs from OpenAI`
+        );
+      }
+    });
+  }
+
+  await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true });
+
+  if (failed > 0) {
+    console.error(`\n${failed} provider parity case(s) failed`);
+    process.exit(1);
+  }
+  console.log('\nAll provider parity cases passed');
+})();

From 1552a1572099f7abedd873f4a3241f72a2d953b2 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 18:55:23 +0200
Subject: [PATCH 44/53] Collapse the OpenAI and Azure analysis onto one
 DocumentAnalyzer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 6 of services/ManualServiceRefactoring.md: one concrete business logic,
the providers as composition roots that wire it to their collaborators. With
decisions 7a, 7k, the field defaults and 7c in place, the four
analyzeDocument() copies only differed in what is really provider-specific,
and llm-provider-parity pins that they return byte-identical results. This
commit introduces the analyzer and moves the two providers whose analysis
was already line for line the same; Custom and Ollama follow. The
refactor-step-6 branch had introduced an analyzer for OpenAI only and moved
the other three classes unchanged; this one is written so that all four fit.

Changes
- services/llm/documentAnalyzer.js (new): analyzeDocument() — client check,
  provider content cut, thumbnail, external API data cut to its token share,
  instructions from promptBuilder, budget, layout, prompt log, request,
  parse, refusal, response log, metadata check, field defaults, failure
  record — plus generateText() and checkStatus() as delegations. Everything
  provider-specific arrives through the constructor: label, transport, token
  budget, prompt layout, model, status probe, the not-ready message and the
  optional analysisTemperature, responseSchema, contextWindow,
  prepareContent and describeFailure. Nothing extends it.
- services/llm/promptLayout.js (new): how instructions and document become
  one request — the chat layout (instructions as the system message) and
  Ollama's (instructions in the prompt, its own system prompt), the latter
  byte-identical to ollamaService's; Ollama starts using it in a later
  commit.
- services/llm/tokenBudget/{tiktoken,char}Budget.js: count() and cut() for
  the external API data, so the analyzer measures it the provider's way.
- services/llm/providers/{openai,azure}Provider.js (new): the composition
  roots. Client factory, generateText() request, Azure's stream_options
  version gate and both status probes moved verbatim; OpenAI keeps leaving
  out the temperature for o3-mini.
- services/{openai,azure}Service.js: one-line re-exports, so aiServiceFactory,
  routes/setup.js, server.js and mistralOcrService.js need no change.
- tests/test-llm-document-analyzer.js (new, area processing): the analyzer
  with every collaborator a fake — what it asks of each, in which order.
- tests/test-native-install-log-paths.js: the response log path is checked
  where it now lives (documentAnalyzer.js), Custom's until it moves.

Testing
llm-document-analyzer: 15 cases pass; checking the metadata on the
defaulted answer instead of the parsed one fails it, and llm-provider-parity
as well. The 46 provider-related tests: 43 pass, the same 3 environmental
failures as on the merge commit; native-install-log-paths passes. ESLint
and Prettier clean on the touched files.

Impact
None intended for OpenAI and Azure: same requests, same results
(llm-provider-parity, llm-transport-request-mapping). The log line of a
failed analysis names the provider.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 scripts/run-tests.js                       |   2 +
 services/azureService.js                   | 380 +--------------------
 services/llm/documentAnalyzer.js           | 346 +++++++++++++++++++
 services/llm/promptLayout.js               |  93 +++++
 services/llm/providers/azureProvider.js    | 172 ++++++++++
 services/llm/providers/openaiProvider.js   | 136 ++++++++
 services/llm/tokenBudget/charBudget.js     |  22 +-
 services/llm/tokenBudget/tiktokenBudget.js |  16 +-
 services/openaiService.js                  | 338 +-----------------
 tests/test-llm-document-analyzer.js        | 295 ++++++++++++++++
 tests/test-native-install-log-paths.js     |   4 +-
 11 files changed, 1086 insertions(+), 718 deletions(-)
 create mode 100644 services/llm/documentAnalyzer.js
 create mode 100644 services/llm/promptLayout.js
 create mode 100644 services/llm/providers/azureProvider.js
 create mode 100644 services/llm/providers/openaiProvider.js
 create mode 100644 tests/test-llm-document-analyzer.js

diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index 2d73088d..a6b87e34 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -98,6 +98,7 @@ const TESTS = {
   'custom-field-monetary-normalization':
     'test-custom-field-monetary-normalization.js',
   'llm-analysis-result-contract': 'test-llm-analysis-result-contract.js',
+  'llm-document-analyzer': 'test-llm-document-analyzer.js',
   'llm-ocr-fallback-phrases': 'test-llm-ocr-fallback-phrases.js',
   'llm-prompt-builder': 'test-llm-prompt-builder.js',
   'llm-response-parser': 'test-llm-response-parser.js',
@@ -260,6 +261,7 @@ const AREAS = {
     'reconciliation-service',
     'settings-paperless-url-fallback',
     'llm-analysis-result-contract',
+    'llm-document-analyzer',
     'llm-ocr-fallback-phrases',
     'llm-response-parser',
     'llm-transport-request-mapping',
diff --git a/services/azureService.js b/services/azureService.js
index d7287ccc..2ccda9af 100644
--- a/services/azureService.js
+++ b/services/azureService.js
@@ -1,377 +1,3 @@
-const {
-  calculateTokens,
-  truncateToTokenLimit,
-  writePromptToFile,
-} = require('./serviceUtils');
-const {
-  hasNumber,
-  hasSystemPrompt,
-  modelOverride,
-  reasoningEffortForOpenAi,
-  reasoningEnabled,
-} = require('./aiGenerateOptions');
-const axios = require('axios');
-const AzureOpenAI = require('openai').AzureOpenAI;
-const config = require('../config/config');
-const fs = require('fs').promises;
-const path = require('path');
-const { cacheThumbnail } = require('./thumbnailCache');
-const { buildAnalysisPrompt } = require('./llm/promptBuilder');
-const tiktokenBudget = require('./llm/tokenBudget/tiktokenBudget');
-const {
-  ChatCompletionsTransport,
-} = require('./llm/transports/chatCompletionsTransport');
-const {
-  buildSuccess,
-  buildFailure,
-  buildInsufficientContent,
-  assertAssignableMetadata,
-  withDocumentDefaults,
-} = require('./llm/analysisResult');
-const {
-  parseAnalysisResponse,
-  isInsufficientContentAnswer,
-} = require('./llm/responseParser');
-const responseLogPath = path.join(
-  process.cwd(),
-  'data',
-  'logs',
-  'response.txt'
-);
-/** What an operator can do about an answer that hit the completion cap. */
-const TRUNCATION_REMEDY = 'Raise Response Tokens (RESPONSE_TOKENS).';
-/**
- * The first Azure API version that answers a streamed request with a usage
- * chunk. Older versions reject `stream_options` outright, which would turn a
- * working deployment into a 400 the moment progress is asked for.
- */
-const STREAM_OPTIONS_SINCE = '2024-06-01';
-/** Version aliases that always mean "the newest surface". */
-const ROLLING_API_VERSIONS = ['preview', 'latest', 'v1'];
-
-/**
- * Whether the configured API version understands
- * `stream_options: { include_usage: true }`.
- *
- * Azure versions are dates, so the comparison is a string comparison on the
- * leading YYYY-MM-DD; a preview of a supported date counts as supported.
- * Anything unrecognized is treated as too old and the token count for that
- * deployment stays an estimate, which is the harmless half of being wrong.
- *
- * @param {string} apiVersion
- * @returns {boolean}
- */
-function supportsStreamOptions(apiVersion) {
-  const version = String(apiVersion || '')
-    .trim()
-    .toLowerCase();
-  if (ROLLING_API_VERSIONS.includes(version)) return true;
-  const date = /^(\d{4}-\d{2}-\d{2})/.exec(version);
-  return date ? date[1] >= STREAM_OPTIONS_SINCE : false;
-}
-
-class AzureOpenAIService {
-  constructor() {
-    this.transport = new ChatCompletionsTransport({
-      label: 'AzureOpenAI',
-      generation: {
-        notReadyMessage: 'AzureOpenAI client not initialized - missing API key',
-        remedy: TRUNCATION_REMEDY,
-        buildRequest: (prompt, options) =>
-          this._buildGenerationRequest(prompt, options),
-        // Only where the configured API version takes it; an older one would
-        // answer 400 instead of streaming, so that deployment estimates.
-        streamUsage: () => supportsStreamOptions(config.azure.apiVersion),
-      },
-      createClient: () =>
-        config.aiProvider === 'azure'
-          ? new AzureOpenAI({
-              apiKey: config.azure.apiKey,
-              endpoint: config.azure.endpoint,
-              deploymentName: config.azure.deploymentName,
-              apiVersion: config.azure.apiVersion,
-            })
-          : null,
-    });
-  }
-
-  /** The SDK client. It lives on the transport; tests replace it here. */
-  get client() {
-    return this.transport.client;
-  }
-
-  set client(client) {
-    this.transport.client = client;
-  }
-
-  /** Usage of the last generateText() call; read by the AI review features. */
-  get lastGenerateTextUsage() {
-    return this.transport.lastGenerateTextUsage;
-  }
-
-  set lastGenerateTextUsage(usage) {
-    this.transport.lastGenerateTextUsage = usage;
-  }
-
-  initialize() {
-    this.transport.initialize();
-  }
-
-  async analyzeDocument(
-    content,
-    existingTags = [],
-    existingCorrespondentList = [],
-    existingDocumentTypesList = [],
-    id,
-    customPrompt = null,
-    options = {}
-  ) {
-    try {
-      this.initialize();
-
-      if (!this.client) {
-        throw new Error('AzureOpenAI client not initialized');
-      }
-
-      // Cache the thumbnail for the UI; never let it affect the analysis.
-      await cacheThumbnail(id);
-
-      // Get external API data if available and validate it
-      let externalApiData = options.externalApiData || null;
-      let validatedExternalApiData = null;
-
-      if (externalApiData) {
-        try {
-          validatedExternalApiData =
-            await this._validateAndTruncateExternalApiData(externalApiData);
-          console.log('[DEBUG] External API data validated and included');
-        } catch (error) {
-          console.warn(
-            '[WARNING] External API data validation failed:',
-            error.message
-          );
-          validatedExternalApiData = null;
-        }
-      }
-
-      const model = process.env.AZURE_DEPLOYMENT_NAME;
-      const { systemPrompt, promptTags } = buildAnalysisPrompt({
-        existingTags,
-        existingCorrespondentList,
-        existingDocumentTypesList,
-        externalApiData: validatedExternalApiData,
-        customPrompt,
-      });
-      if (customPrompt) {
-        console.log(
-          '[DEBUG] Replace system prompt with custom prompt via WebHook'
-        );
-      }
-
-      // Calculate tokens AFTER all prompt modifications are complete
-      const { availableTokens } = await tiktokenBudget.reserve(
-        systemPrompt,
-        process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [],
-        model
-      );
-      console.log(
-        `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
-      );
-      console.log(
-        `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`
-      );
-
-      const truncatedContent = await tiktokenBudget.fit(
-        content,
-        availableTokens,
-        model
-      );
-
-      await writePromptToFile(systemPrompt, truncatedContent);
-
-      const { payload, metrics: mappedUsage } = await this.transport.complete({
-        systemPrompt,
-        userContent: truncatedContent,
-        model,
-        temperature: config.aiTemperatureAnalysis,
-      });
-      let jsonContent = payload;
-
-      let parsedResponse;
-      try {
-        const parsedResult = parseAnalysisResponse(jsonContent);
-        parsedResponse = parsedResult.parsed;
-        jsonContent = parsedResult.normalized;
-      } catch (error) {
-        console.error(`Failed to parse JSON response: ${error.message}`);
-        console.debug(error);
-        if (isInsufficientContentAnswer(jsonContent)) {
-          console.warn(`Document ${id} has insufficient content for analysis`);
-          // Return a default structure instead of throwing to prevent retry loops
-          return buildInsufficientContent(id, mappedUsage);
-        }
-        throw error;
-      }
-
-      try {
-        await fs.mkdir(path.dirname(responseLogPath), { recursive: true });
-        await fs.appendFile(responseLogPath, `${jsonContent}\n`);
-      } catch (logError) {
-        console.warn('Failed to write AI response log:', logError.message);
-      }
-
-      assertAssignableMetadata(parsedResponse);
-
-      return buildSuccess(
-        withDocumentDefaults(parsedResponse),
-        mappedUsage,
-        truncatedContent.length < content.length
-      );
-    } catch (error) {
-      console.error(`Failed to analyze document: ${error.message}`);
-      console.debug(error);
-      return buildFailure(error);
-    }
-  }
-
-  /**
-   * Validate and truncate external API data to prevent token overflow
-   * @param {any} apiData - The external API data to validate
-   * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500)
-   * @returns {string} - Validated and potentially truncated data string
-   */
-  async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
-    if (!apiData) {
-      return null;
-    }
-
-    const dataString =
-      typeof apiData === 'object'
-        ? JSON.stringify(apiData, null, 2)
-        : String(apiData);
-
-    // Calculate tokens for the data
-    const dataTokens = await calculateTokens(
-      dataString,
-      process.env.AZURE_DEPLOYMENT_NAME
-    );
-
-    if (dataTokens > maxTokens) {
-      console.warn(
-        `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating`
-      );
-      return await truncateToTokenLimit(
-        dataString,
-        maxTokens,
-        process.env.AZURE_DEPLOYMENT_NAME
-      );
-    }
-
-    console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`);
-    return dataString;
-  }
-
-  /**
-   * Generate text based on a prompt
-   * @param {string} prompt - The prompt to generate text from
-   * @param {object} [options] - Optional overrides for this one call
-   * @param {string} [options.systemPrompt] - Sent as the system message
-   * @param {number} [options.temperature] - Overrides AI_TEMPERATURE_GENERATION
-   * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS
-   * @param {AbortSignal} [options.signal] - Cancels the request in flight
-   * @param {Function} [options.onProgress] - Streams the answer and reports it
-   *   as it grows; see GenerateTextProgress in aiGenerateOptions
-   * @param {boolean} [options.reasoning] - false sends reasoning_effort on the
-   *   deployments that take it and nothing on the others, which have no
-   *   thinking switch; true and null send nothing
-   * @returns {Promise<string>} - The generated text
-   */
-  async generateText(prompt, options = {}) {
-    return this.transport.generate(prompt, options);
-  }
-
-  /**
-   * This provider's generateText() request. The transport sends it, streams
-   * it and reads the answer; what differs between the providers is here.
-   * @param {string} prompt
-   * @param {object} options - see generateText()
-   * @returns {Object} The chat completions request
-   */
-  _buildGenerationRequest(prompt, options) {
-    const model = modelOverride(options) || process.env.AZURE_DEPLOYMENT_NAME;
-    const messages = [];
-    if (hasSystemPrompt(options)) {
-      messages.push({ role: 'system', content: options.systemPrompt });
-    }
-    messages.push({ role: 'user', content: prompt });
-
-    const request = {
-      model: model,
-      messages,
-      temperature: hasNumber(options.temperature)
-        ? options.temperature
-        : config.aiTemperatureGeneration,
-      // Was a hardcoded 1000, which quarrelled with the setting the same way
-      // Ollama's num_predict did — the reservation followed the operator,
-      // the limit did not.
-      max_tokens:
-        hasNumber(options.maxTokens) && options.maxTokens > 0
-          ? Math.floor(options.maxTokens)
-          : Number(config.responseTokens),
-    };
-    // As on OpenAI: the reasoning deployments take reasoning_effort, every
-    // other one answers 400 for it, so it is sent for those names only and
-    // only when the caller asked for no thinking.
-    if (reasoningEnabled(options) === false) {
-      const effort = reasoningEffortForOpenAi(model);
-      if (effort) request.reasoning_effort = effort;
-    }
-
-    return request;
-  }
-
-  async checkStatus() {
-    try {
-      this.initialize();
-
-      if (!this.client) {
-        throw new Error(
-          'Azure OpenAI client not initialized - missing API key'
-        );
-      }
-
-      const model = process.env.AZURE_DEPLOYMENT_NAME;
-
-      // Token-free health check against deployment metadata endpoint.
-      const endpoint = (config.azure.endpoint || '').replace(/\/$/, '');
-      const apiVersion = config.azure.apiVersion;
-      if (!endpoint || !apiVersion || !model) {
-        throw new Error(
-          'Azure endpoint, apiVersion, or deployment name missing'
-        );
-      }
-
-      const url = `${endpoint}/openai/deployments/${model}?api-version=${encodeURIComponent(apiVersion)}`;
-      const response = await axios.get(url, {
-        headers: {
-          'api-key': config.azure.apiKey,
-        },
-        timeout: 10000,
-      });
-
-      if (response.status < 200 || response.status >= 300) {
-        return { status: 'error' };
-      }
-
-      return { status: 'ok', model: model };
-    } catch (error) {
-      console.error(
-        `Error generating text with Azure OpenAI: ${error.message}`
-      );
-      console.debug(error);
-      return { status: 'error', error: error.message };
-    }
-  }
-}
-
-module.exports = new AzureOpenAIService();
+// The Azure OpenAI provider is a composition root over the shared DocumentAnalyzer
+// (services/llm/). This path stays so its callers need no change.
+module.exports = require('./llm/providers/azureProvider');
diff --git a/services/llm/documentAnalyzer.js b/services/llm/documentAnalyzer.js
new file mode 100644
index 00000000..e4d174ef
--- /dev/null
+++ b/services/llm/documentAnalyzer.js
@@ -0,0 +1,346 @@
+'use strict';
+
+/**
+ * The single business logic behind the LLM document analysis.
+ *
+ * Each provider is a composition root (services/llm/providers/): it hands
+ * this class its transport, its token budget, its prompt layout and the few
+ * knobs that really differ, and exports the instance as its singleton.
+ * Nothing extends DocumentAnalyzer. What happens between the document and
+ * the AnalysisResult — prompt, budget, request, parsing, the metadata check,
+ * the field defaults, the refusal — happens here, once, for every provider.
+ */
+
+const path = require('path');
+const fs = require('fs').promises;
+const config = require('../../config/config');
+const { writePromptToFile } = require('../serviceUtils');
+const { hasNumber } = require('../aiGenerateOptions');
+const { cacheThumbnail } = require('../thumbnailCache');
+const defaultPromptBuilder = require('./promptBuilder');
+const defaultResponseParser = require('./responseParser');
+const {
+  buildSuccess,
+  buildFailure,
+  buildInsufficientContent,
+  assertAssignableMetadata,
+  withDocumentDefaults,
+} = require('./analysisResult');
+
+/** Tokens the external API data may take up in the prompt. */
+const EXTERNAL_API_DATA_MAX_TOKENS = 500;
+
+class DocumentAnalyzer {
+  /**
+   * @param {Object} deps
+   * @param {string} deps.label - Provider name for the log
+   * @param {Object} deps.transport - One API family: initialize(), isReady(),
+   *   complete(request), generate(prompt, options[, sizes]), and the
+   *   `client` / `lastGenerateTextUsage` the tests and the AI review read
+   * @param {Object} deps.tokenBudget - reserve(), fit(), count(), cut()
+   * @param {Object} deps.promptLayout - frame() and build(); see promptLayout.js
+   * @param {() => string} deps.resolveModel - The model of this request
+   * @param {(provider: DocumentAnalyzer) => Promise<Object>} deps.checkStatus
+   *   The provider's own status probe (row g: kept as it was)
+   * @param {string} deps.clientNotReadyMessage - The failure without a client
+   * @param {(model: string) => number|undefined} [deps.analysisTemperature]
+   *   undefined leaves the parameter out of the request
+   * @param {Object|null} [deps.responseSchema] - Structured-output schema
+   * @param {(promptText: string, responseTokens: number) => number} [deps.contextWindow]
+   *   The context to request for a prompt; set by the providers that size it
+   *   themselves (Ollama's num_ctx)
+   * @param {(content: string) => string} [deps.prepareContent] - A cut the
+   *   provider applies before anything else (Ollama's CONTENT_MAX_LENGTH)
+   * @param {(error: Error) => string} [deps.describeFailure] - The failure
+   *   message, when the provider rewords one (Custom's timeouts, rows e/m)
+   * @param {Object} [deps.promptBuilder] - buildAnalysisPrompt()
+   * @param {Object} [deps.responseParser] - parseAnalysisResponse(),
+   *   isInsufficientContentAnswer()
+   * @param {(id: string|number) => Promise<boolean>} [deps.cacheThumbnail]
+   */
+  constructor({
+    label,
+    transport,
+    tokenBudget,
+    promptLayout,
+    resolveModel,
+    checkStatus,
+    clientNotReadyMessage,
+    analysisTemperature = () => config.aiTemperatureAnalysis,
+    responseSchema = null,
+    contextWindow = null,
+    prepareContent = (content) => content,
+    describeFailure = (error) => error.message,
+    promptBuilder = defaultPromptBuilder,
+    responseParser = defaultResponseParser,
+    cacheThumbnail: cacheThumbnailFn = cacheThumbnail,
+  }) {
+    this.label = label;
+    this.transport = transport;
+    this.tokenBudget = tokenBudget;
+    this.promptLayout = promptLayout;
+    this.resolveModel = resolveModel;
+    this.clientNotReadyMessage = clientNotReadyMessage;
+    this.analysisTemperature = analysisTemperature;
+    this.responseSchema = responseSchema;
+    this.contextWindow = contextWindow;
+    this.prepareContent = prepareContent;
+    this.describeFailure = describeFailure;
+    this.promptBuilder = promptBuilder;
+    this.responseParser = responseParser;
+    this.cacheThumbnail = cacheThumbnailFn;
+    this._checkStatus = checkStatus;
+  }
+
+  /** The SDK or axios client. It lives on the transport; tests replace it here. */
+  get client() {
+    return this.transport.client;
+  }
+
+  set client(client) {
+    this.transport.client = client;
+  }
+
+  /** Usage of the last generateText() call; read by the AI review features. */
+  get lastGenerateTextUsage() {
+    return this.transport.lastGenerateTextUsage;
+  }
+
+  set lastGenerateTextUsage(usage) {
+    this.transport.lastGenerateTextUsage = usage;
+  }
+
+  initialize() {
+    this.transport.initialize();
+  }
+
+  /**
+   * Analyze one document. Never throws: a failure is returned as the record
+   * the OCR fallback matches on (analysisResult.js).
+   *
+   * @param {string} content - The document text
+   * @param {Array<Object|string>} [existingTags]
+   * @param {Array<Object|string>} [existingCorrespondentList]
+   * @param {Array<Object|string>} [existingDocumentTypesList]
+   * @param {string|number} id - Paperless document id
+   * @param {string|null} [customPrompt] - Replaces the instructions (webhook)
+   * @param {Object} [options]
+   * @param {Object|string} [options.externalApiData] - Context to append
+   * @returns {Promise<Object>} The AnalysisResult
+   */
+  async analyzeDocument(
+    content,
+    existingTags = [],
+    existingCorrespondentList = [],
+    existingDocumentTypesList = [],
+    id,
+    customPrompt = null,
+    options = {}
+  ) {
+    try {
+      this.initialize();
+
+      if (!this.transport.isReady()) {
+        throw new Error(this.clientNotReadyMessage);
+      }
+
+      content = this.prepareContent(content);
+
+      // Cache the thumbnail for the UI; never let it affect the analysis.
+      await this.cacheThumbnail(id);
+
+      const externalApiData = await this._externalApiData(
+        options.externalApiData
+      );
+
+      const model = this.resolveModel();
+      const { systemPrompt: instructions, promptTags } =
+        this.promptBuilder.buildAnalysisPrompt({
+          existingTags,
+          existingCorrespondentList,
+          existingDocumentTypesList,
+          externalApiData,
+          customPrompt,
+        });
+      if (customPrompt) {
+        console.log('[DEBUG] Replace system prompt with custom prompt');
+      }
+
+      // Calculate tokens AFTER all prompt modifications are complete
+      const { availableTokens } = await this.tokenBudget.reserve(
+        this.promptLayout.frame(instructions, { customPrompt }),
+        process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [],
+        model
+      );
+      console.log(
+        `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
+      );
+      console.log(
+        `[DEBUG] External API data: ${externalApiData ? 'included' : 'none'}`
+      );
+
+      const fittedContent = await this.tokenBudget.fit(
+        content,
+        availableTokens,
+        model
+      );
+      const { systemPrompt, userContent } = this.promptLayout.build(
+        instructions,
+        fittedContent,
+        { customPrompt }
+      );
+
+      await writePromptToFile(systemPrompt, userContent);
+
+      const maxResponseTokens = Number(config.responseTokens);
+      const { payload, metrics } = await this.transport.complete({
+        systemPrompt,
+        userContent,
+        model,
+        temperature: this.analysisTemperature(model),
+        maxResponseTokens,
+        schema: this.responseSchema,
+        numCtx: this.contextWindow
+          ? this.contextWindow(userContent, maxResponseTokens)
+          : undefined,
+      });
+
+      let parsed;
+      let normalized;
+      try {
+        ({ parsed, normalized } =
+          this.responseParser.parseAnalysisResponse(payload));
+      } catch (error) {
+        console.error(`Failed to parse JSON response: ${error.message}`);
+        console.debug(error);
+        // A model that declined the document is not a broken answer: the
+        // placeholder keeps the retry loop from asking again, and the
+        // document still reaches OCR (Finding 7c).
+        if (this.responseParser.isInsufficientContentAnswer(payload)) {
+          console.warn(`Document ${id} has insufficient content for analysis`);
+          return buildInsufficientContent(id, metrics);
+        }
+        throw error;
+      }
+
+      await this._logResponse(normalized);
+
+      // On the parsed answer: the defaults would turn a missing `tags` into
+      // [] and let every answer pass (Finding 7k).
+      assertAssignableMetadata(parsed);
+
+      return buildSuccess(
+        withDocumentDefaults(parsed),
+        metrics,
+        String(fittedContent).length < String(content).length
+      );
+    } catch (error) {
+      const message = this.describeFailure(error);
+      console.error(
+        `Failed to analyze document with ${this.label}: ${message}`
+      );
+      console.debug(error);
+      return buildFailure(error, message);
+    }
+  }
+
+  /**
+   * The external API data as prompt text, cut to its token share; null when
+   * there is none or it cannot be read.
+   *
+   * @param {Object|string|undefined} apiData
+   * @returns {Promise<string|null>}
+   */
+  async _externalApiData(apiData) {
+    if (!apiData) {
+      return null;
+    }
+    try {
+      const data = await this._validateAndTruncateExternalApiData(apiData);
+      console.log('[DEBUG] External API data validated and included');
+      return data;
+    } catch (error) {
+      console.warn(
+        '[WARNING] External API data validation failed:',
+        error.message
+      );
+      return null;
+    }
+  }
+
+  /**
+   * Validate and truncate external API data to prevent token overflow
+   * @param {any} apiData - The external API data to validate
+   * @param {number} maxTokens - Maximum tokens allowed for external data
+   * @returns {Promise<string>} - Validated and potentially truncated data string
+   */
+  async _validateAndTruncateExternalApiData(
+    apiData,
+    maxTokens = EXTERNAL_API_DATA_MAX_TOKENS
+  ) {
+    const dataString =
+      typeof apiData === 'object'
+        ? JSON.stringify(apiData, null, 2)
+        : String(apiData);
+
+    const model = this.resolveModel();
+    const dataTokens = await this.tokenBudget.count(dataString, model);
+
+    if (dataTokens > maxTokens) {
+      console.warn(
+        `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating`
+      );
+      return await this.tokenBudget.cut(dataString, maxTokens, model);
+    }
+
+    console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`);
+    return dataString;
+  }
+
+  /** Append the answer to data/logs/response.txt; best effort. */
+  async _logResponse(normalized) {
+    const responseLogPath = path.join(
+      process.cwd(),
+      'data',
+      'logs',
+      'response.txt'
+    );
+    try {
+      await fs.mkdir(path.dirname(responseLogPath), { recursive: true });
+      await fs.appendFile(responseLogPath, `${normalized}\n`);
+    } catch (logError) {
+      console.warn('Failed to write AI response log:', logError.message);
+    }
+  }
+
+  /**
+   * Generate text based on a prompt. The request dialect, streaming and the
+   * usage seam live on the transport and its provider's buildRequest(); a
+   * provider that sizes its own context gets the sizes handed in.
+   *
+   * @param {string} prompt
+   * @param {object} [options] - As read by aiGenerateOptions: systemPrompt,
+   *   temperature, maxTokens, model, signal, onProgress, reasoning
+   * @returns {Promise<string>} The generated text
+   */
+  async generateText(prompt, options = {}) {
+    if (!this.contextWindow) {
+      return this.transport.generate(prompt, options);
+    }
+    const responseTokens =
+      hasNumber(options.maxTokens) && options.maxTokens > 0
+        ? Math.floor(options.maxTokens)
+        : Number(config.responseTokens);
+    return this.transport.generate(prompt, options, {
+      responseTokens,
+      numCtx: this.contextWindow(prompt, responseTokens),
+    });
+  }
+
+  /** The provider's status probe, unchanged (row g; decided with step 8). */
+  checkStatus() {
+    return this._checkStatus(this);
+  }
+}
+
+module.exports = { DocumentAnalyzer };
diff --git a/services/llm/promptLayout.js b/services/llm/promptLayout.js
new file mode 100644
index 00000000..9dc6f29b
--- /dev/null
+++ b/services/llm/promptLayout.js
@@ -0,0 +1,93 @@
+'use strict';
+
+/**
+ * How the analysis instructions and the document are laid out into one
+ * request, per API family.
+ *
+ * promptBuilder.js builds the instructions once for every provider. The chat
+ * providers (openai, azure, custom) send them as the system message and the
+ * document as the user message. Ollama puts them in front of the document in
+ * its prompt and sends a short system prompt of its own. Both layouts are
+ * kept exactly as the providers always sent them, down to the whitespace.
+ *
+ * A layout has two parts:
+ *  - frame(instructions, { customPrompt }): the fixed text the token budget
+ *    reserves room for before the document is cut;
+ *  - build(instructions, content, { customPrompt }): the request's
+ *    { systemPrompt, userContent }.
+ */
+
+const { buildCustomFieldsBlock } = require('./customFieldsTemplate');
+
+/** openai, azure, custom: the instructions are the system message. */
+const chatPromptLayout = {
+  frame(instructions) {
+    return instructions;
+  },
+  build(instructions, content) {
+    return { systemPrompt: instructions, userContent: content };
+  },
+};
+
+/** Ollama's own system prompt; %CUSTOMFIELDS% is the configured block. */
+const OLLAMA_SYSTEM_PROMPT_TEMPLATE = `
+            You are a document analyzer. Your task is to analyze documents and extract relevant information. You do not ask back questions. 
+            YOU MUSTNOT: Ask for additional information or clarification, or ask questions about the document, or ask for additional context.
+            YOU MUSTNOT: Return a response without the desired JSON format.
+            YOU MUST: Return the result EXCLUSIVELY as a JSON object. The Tags, Title and Document_Type MUST be in the language that is used in the document.:
+            IMPORTANT: The custom_fields are optional and can be left out if not needed, only try to fill out the values if you find a matching information in the document.
+            Do not change the value of field_name, only fill out the values. If the field is about money only add the number without currency and always use a . for decimal places.
+            {
+                "title": "xxxxx",
+                "correspondent": "xxxxxxxx",
+                "tags": ["Tag1", "Tag2", "Tag3", "Tag4"],
+                "document_type": "Invoice/Contract/...",
+                "document_date": "YYYY-MM-DD",
+                "language": "en/de/es/...",
+                %CUSTOMFIELDS%
+            }
+            ALWAYS USE THE INFORMATION TO FILL OUT THE JSON OBJECT. DO NOT ASK BACK QUESTIONS.
+        `;
+
+/** Ollama's system prompt with the configured custom fields. */
+function buildOllamaSystemPrompt(customFieldsBlock = buildCustomFieldsBlock()) {
+  return OLLAMA_SYSTEM_PROMPT_TEMPLATE.replace(
+    '%CUSTOMFIELDS%',
+    customFieldsBlock
+  );
+}
+
+/** The instructions, then the document as a JSON string. */
+function ollamaPrompt(instructions, content) {
+  return `${instructions}
+        ${JSON.stringify(content)}
+        `;
+}
+
+/**
+ * Ollama: instructions and document both go into the prompt. A custom
+ * (webhook) prompt is followed by a blank line instead of the indented
+ * block, as it always was.
+ */
+const ollamaPromptLayout = {
+  frame(instructions, { customPrompt } = {}) {
+    return customPrompt
+      ? `${instructions}\n\n`
+      : ollamaPrompt(instructions, '');
+  },
+  build(instructions, content, { customPrompt } = {}) {
+    return {
+      systemPrompt: buildOllamaSystemPrompt(),
+      userContent: customPrompt
+        ? `${instructions}\n\n${JSON.stringify(content)}`
+        : ollamaPrompt(instructions, content),
+    };
+  },
+};
+
+module.exports = {
+  chatPromptLayout,
+  ollamaPromptLayout,
+  ollamaPrompt,
+  buildOllamaSystemPrompt,
+};
diff --git a/services/llm/providers/azureProvider.js b/services/llm/providers/azureProvider.js
new file mode 100644
index 00000000..860ffcfa
--- /dev/null
+++ b/services/llm/providers/azureProvider.js
@@ -0,0 +1,172 @@
+'use strict';
+
+/**
+ * The Azure OpenAI composition root: the DocumentAnalyzer wired to the chat
+ * completions transport and the tiktoken budget. What is Azure's own lives
+ * here — the client, the deployment as the model, the generateText()
+ * request dialect, the API-version gate for streamed usage and the
+ * deployment probe. services/azureService.js re-exports the singleton.
+ */
+
+const axios = require('axios');
+const AzureOpenAI = require('openai').AzureOpenAI;
+const config = require('../../../config/config');
+const {
+  hasNumber,
+  hasSystemPrompt,
+  modelOverride,
+  reasoningEffortForOpenAi,
+  reasoningEnabled,
+} = require('../../aiGenerateOptions');
+const { DocumentAnalyzer } = require('../documentAnalyzer');
+const {
+  ChatCompletionsTransport,
+} = require('../transports/chatCompletionsTransport');
+const tiktokenBudget = require('../tokenBudget/tiktokenBudget');
+const { chatPromptLayout } = require('../promptLayout');
+
+/** What an operator can do about an answer that hit the completion cap. */
+const TRUNCATION_REMEDY = 'Raise Response Tokens (RESPONSE_TOKENS).';
+/**
+ * The first Azure API version that answers a streamed request with a usage
+ * chunk. Older versions reject `stream_options` outright, which would turn a
+ * working deployment into a 400 the moment progress is asked for.
+ */
+const STREAM_OPTIONS_SINCE = '2024-06-01';
+/** Version aliases that always mean "the newest surface". */
+const ROLLING_API_VERSIONS = ['preview', 'latest', 'v1'];
+
+/**
+ * Whether the configured API version understands
+ * `stream_options: { include_usage: true }`.
+ *
+ * Azure versions are dates, so the comparison is a string comparison on the
+ * leading YYYY-MM-DD; a preview of a supported date counts as supported.
+ * Anything unrecognized is treated as too old and the token count for that
+ * deployment stays an estimate, which is the harmless half of being wrong.
+ *
+ * @param {string} apiVersion
+ * @returns {boolean}
+ */
+function supportsStreamOptions(apiVersion) {
+  const version = String(apiVersion || '')
+    .trim()
+    .toLowerCase();
+  if (ROLLING_API_VERSIONS.includes(version)) return true;
+  const date = /^(\d{4}-\d{2}-\d{2})/.exec(version);
+  return date ? date[1] >= STREAM_OPTIONS_SINCE : false;
+}
+
+function createClient() {
+  return config.aiProvider === 'azure'
+    ? new AzureOpenAI({
+        apiKey: config.azure.apiKey,
+        endpoint: config.azure.endpoint,
+        deploymentName: config.azure.deploymentName,
+        apiVersion: config.azure.apiVersion,
+      })
+    : null;
+}
+
+/**
+ * The generateText() request. The transport sends it, streams it and reads
+ * the answer; what differs between the providers is here.
+ *
+ * @param {string} prompt
+ * @param {object} options - see DocumentAnalyzer.generateText()
+ * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS
+ * @param {boolean} [options.reasoning] - false sends reasoning_effort on the
+ *   deployments that take it and nothing on the others, which have no
+ *   thinking switch; true and null send nothing
+ * @returns {Object} The chat completions request
+ */
+function buildGenerationRequest(prompt, options) {
+  const model = modelOverride(options) || process.env.AZURE_DEPLOYMENT_NAME;
+  const messages = [];
+  if (hasSystemPrompt(options)) {
+    messages.push({ role: 'system', content: options.systemPrompt });
+  }
+  messages.push({ role: 'user', content: prompt });
+
+  const request = {
+    model: model,
+    messages,
+    temperature: hasNumber(options.temperature)
+      ? options.temperature
+      : config.aiTemperatureGeneration,
+    // Was a hardcoded 1000, which quarrelled with the setting the same way
+    // Ollama's num_predict did — the reservation followed the operator,
+    // the limit did not.
+    max_tokens:
+      hasNumber(options.maxTokens) && options.maxTokens > 0
+        ? Math.floor(options.maxTokens)
+        : Number(config.responseTokens),
+  };
+  // As on OpenAI: the reasoning deployments take reasoning_effort, every
+  // other one answers 400 for it, so it is sent for those names only and
+  // only when the caller asked for no thinking.
+  if (reasoningEnabled(options) === false) {
+    const effort = reasoningEffortForOpenAi(model);
+    if (effort) request.reasoning_effort = effort;
+  }
+
+  return request;
+}
+
+async function checkStatus(provider) {
+  try {
+    provider.initialize();
+
+    if (!provider.client) {
+      throw new Error('Azure OpenAI client not initialized - missing API key');
+    }
+
+    const model = process.env.AZURE_DEPLOYMENT_NAME;
+
+    // Token-free health check against deployment metadata endpoint.
+    const endpoint = (config.azure.endpoint || '').replace(/\/$/, '');
+    const apiVersion = config.azure.apiVersion;
+    if (!endpoint || !apiVersion || !model) {
+      throw new Error('Azure endpoint, apiVersion, or deployment name missing');
+    }
+
+    const url = `${endpoint}/openai/deployments/${model}?api-version=${encodeURIComponent(apiVersion)}`;
+    const response = await axios.get(url, {
+      headers: {
+        'api-key': config.azure.apiKey,
+      },
+      timeout: 10000,
+    });
+
+    if (response.status < 200 || response.status >= 300) {
+      return { status: 'error' };
+    }
+
+    return { status: 'ok', model: model };
+  } catch (error) {
+    console.error(`Error generating text with Azure OpenAI: ${error.message}`);
+    console.debug(error);
+    return { status: 'error', error: error.message };
+  }
+}
+
+module.exports = new DocumentAnalyzer({
+  label: 'AzureOpenAI',
+  transport: new ChatCompletionsTransport({
+    label: 'AzureOpenAI',
+    createClient,
+    generation: {
+      notReadyMessage: 'AzureOpenAI client not initialized - missing API key',
+      remedy: TRUNCATION_REMEDY,
+      buildRequest: buildGenerationRequest,
+      // Only where the configured API version takes it; an older one would
+      // answer 400 instead of streaming, so that deployment estimates.
+      streamUsage: () => supportsStreamOptions(config.azure.apiVersion),
+    },
+  }),
+  tokenBudget: tiktokenBudget,
+  promptLayout: chatPromptLayout,
+  resolveModel: () => process.env.AZURE_DEPLOYMENT_NAME,
+  checkStatus,
+  clientNotReadyMessage: 'AzureOpenAI client not initialized',
+});
diff --git a/services/llm/providers/openaiProvider.js b/services/llm/providers/openaiProvider.js
new file mode 100644
index 00000000..c4eae7c7
--- /dev/null
+++ b/services/llm/providers/openaiProvider.js
@@ -0,0 +1,136 @@
+'use strict';
+
+/**
+ * The OpenAI composition root: the DocumentAnalyzer wired to the chat
+ * completions transport and the tiktoken budget. What is OpenAI's own lives
+ * here — the client, the model, the generateText() request dialect and the
+ * status probe. services/openaiService.js re-exports the singleton.
+ */
+
+const OpenAI = require('openai');
+const config = require('../../../config/config');
+const {
+  hasNumber,
+  hasSystemPrompt,
+  modelOverride,
+  reasoningEffortForOpenAi,
+  reasoningEnabled,
+} = require('../../aiGenerateOptions');
+const { DocumentAnalyzer } = require('../documentAnalyzer');
+const {
+  ChatCompletionsTransport,
+} = require('../transports/chatCompletionsTransport');
+const tiktokenBudget = require('../tokenBudget/tiktokenBudget');
+const { chatPromptLayout } = require('../promptLayout');
+
+/** What an operator can do about an answer that hit the completion cap. */
+const TRUNCATION_REMEDY =
+  'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.';
+
+function createClient() {
+  if (config.aiProvider === 'ollama') {
+    return new OpenAI({
+      baseURL: config.ollama.apiUrl + '/v1',
+      // Ollama ignores the key unless the endpoint enforces auth (e.g. a
+      // reverse proxy); the OpenAI SDK requires a non-empty value.
+      apiKey: config.ollama.apiKey || 'ollama',
+    });
+  }
+  if (config.aiProvider === 'custom') {
+    return new OpenAI({
+      baseURL: config.custom.apiUrl,
+      apiKey: config.custom.apiKey,
+    });
+  }
+  if (config.aiProvider === 'openai' && config.openai.apiKey) {
+    return new OpenAI({
+      apiKey: config.openai.apiKey,
+    });
+  }
+  return null;
+}
+
+/**
+ * The generateText() request. The transport sends it, streams it and reads
+ * the answer; what differs between the providers is here.
+ *
+ * @param {string} prompt
+ * @param {object} options - see DocumentAnalyzer.generateText()
+ * @param {number} [options.maxTokens] - Completion cap for this call; without
+ *   one the model answers with whatever the deployment allows
+ * @param {boolean} [options.reasoning] - false sends reasoning_effort on the
+ *   model families that take it and nothing on the others, which have no
+ *   thinking switch; true and null send nothing
+ * @returns {Object} The chat completions request
+ */
+function buildGenerationRequest(prompt, options) {
+  const model =
+    modelOverride(options) || process.env.OPENAI_MODEL || config.openai.model;
+  const messages = [];
+  if (hasSystemPrompt(options)) {
+    messages.push({ role: 'system', content: options.systemPrompt });
+  }
+  messages.push({ role: 'user', content: prompt });
+
+  const request = {
+    model: model,
+    messages,
+    temperature: hasNumber(options.temperature)
+      ? options.temperature
+      : config.aiTemperatureGeneration,
+  };
+  // Without a cap the model answers with whatever the deployment allows,
+  // which is what every caller before the AI review wanted.
+  if (hasNumber(options.maxTokens) && options.maxTokens > 0) {
+    request.max_tokens = Math.floor(options.maxTokens);
+  }
+  // The one thinking switch this API has, and it exists on the reasoning
+  // families only: an ordinary model answers 400 for the field, so it is
+  // sent for those names and for no others, and only when the caller
+  // asked for no thinking.
+  if (reasoningEnabled(options) === false) {
+    const effort = reasoningEffortForOpenAi(model);
+    if (effort) request.reasoning_effort = effort;
+  }
+
+  return request;
+}
+
+async function checkStatus(provider) {
+  // Use a token-free endpoint for connectivity/auth checks.
+  try {
+    provider.initialize();
+
+    if (!provider.client) {
+      throw new Error('OpenAI client not initialized - missing API key');
+    }
+
+    await provider.client.models.list();
+    return { status: 'ok', model: process.env.OPENAI_MODEL };
+  } catch (error) {
+    console.error(`Error checking OpenAI status: ${error.message}`);
+    console.debug(error);
+    return { status: 'error', error: error.message };
+  }
+}
+
+module.exports = new DocumentAnalyzer({
+  label: 'OpenAI',
+  transport: new ChatCompletionsTransport({
+    label: 'OpenAI',
+    createClient,
+    generation: {
+      notReadyMessage: 'OpenAI client not initialized - missing API key',
+      remedy: TRUNCATION_REMEDY,
+      buildRequest: buildGenerationRequest,
+    },
+  }),
+  tokenBudget: tiktokenBudget,
+  promptLayout: chatPromptLayout,
+  resolveModel: () => process.env.OPENAI_MODEL,
+  // o3-mini rejects the parameter.
+  analysisTemperature: (model) =>
+    model !== 'o3-mini' ? config.aiTemperatureAnalysis : undefined,
+  checkStatus,
+  clientNotReadyMessage: 'OpenAI client not initialized',
+});
diff --git a/services/llm/tokenBudget/charBudget.js b/services/llm/tokenBudget/charBudget.js
index f26512ba..bcbad26c 100644
--- a/services/llm/tokenBudget/charBudget.js
+++ b/services/llm/tokenBudget/charBudget.js
@@ -10,7 +10,7 @@
  * the context window and the document cut agree by construction (Finding 5
  * of services/ManualServiceRefactoring.md — do not unify it with tiktoken's).
  *
- * Shares its reserve()/fit() shape with tiktokenBudget.js.
+ * Shares its reserve()/fit()/count()/cut() shape with tiktokenBudget.js.
  */
 
 const config = require('../../../config/config');
@@ -90,4 +90,22 @@ async function fit(content, availableTokens) {
   return text.substring(0, maxChars);
 }
 
-module.exports = { estimateTokens, numCtx, reserve, fit };
+/**
+ * Cut a text to a token count, e.g. the external API data.
+ *
+ * @param {string} text
+ * @param {number} maxTokens
+ * @returns {string}
+ */
+function cut(text, maxTokens) {
+  return String(text).substring(0, maxTokens * CHARS_PER_TOKEN);
+}
+
+module.exports = {
+  estimateTokens,
+  count: estimateTokens,
+  cut,
+  numCtx,
+  reserve,
+  fit,
+};
diff --git a/services/llm/tokenBudget/tiktokenBudget.js b/services/llm/tokenBudget/tiktokenBudget.js
index 6ea58460..2549f6bb 100644
--- a/services/llm/tokenBudget/tiktokenBudget.js
+++ b/services/llm/tokenBudget/tiktokenBudget.js
@@ -7,11 +7,13 @@
  * not know), the configured answer is reserved, and the document gets what is
  * left of TOKEN_LIMIT. This block used to be copied into all three services.
  *
- * Shares its reserve()/fit() shape with charBudget.js, which is Ollama's.
+ * Shares its reserve()/fit()/count()/cut() shape with charBudget.js, which is
+ * Ollama's.
  */
 
 const config = require('../../../config/config');
 const {
+  calculateTokens,
   calculateTotalPromptTokens,
   truncateToTokenLimit,
 } = require('../../serviceUtils');
@@ -66,4 +68,14 @@ function fit(content, availableTokens, model) {
   return truncateToTokenLimit(content, availableTokens, model);
 }
 
-module.exports = { reserve, fit };
+/** Tokens of a text, e.g. the external API data. */
+function count(text, model) {
+  return calculateTokens(text, model);
+}
+
+/** Cut a text to a token count, e.g. the external API data. */
+function cut(text, maxTokens, model) {
+  return truncateToTokenLimit(text, maxTokens, model);
+}
+
+module.exports = { reserve, fit, count, cut };
diff --git a/services/openaiService.js b/services/openaiService.js
index e6a32b3a..7c0ba525 100644
--- a/services/openaiService.js
+++ b/services/openaiService.js
@@ -1,335 +1,3 @@
-const {
-  calculateTokens,
-  truncateToTokenLimit,
-  writePromptToFile,
-} = require('./serviceUtils');
-const {
-  hasNumber,
-  hasSystemPrompt,
-  modelOverride,
-  reasoningEffortForOpenAi,
-  reasoningEnabled,
-} = require('./aiGenerateOptions');
-const OpenAI = require('openai');
-const config = require('../config/config');
-const fs = require('fs').promises;
-const path = require('path');
-const { cacheThumbnail } = require('./thumbnailCache');
-const { buildAnalysisPrompt } = require('./llm/promptBuilder');
-const tiktokenBudget = require('./llm/tokenBudget/tiktokenBudget');
-const {
-  ChatCompletionsTransport,
-} = require('./llm/transports/chatCompletionsTransport');
-const {
-  buildSuccess,
-  buildFailure,
-  buildInsufficientContent,
-  assertAssignableMetadata,
-  withDocumentDefaults,
-} = require('./llm/analysisResult');
-const {
-  parseAnalysisResponse,
-  isInsufficientContentAnswer,
-} = require('./llm/responseParser');
-const responseLogPath = path.join(
-  process.cwd(),
-  'data',
-  'logs',
-  'response.txt'
-);
-/** What an operator can do about an answer that hit the completion cap. */
-const TRUNCATION_REMEDY =
-  'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.';
-
-class OpenAIService {
-  constructor() {
-    this.transport = new ChatCompletionsTransport({
-      label: 'OpenAI',
-      generation: {
-        notReadyMessage: 'OpenAI client not initialized - missing API key',
-        remedy: TRUNCATION_REMEDY,
-        buildRequest: (prompt, options) =>
-          this._buildGenerationRequest(prompt, options),
-      },
-      createClient: () => {
-        if (config.aiProvider === 'ollama') {
-          return new OpenAI({
-            baseURL: config.ollama.apiUrl + '/v1',
-            // Ollama ignores the key unless the endpoint enforces auth (e.g. a
-            // reverse proxy); the OpenAI SDK requires a non-empty value.
-            apiKey: config.ollama.apiKey || 'ollama',
-          });
-        }
-        if (config.aiProvider === 'custom') {
-          return new OpenAI({
-            baseURL: config.custom.apiUrl,
-            apiKey: config.custom.apiKey,
-          });
-        }
-        if (config.aiProvider === 'openai' && config.openai.apiKey) {
-          return new OpenAI({
-            apiKey: config.openai.apiKey,
-          });
-        }
-        return null;
-      },
-    });
-  }
-
-  /** The SDK client. It lives on the transport; tests replace it here. */
-  get client() {
-    return this.transport.client;
-  }
-
-  set client(client) {
-    this.transport.client = client;
-  }
-
-  /** Usage of the last generateText() call; read by the AI review features. */
-  get lastGenerateTextUsage() {
-    return this.transport.lastGenerateTextUsage;
-  }
-
-  set lastGenerateTextUsage(usage) {
-    this.transport.lastGenerateTextUsage = usage;
-  }
-
-  initialize() {
-    this.transport.initialize();
-  }
-
-  async analyzeDocument(
-    content,
-    existingTags = [],
-    existingCorrespondentList = [],
-    existingDocumentTypesList = [],
-    id,
-    customPrompt = null,
-    options = {}
-  ) {
-    try {
-      this.initialize();
-
-      if (!this.client) {
-        throw new Error('OpenAI client not initialized');
-      }
-
-      // Cache the thumbnail for the UI; never let it affect the analysis.
-      await cacheThumbnail(id);
-
-      // Get external API data if available and validate it
-      let externalApiData = options.externalApiData || null;
-      let validatedExternalApiData = null;
-
-      if (externalApiData) {
-        try {
-          validatedExternalApiData =
-            await this._validateAndTruncateExternalApiData(externalApiData);
-          console.log('[DEBUG] External API data validated and included');
-        } catch (error) {
-          console.warn(
-            '[WARNING] External API data validation failed:',
-            error.message
-          );
-          validatedExternalApiData = null;
-        }
-      }
-
-      const model = process.env.OPENAI_MODEL;
-      const { systemPrompt, promptTags } = buildAnalysisPrompt({
-        existingTags,
-        existingCorrespondentList,
-        existingDocumentTypesList,
-        externalApiData: validatedExternalApiData,
-        customPrompt,
-      });
-      if (customPrompt) {
-        console.log(
-          '[DEBUG] Replace system prompt with custom prompt via WebHook'
-        );
-      }
-
-      // Calculate tokens AFTER all prompt modifications are complete
-      const { availableTokens } = await tiktokenBudget.reserve(
-        systemPrompt,
-        process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [],
-        model
-      );
-      console.log(
-        `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
-      );
-      console.log(
-        `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`
-      );
-
-      const truncatedContent = await tiktokenBudget.fit(
-        content,
-        availableTokens,
-        model
-      );
-
-      await writePromptToFile(systemPrompt, truncatedContent);
-
-      const { payload, metrics: mappedUsage } = await this.transport.complete({
-        systemPrompt,
-        userContent: truncatedContent,
-        model,
-        // o3-mini rejects the parameter.
-        temperature:
-          model !== 'o3-mini' ? config.aiTemperatureAnalysis : undefined,
-      });
-      let jsonContent = payload;
-
-      let parsedResponse;
-      try {
-        const parsedResult = parseAnalysisResponse(jsonContent);
-        parsedResponse = parsedResult.parsed;
-        jsonContent = parsedResult.normalized;
-      } catch (error) {
-        console.error(`Failed to parse JSON response: ${error.message}`);
-        console.debug(error);
-        if (isInsufficientContentAnswer(jsonContent)) {
-          console.warn(`Document ${id} has insufficient content for analysis`);
-          // Return a default structure instead of throwing to prevent retry loops
-          return buildInsufficientContent(id, mappedUsage);
-        }
-        throw error;
-      }
-
-      try {
-        await fs.mkdir(path.dirname(responseLogPath), { recursive: true });
-        await fs.appendFile(responseLogPath, `${jsonContent}\n`);
-      } catch (logError) {
-        console.warn('Failed to write AI response log:', logError.message);
-      }
-
-      assertAssignableMetadata(parsedResponse);
-
-      return buildSuccess(
-        withDocumentDefaults(parsedResponse),
-        mappedUsage,
-        truncatedContent.length < content.length
-      );
-    } catch (error) {
-      console.error(`Failed to analyze document: ${error.message}`);
-      console.debug(error);
-      return buildFailure(error);
-    }
-  }
-
-  /**
-   * Validate and truncate external API data to prevent token overflow
-   * @param {any} apiData - The external API data to validate
-   * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500)
-   * @returns {string} - Validated and potentially truncated data string
-   */
-  async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
-    if (!apiData) {
-      return null;
-    }
-
-    const dataString =
-      typeof apiData === 'object'
-        ? JSON.stringify(apiData, null, 2)
-        : String(apiData);
-
-    // Calculate tokens for the data
-    const dataTokens = await calculateTokens(
-      dataString,
-      process.env.OPENAI_MODEL
-    );
-
-    if (dataTokens > maxTokens) {
-      console.warn(
-        `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating`
-      );
-      return await truncateToTokenLimit(
-        dataString,
-        maxTokens,
-        process.env.OPENAI_MODEL
-      );
-    }
-
-    console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`);
-    return dataString;
-  }
-
-  /**
-   * Generate text based on a prompt
-   * @param {string} prompt - The prompt to generate text from
-   * @param {object} [options] - Optional overrides for this one call
-   * @param {string} [options.systemPrompt] - Sent as the system message
-   * @param {number} [options.temperature] - Overrides AI_TEMPERATURE_GENERATION
-   * @param {number} [options.maxTokens] - Completion cap for this call
-   * @param {AbortSignal} [options.signal] - Cancels the request in flight
-   * @param {Function} [options.onProgress] - Streams the answer and reports it
-   *   as it grows; see GenerateTextProgress in aiGenerateOptions
-   * @param {boolean} [options.reasoning] - false sends reasoning_effort on the
-   *   model families that take it and nothing on the others, which have no
-   *   thinking switch; true and null send nothing
-   * @returns {Promise<string>} - The generated text
-   */
-  async generateText(prompt, options = {}) {
-    return this.transport.generate(prompt, options);
-  }
-
-  /**
-   * This provider's generateText() request. The transport sends it, streams
-   * it and reads the answer; what differs between the providers is here.
-   * @param {string} prompt
-   * @param {object} options - see generateText()
-   * @returns {Object} The chat completions request
-   */
-  _buildGenerationRequest(prompt, options) {
-    const model =
-      modelOverride(options) || process.env.OPENAI_MODEL || config.openai.model;
-    const messages = [];
-    if (hasSystemPrompt(options)) {
-      messages.push({ role: 'system', content: options.systemPrompt });
-    }
-    messages.push({ role: 'user', content: prompt });
-
-    const request = {
-      model: model,
-      messages,
-      temperature: hasNumber(options.temperature)
-        ? options.temperature
-        : config.aiTemperatureGeneration,
-    };
-    // Without a cap the model answers with whatever the deployment allows,
-    // which is what every caller before the AI review wanted.
-    if (hasNumber(options.maxTokens) && options.maxTokens > 0) {
-      request.max_tokens = Math.floor(options.maxTokens);
-    }
-    // The one thinking switch this API has, and it exists on the reasoning
-    // families only: an ordinary model answers 400 for the field, so it is
-    // sent for those names and for no others, and only when the caller
-    // asked for no thinking.
-    if (reasoningEnabled(options) === false) {
-      const effort = reasoningEffortForOpenAi(model);
-      if (effort) request.reasoning_effort = effort;
-    }
-
-    return request;
-  }
-
-  async checkStatus() {
-    // Use a token-free endpoint for connectivity/auth checks.
-    try {
-      this.initialize();
-
-      if (!this.client) {
-        throw new Error('OpenAI client not initialized - missing API key');
-      }
-
-      await this.client.models.list();
-      return { status: 'ok', model: process.env.OPENAI_MODEL };
-    } catch (error) {
-      console.error(`Error checking OpenAI status: ${error.message}`);
-      console.debug(error);
-      return { status: 'error', error: error.message };
-    }
-  }
-}
-
-module.exports = new OpenAIService();
+// The OpenAI provider is a composition root over the shared DocumentAnalyzer
+// (services/llm/). This path stays so its callers need no change.
+module.exports = require('./llm/providers/openaiProvider');
diff --git a/tests/test-llm-document-analyzer.js b/tests/test-llm-document-analyzer.js
new file mode 100644
index 00000000..5eba84dd
--- /dev/null
+++ b/tests/test-llm-document-analyzer.js
@@ -0,0 +1,295 @@
+/**
+ * The DocumentAnalyzer on its own, every collaborator a fake.
+ *
+ * Step 6 of the LLM service refactoring (services/ManualServiceRefactoring.md):
+ * the four providers are composition roots over this one class, so the order
+ * of its steps is the business logic of all four. The provider suites
+ * (llm-analysis-result-contract, llm-provider-parity, ...) pin the outcome
+ * through the real singletons; this one pins what the analyzer asks of its
+ * collaborators, which those suites cannot see.
+ */
+
+const assert = require('assert');
+
+process.env.SYSTEM_PROMPT = 'Analyse the document.';
+process.env.USE_PROMPT_TAGS = 'no';
+process.env.RESPONSE_TOKENS = '700';
+
+const { DocumentAnalyzer } = require('../services/llm/documentAnalyzer');
+const responseParser = require('../services/llm/responseParser');
+
+let failed = 0;
+const check = async (label, fn) => {
+  try {
+    await fn();
+    console.log(`  ✓ ${label}`);
+  } catch (error) {
+    failed += 1;
+    console.error(`  ✗ ${label}: ${error.message}`);
+  }
+};
+
+const METRICS = { promptTokens: 9, completionTokens: 3, totalTokens: 12 };
+const ANSWER = JSON.stringify({ tags: ['invoice'], correspondent: 'Telekom' });
+
+/** An analyzer whose collaborators record what they were asked. */
+function analyzer({ payload = ANSWER, ...overrides } = {}) {
+  const calls = { complete: [], reserve: [], fit: [], prompt: [], cut: [] };
+  const transport = {
+    client: {},
+    lastGenerateTextUsage: null,
+    initialize() {},
+    isReady() {
+      return Boolean(this.client);
+    },
+    async complete(request) {
+      calls.complete.push(request);
+      if (payload instanceof Error) throw payload;
+      return { payload, metrics: METRICS };
+    },
+    async generate(...args) {
+      calls.generate = args;
+      return 'generated';
+    },
+  };
+  const subject = new DocumentAnalyzer({
+    label: 'Fake',
+    transport,
+    tokenBudget: {
+      async reserve(frame, extras, model) {
+        calls.reserve.push({ frame, extras, model });
+        return { availableTokens: 4 };
+      },
+      async fit(content, availableTokens) {
+        calls.fit.push({ content, availableTokens });
+        return content.slice(0, availableTokens);
+      },
+      count: (text) => text.length,
+      cut: (text, maxTokens) => {
+        calls.cut.push(maxTokens);
+        return text.slice(0, maxTokens);
+      },
+    },
+    promptLayout: {
+      frame: (instructions) => `frame:${instructions}`,
+      build: (instructions, content) => ({
+        systemPrompt: `system:${instructions}`,
+        userContent: `user:${content}`,
+      }),
+    },
+    promptBuilder: {
+      buildAnalysisPrompt(input) {
+        calls.prompt.push(input);
+        return { systemPrompt: 'instructions', promptTags: '' };
+      },
+    },
+    responseParser,
+    resolveModel: () => 'fake-model',
+    checkStatus: async (provider) => ({ status: 'ok', provider }),
+    clientNotReadyMessage: 'Fake client not initialized',
+    cacheThumbnail: async () => false,
+    ...overrides,
+  });
+  return { subject, transport, calls };
+}
+
+(async () => {
+  console.log('\n=== LLM DocumentAnalyzer ===');
+
+  await check('the request is laid out, budgeted and sent once', async () => {
+    const { subject, calls } = analyzer({
+      analysisTemperature: (model) => (model === 'fake-model' ? 0.4 : 1),
+      responseSchema: { type: 'object' },
+      contextWindow: (text, responseTokens) => text.length + responseTokens,
+    });
+    const result = await subject.analyzeDocument('abcdefgh', [], [], [], 1);
+
+    assert.strictEqual(result.error, undefined, result.error);
+    assert.deepStrictEqual(calls.reserve, [
+      { frame: 'frame:instructions', extras: [], model: 'fake-model' },
+    ]);
+    assert.deepStrictEqual(calls.complete, [
+      {
+        systemPrompt: 'system:instructions',
+        userContent: 'user:abcd',
+        model: 'fake-model',
+        temperature: 0.4,
+        maxResponseTokens: 700,
+        schema: { type: 'object' },
+        numCtx: 'user:abcd'.length + 700,
+      },
+    ]);
+  });
+
+  await check('a success is default-shaped and knows it was cut', async () => {
+    const { subject } = analyzer();
+    const result = await subject.analyzeDocument('abcdefgh', [], [], [], 1);
+    assert.deepStrictEqual(result, {
+      document: {
+        tags: ['invoice'],
+        correspondent: 'Telekom',
+        title: null,
+        document_date: null,
+        document_type: null,
+        language: null,
+        custom_fields: null,
+      },
+      metrics: METRICS,
+      truncated: true,
+    });
+  });
+
+  await check('without a context window no numCtx is sent', async () => {
+    const { subject, calls } = analyzer();
+    await subject.analyzeDocument('abc', [], [], [], 1);
+    assert.strictEqual(calls.complete[0].numCtx, undefined);
+    assert.strictEqual(calls.complete[0].schema, null);
+  });
+
+  await check(
+    'the metadata check sees the answer before the defaults',
+    async () => {
+      const { subject } = analyzer({ payload: '{"title": "only a title"}' });
+      const result = await subject.analyzeDocument('abc', [], [], [], 1);
+      assert.match(result.error, /could not determine assignable metadata/);
+      assert.deepStrictEqual(result.document, {
+        tags: [],
+        correspondent: null,
+      });
+    }
+  );
+
+  await check('a refusal returns the placeholder with its tokens', async () => {
+    const { subject } = analyzer({ payload: "I'm sorry, I cannot." });
+    const result = await subject.analyzeDocument('abc', [], [], [], 7);
+    assert.strictEqual(result.error, 'Insufficient content for AI analysis');
+    assert.strictEqual(result.document.title, 'Document 7');
+    assert.deepStrictEqual(result.metrics, METRICS);
+  });
+
+  await check(
+    'a refusal phrase inside readable JSON is no refusal',
+    async () => {
+      const { subject } = analyzer({
+        payload: JSON.stringify({
+          tags: ['insufficient funds'],
+          correspondent: null,
+        }),
+      });
+      const result = await subject.analyzeDocument('abc', [], [], [], 1);
+      assert.strictEqual(result.error, undefined, result.error);
+      assert.deepStrictEqual(result.document.tags, ['insufficient funds']);
+    }
+  );
+
+  await check('without a client nothing is sent', async () => {
+    const { subject, transport, calls } = analyzer();
+    transport.client = null;
+    const result = await subject.analyzeDocument('abc', [], [], [], 1);
+    assert.strictEqual(result.error, 'Fake client not initialized');
+    assert.strictEqual(calls.complete.length, 0);
+  });
+
+  await check(
+    'a transport failure is returned, reworded by the provider',
+    async () => {
+      const failure = new Error('socket hang up');
+      failure.code = 'ECONNRESET';
+      const { subject } = analyzer({
+        payload: failure,
+        describeFailure: (error) => `reworded: ${error.message}`,
+      });
+      const result = await subject.analyzeDocument('abc', [], [], [], 1);
+      assert.deepStrictEqual(result, {
+        document: { tags: [], correspondent: null },
+        metrics: null,
+        error: 'reworded: socket hang up',
+        errorCode: 'ECONNRESET',
+      });
+    }
+  );
+
+  await check(
+    'the provider cut comes first and does not count as truncation',
+    async () => {
+      const { subject, calls } = analyzer({
+        prepareContent: (content) => content.slice(0, 3),
+      });
+      const result = await subject.analyzeDocument('abcdefgh', [], [], [], 1);
+      assert.strictEqual(calls.fit[0].content, 'abc');
+      assert.strictEqual(result.truncated, false);
+    }
+  );
+
+  await check('external API data is cut to its token share', async () => {
+    const { subject, calls } = analyzer();
+    await subject.analyzeDocument('abc', [], [], [], 1, null, {
+      externalApiData: 'x'.repeat(600),
+    });
+    assert.deepStrictEqual(calls.cut, [500]);
+    assert.strictEqual(calls.prompt[0].externalApiData, 'x'.repeat(500));
+  });
+
+  await check('the custom prompt reaches the prompt builder', async () => {
+    const { subject, calls } = analyzer();
+    await subject.analyzeDocument('abc', ['t'], ['c'], ['d'], 1, 'Custom.');
+    assert.deepStrictEqual(calls.prompt[0], {
+      existingTags: ['t'],
+      existingCorrespondentList: ['c'],
+      existingDocumentTypesList: ['d'],
+      externalApiData: null,
+      customPrompt: 'Custom.',
+    });
+  });
+
+  await check('generateText() is handed on as it came', async () => {
+    const { subject, calls } = analyzer();
+    const options = { maxTokens: 5 };
+    assert.strictEqual(await subject.generateText('hi', options), 'generated');
+    assert.deepStrictEqual(calls.generate, ['hi', options]);
+  });
+
+  await check(
+    'generateText() gets its sizes where the provider sizes the context',
+    async () => {
+      const { subject, calls } = analyzer({
+        contextWindow: (text, responseTokens) =>
+          text.length * 100 + responseTokens,
+      });
+      await subject.generateText('hi', { maxTokens: 50.9 });
+      assert.deepStrictEqual(calls.generate[2], {
+        responseTokens: 50,
+        numCtx: 250,
+      });
+      await subject.generateText('hi');
+      assert.deepStrictEqual(calls.generate[2], {
+        responseTokens: 700,
+        numCtx: 900,
+      });
+    }
+  );
+
+  await check(
+    'checkStatus() runs the provider probe on the provider',
+    async () => {
+      const { subject } = analyzer();
+      const status = await subject.checkStatus();
+      assert.strictEqual(status.provider, subject);
+    }
+  );
+
+  await check('client and lastGenerateTextUsage are the transport’s', () => {
+    const { subject, transport } = analyzer();
+    const client = {};
+    subject.client = client;
+    subject.lastGenerateTextUsage = METRICS;
+    assert.strictEqual(transport.client, client);
+    assert.strictEqual(transport.lastGenerateTextUsage, METRICS);
+  });
+
+  if (failed > 0) {
+    console.error(`\n${failed} DocumentAnalyzer case(s) failed`);
+    process.exit(1);
+  }
+  console.log('\nAll DocumentAnalyzer cases passed');
+})();
diff --git a/tests/test-native-install-log-paths.js b/tests/test-native-install-log-paths.js
index cde4821c..277f6a41 100644
--- a/tests/test-native-install-log-paths.js
+++ b/tests/test-native-install-log-paths.js
@@ -24,10 +24,10 @@ function assertMatches(content, regex, message) {
 function run() {
   console.log('\n=== Native Install Log Path Checks ===');
 
+  // The chat providers share the response log of the DocumentAnalyzer.
   const filesUsingResponseLogPath = [
-    'services/openaiService.js',
+    'services/llm/documentAnalyzer.js',
     'services/customService.js',
-    'services/azureService.js',
   ];
 
   filesUsingResponseLogPath.forEach((relativePath) => {

From fc8a28fc305462c5bfc19742ec0d134bdf08a29a Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:00:58 +0200
Subject: [PATCH 45/53] Move the Custom provider onto the DocumentAnalyzer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 6 of services/ManualServiceRefactoring.md, second of three structural
commits: Custom's analyzeDocument() was the OpenAI/Azure one plus its own
timeout wording, so it becomes a composition root over the shared
DocumentAnalyzer.

Changes
- services/llm/providers/customProvider.js (new): the composition root.
  Client factory with the no-auth fallback key, the detailed "Invalid API
  response structure: …" messages for a missing and an empty answer, the
  generateText() request with its context clamp and three reasoning
  dialects (/no_think, chat_template_kwargs, reasoning_effort), and the
  status probe moved verbatim. The timeout wording of rows e/m stays
  Custom's own, as describeFailure(): "[TIMEOUT][AI]" logged, the failure
  reworded by buildTimeoutErrorMessage('AI').
- services/customService.js: a one-line re-export.
- tests/test-native-install-log-paths.js: the response log path lives in
  documentAnalyzer.js only.

Testing
The 46 provider-related tests plus llm-document-analyzer and
native-install-log-paths: 45 pass, the same 3 environmental failures as on
the merge commit (two native crashes, ai-run-cost). llm-provider-parity and
llm-transport-request-mapping pass unmodified. ESLint and Prettier clean on
the touched files.

Impact
Same requests and results. Custom now also writes the analysis prompt to
data/logs/prompt.txt, as OpenAI and Azure always did; it was the only chat
provider that did not.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/customService.js                | 414 +----------------------
 services/llm/providers/customProvider.js | 216 ++++++++++++
 tests/test-native-install-log-paths.js   |   7 +-
 3 files changed, 222 insertions(+), 415 deletions(-)
 create mode 100644 services/llm/providers/customProvider.js

diff --git a/services/customService.js b/services/customService.js
index 49a5b707..1e975e81 100644
--- a/services/customService.js
+++ b/services/customService.js
@@ -1,410 +1,4 @@
-const {
-  calculateTokens,
-  truncateToTokenLimit,
-  isTimeoutError,
-  buildTimeoutErrorMessage,
-} = require('./serviceUtils');
-const {
-  hasNumber,
-  hasSystemPrompt,
-  modelOverride,
-  reasoningEffortForCompatible,
-  reasoningEnabled,
-} = require('./aiGenerateOptions');
-const OpenAI = require('openai');
-const config = require('../config/config');
-const fs = require('fs').promises;
-const path = require('path');
-const { cacheThumbnail } = require('./thumbnailCache');
-const { buildAnalysisPrompt } = require('./llm/promptBuilder');
-const tiktokenBudget = require('./llm/tokenBudget/tiktokenBudget');
-const {
-  ChatCompletionsTransport,
-} = require('./llm/transports/chatCompletionsTransport');
-const {
-  buildSuccess,
-  buildFailure,
-  buildInsufficientContent,
-  assertAssignableMetadata,
-  withDocumentDefaults,
-} = require('./llm/analysisResult');
-const {
-  parseAnalysisResponse,
-  isInsufficientContentAnswer,
-} = require('./llm/responseParser');
-const responseLogPath = path.join(
-  process.cwd(),
-  'data',
-  'logs',
-  'response.txt'
-);
-const CUSTOM_PROVIDER_FALLBACK_API_KEY = 'no-auth-required';
-/** What an operator can do about an answer that hit the completion cap. */
-const TRUNCATION_REMEDY = 'Raise Response Tokens (RESPONSE_TOKENS).';
-/**
- * The soft switch the Qwen family reads out of the prompt itself.
- *
- * `chat_template_kwargs` only reaches a server that applies the model's chat
- * template; a gateway that forwards the messages and drops the rest leaves the
- * model thinking anyway. Qwen understands the same wish written into the last
- * line of the user message, so both are sent and whichever arrives wins.
- */
-const QWEN_NO_THINK_SWITCH = '/no_think';
-/** Models that understand the line above. */
-const QWEN_MODEL_PATTERN = /qwen/i;
-
-class CustomOpenAIService {
-  constructor() {
-    this.transport = new ChatCompletionsTransport({
-      label: 'Custom OpenAI',
-      generation: {
-        notReadyMessage:
-          'Custom OpenAI client not initialized - missing API key',
-        remedy: TRUNCATION_REMEDY,
-        buildRequest: (prompt, options) =>
-          this._buildGenerationRequest(prompt, options),
-        // Say what came back: a gateway that timed out behind a 200, a
-        // model that stopped after its reasoning without an answer, or an
-        // empty choices array all end here and need telling apart.
-        describeEmptyText: (response) => {
-          const finishReason =
-            response?.choices?.[0]?.finish_reason ?? 'unknown';
-          const head = JSON.stringify(response ?? null).slice(0, 200);
-          return `Invalid API response structure: the answer carried no text (finish_reason ${finishReason}; the response begins ${head})`;
-        },
-      },
-      createClient: () =>
-        config.aiProvider === 'custom'
-          ? new OpenAI({
-              baseURL: config.custom.apiUrl,
-              // OpenAI-compatible SDKs require an apiKey value even when auth is disabled.
-              apiKey: config.custom.apiKey || CUSTOM_PROVIDER_FALLBACK_API_KEY,
-            })
-          : null,
-      // Every message keeps the "Invalid API response structure" prefix:
-      // shouldQueueForOcrOnAiError() matches on that phrase, and without it
-      // the document no longer reaches the OCR fallback.
-      describeMissingContent: (response, message) => {
-        const missing = !response
-          ? 'the response is empty'
-          : !response.choices
-            ? 'missing choices'
-            : !response.choices[0]
-              ? 'missing choices[0]'
-              : !response.choices[0].message
-                ? 'missing choices[0].message'
-                : !response.choices[0].message.content
-                  ? 'missing choices[0].message.content'
-                  : `unusable content ${message?.content}`;
-        return `Invalid API response structure: ${missing}.`;
-      },
-    });
-    this.tokenizer = null;
-  }
-
-  /** The SDK client. It lives on the transport; tests replace it here. */
-  get client() {
-    return this.transport.client;
-  }
-
-  set client(client) {
-    this.transport.client = client;
-  }
-
-  /** Usage of the last generateText() call; read by the AI review features. */
-  get lastGenerateTextUsage() {
-    return this.transport.lastGenerateTextUsage;
-  }
-
-  set lastGenerateTextUsage(usage) {
-    this.transport.lastGenerateTextUsage = usage;
-  }
-
-  initialize() {
-    this.transport.initialize();
-  }
-
-  async analyzeDocument(
-    content,
-    existingTags = [],
-    existingCorrespondentList = [],
-    existingDocumentTypesList = [],
-    id,
-    customPrompt = null,
-    options = {}
-  ) {
-    try {
-      this.initialize();
-
-      if (!this.client) {
-        throw new Error('Custom OpenAI client not initialized');
-      }
-
-      // Cache the thumbnail for the UI; never let it affect the analysis.
-      await cacheThumbnail(id);
-
-      // Get external API data if available and validate it
-      let externalApiData = options.externalApiData || null;
-      let validatedExternalApiData = null;
-
-      if (externalApiData) {
-        try {
-          validatedExternalApiData =
-            await this._validateAndTruncateExternalApiData(externalApiData);
-          console.log('[DEBUG] External API data validated and included');
-        } catch (error) {
-          console.warn(
-            '[WARNING] External API data validation failed:',
-            error.message
-          );
-          validatedExternalApiData = null;
-        }
-      }
-
-      const model = config.custom.model;
-      const { systemPrompt, promptTags } = buildAnalysisPrompt({
-        existingTags,
-        existingCorrespondentList,
-        existingDocumentTypesList,
-        externalApiData: validatedExternalApiData,
-        customPrompt,
-      });
-      if (customPrompt) {
-        console.log('[DEBUG] Replace system prompt with custom prompt');
-      }
-
-      // Calculate tokens AFTER all prompt modifications are complete
-      const { availableTokens } = await tiktokenBudget.reserve(
-        systemPrompt,
-        process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [],
-        model
-      );
-      console.log(
-        `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
-      );
-      console.log(
-        `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`
-      );
-
-      const truncatedContent = await tiktokenBudget.fit(
-        content,
-        availableTokens,
-        model
-      );
-
-      // console.log('######################################################################');
-      // console.log(`[DEBUG] Content length: ${content.length}, Truncated content length: ${truncatedContent.length}`);
-      // console.log(`[DEBUG] Truncated content: ${truncatedContent}`);
-      // console.log(`[DEBUG] System prompt: ${systemPrompt}`);
-      // console.log(`[DEBUG] Prompt tags: ${promptTags}`);
-      // console.log(`[DEBUG] Model: ${model}`);
-      // console.log(`[DEBUG] Custom fields: ${customFieldsStr}`);
-      // console.log(`[DEBUG] Custom prompt: ${customPrompt}`);
-      // console.log(`[DEBUG] External API data: ${validatedExternalApiData}`);
-      // console.log('######################################################################');
-
-      const { payload, metrics: mappedUsage } = await this.transport.complete({
-        systemPrompt,
-        userContent: truncatedContent,
-        model,
-        temperature: config.aiTemperatureAnalysis,
-      });
-      let jsonContent = payload;
-
-      let parsedResponse;
-      try {
-        const parsedResult = parseAnalysisResponse(jsonContent);
-        parsedResponse = parsedResult.parsed;
-        jsonContent = parsedResult.normalized;
-      } catch (error) {
-        console.log(`[ERROR] Failed to parse JSON response: ${error.message}`);
-        console.debug(error);
-        if (isInsufficientContentAnswer(jsonContent)) {
-          console.warn(`Document ${id} has insufficient content for analysis`);
-          // Return a default structure instead of throwing to prevent retry loops
-          return buildInsufficientContent(id, mappedUsage);
-        }
-        throw error;
-      }
-
-      try {
-        await fs.mkdir(path.dirname(responseLogPath), { recursive: true });
-        await fs.appendFile(responseLogPath, `${jsonContent}\n`);
-      } catch (logError) {
-        console.warn('Failed to write AI response log:', logError.message);
-      }
-
-      assertAssignableMetadata(parsedResponse);
-
-      return buildSuccess(
-        withDocumentDefaults(parsedResponse),
-        mappedUsage,
-        truncatedContent.length < content.length
-      );
-    } catch (error) {
-      const normalizedMessage = isTimeoutError(error)
-        ? buildTimeoutErrorMessage('AI')
-        : error.message;
-
-      if (isTimeoutError(error)) {
-        console.error(
-          `[TIMEOUT][AI] Custom provider request timed out: ${error.message}`
-        );
-      }
-
-      console.error(`Failed to analyze document: ${normalizedMessage}`);
-      console.debug(error);
-      return buildFailure(error, normalizedMessage);
-    }
-  }
-
-  /**
-   * Validate and truncate external API data to prevent token overflow
-   * @param {any} apiData - The external API data to validate
-   * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500)
-   * @returns {string} - Validated and potentially truncated data string
-   */
-  async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
-    if (!apiData) {
-      return null;
-    }
-
-    const dataString =
-      typeof apiData === 'object'
-        ? JSON.stringify(apiData, null, 2)
-        : String(apiData);
-
-    // Calculate tokens for the data
-    const dataTokens = await calculateTokens(dataString, config.custom.model);
-
-    if (dataTokens > maxTokens) {
-      console.warn(
-        `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating`
-      );
-      return await truncateToTokenLimit(
-        dataString,
-        maxTokens,
-        config.custom.model
-      );
-    }
-
-    console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`);
-    return dataString;
-  }
-
-  /**
-   * Generate text based on a prompt
-   * @param {string} prompt - The prompt to generate text from
-   * @param {object} [options] - Optional overrides for this one call
-   * @param {string} [options.systemPrompt] - Sent as the system message
-   * @param {number} [options.temperature] - Overrides AI_TEMPERATURE_GENERATION
-   * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS, still clamped
-   * @param {AbortSignal} [options.signal] - Cancels the request in flight
-   * @param {Function} [options.onProgress] - Streams the answer and reports it
-   *   as it grows; see GenerateTextProgress in aiGenerateOptions
-   * @param {boolean} [options.reasoning] - Switches the model's thinking off
-   *   or on for this one call; nothing new is sent when the caller says
-   *   nothing. Off also sends reasoning_effort where the model family takes it
-   * @returns {Promise<string>} - The generated text
-   */
-  async generateText(prompt, options = {}) {
-    return this.transport.generate(prompt, options);
-  }
-
-  /**
-   * This provider's generateText() request. The transport sends it, streams
-   * it and reads the answer; what differs between the providers is here.
-   * @param {string} prompt
-   * @param {object} options - see generateText()
-   * @returns {Promise<Object>} The chat completions request
-   */
-  async _buildGenerationRequest(prompt, options) {
-    const model = modelOverride(options) || config.custom.model;
-    const maxContextTokens = Number(config.tokenLimit) || 128000;
-    const desiredCompletionTokens =
-      hasNumber(options.maxTokens) && options.maxTokens > 0
-        ? Math.floor(options.maxTokens)
-        : Number(config.responseTokens) || 1000;
-    const promptTokens = await calculateTokens(prompt, model);
-    const availableCompletionTokens = Math.max(
-      1,
-      maxContextTokens - promptTokens - 64
-    );
-    const maxCompletionTokens = Math.max(
-      1,
-      Math.min(desiredCompletionTokens, availableCompletionTokens)
-    );
-
-    if (maxCompletionTokens < desiredCompletionTokens) {
-      console.log(
-        `[DEBUG] Clamped max_tokens for custom generateText from ${desiredCompletionTokens} to ${maxCompletionTokens} ` +
-          `(context limit: ${maxContextTokens}, prompt tokens: ${promptTokens})`
-      );
-    }
-
-    const reasoning = reasoningEnabled(options);
-    const messages = [];
-    if (hasSystemPrompt(options)) {
-      messages.push({ role: 'system', content: options.systemPrompt });
-    }
-    messages.push({
-      role: 'user',
-      content:
-        reasoning === false && QWEN_MODEL_PATTERN.test(model)
-          ? `${prompt}\n${QWEN_NO_THINK_SWITCH}`
-          : prompt,
-    });
-
-    const request = {
-      model: model,
-      messages,
-      temperature: hasNumber(options.temperature)
-        ? options.temperature
-        : config.aiTemperatureGeneration,
-      max_tokens: maxCompletionTokens,
-    };
-    // The key every OpenAI-compatible server that applies a chat template
-    // understands; one that does not know it passes it through and the
-    // model decides for itself, as it did before.
-    if (reasoning !== null) {
-      request.chat_template_kwargs = { enable_thinking: reasoning };
-    }
-    // A hosted reasoning model reads neither of the two above and thinks
-    // anyway; the families that take reasoning_effort are told in their own
-    // dialect. Only when the caller asked for no thinking, and only for
-    // those names — another model answers 400 for the field.
-    if (reasoning === false) {
-      const effort = reasoningEffortForCompatible(model);
-      if (effort) request.reasoning_effort = effort;
-    }
-
-    return request;
-  }
-
-  async checkStatus() {
-    try {
-      this.initialize();
-
-      if (!this.client) {
-        throw new Error(
-          'Custom OpenAI client not initialized - missing API key'
-        );
-      }
-
-      const model = config.custom.model;
-
-      // Use token-free endpoint where supported by OpenAI-compatible providers.
-      await this.client.models.list();
-
-      return { status: 'ok', model: model };
-    } catch (error) {
-      console.error(
-        `Error generating text with Custom OpenAI: ${error.message}`
-      );
-      console.debug(error);
-      return { status: 'error', error: error.message };
-    }
-  }
-}
-
-module.exports = new CustomOpenAIService();
+// The Custom OpenAI provider is a composition root over the shared
+// DocumentAnalyzer (services/llm/). This path stays so its callers need no
+// change.
+module.exports = require('./llm/providers/customProvider');
diff --git a/services/llm/providers/customProvider.js b/services/llm/providers/customProvider.js
new file mode 100644
index 00000000..81b488d0
--- /dev/null
+++ b/services/llm/providers/customProvider.js
@@ -0,0 +1,216 @@
+'use strict';
+
+/**
+ * The composition root for any OpenAI-compatible server behind a custom base
+ * URL: the DocumentAnalyzer wired to the chat completions transport and the
+ * tiktoken budget. What is Custom's own lives here — the client, the
+ * detailed "Invalid API response structure" messages, the generateText()
+ * request with its context clamp and three reasoning dialects, the timeout
+ * wording (rows e/m, kept as they are) and the status probe.
+ * services/customService.js re-exports the singleton.
+ */
+
+const OpenAI = require('openai');
+const config = require('../../../config/config');
+const {
+  calculateTokens,
+  isTimeoutError,
+  buildTimeoutErrorMessage,
+} = require('../../serviceUtils');
+const {
+  hasNumber,
+  hasSystemPrompt,
+  modelOverride,
+  reasoningEffortForCompatible,
+  reasoningEnabled,
+} = require('../../aiGenerateOptions');
+const { DocumentAnalyzer } = require('../documentAnalyzer');
+const {
+  ChatCompletionsTransport,
+} = require('../transports/chatCompletionsTransport');
+const tiktokenBudget = require('../tokenBudget/tiktokenBudget');
+const { chatPromptLayout } = require('../promptLayout');
+
+const CUSTOM_PROVIDER_FALLBACK_API_KEY = 'no-auth-required';
+/** What an operator can do about an answer that hit the completion cap. */
+const TRUNCATION_REMEDY = 'Raise Response Tokens (RESPONSE_TOKENS).';
+/**
+ * The soft switch the Qwen family reads out of the prompt itself.
+ *
+ * `chat_template_kwargs` only reaches a server that applies the model's chat
+ * template; a gateway that forwards the messages and drops the rest leaves the
+ * model thinking anyway. Qwen understands the same wish written into the last
+ * line of the user message, so both are sent and whichever arrives wins.
+ */
+const QWEN_NO_THINK_SWITCH = '/no_think';
+/** Models that understand the line above. */
+const QWEN_MODEL_PATTERN = /qwen/i;
+
+function createClient() {
+  return config.aiProvider === 'custom'
+    ? new OpenAI({
+        baseURL: config.custom.apiUrl,
+        // OpenAI-compatible SDKs require an apiKey value even when auth is disabled.
+        apiKey: config.custom.apiKey || CUSTOM_PROVIDER_FALLBACK_API_KEY,
+      })
+    : null;
+}
+
+/**
+ * Every message keeps the "Invalid API response structure" prefix:
+ * shouldQueueForOcrOnAiError() matches on that phrase, and without it
+ * the document no longer reaches the OCR fallback.
+ */
+function describeMissingContent(response, message) {
+  const missing = !response
+    ? 'the response is empty'
+    : !response.choices
+      ? 'missing choices'
+      : !response.choices[0]
+        ? 'missing choices[0]'
+        : !response.choices[0].message
+          ? 'missing choices[0].message'
+          : !response.choices[0].message.content
+            ? 'missing choices[0].message.content'
+            : `unusable content ${message?.content}`;
+  return `Invalid API response structure: ${missing}.`;
+}
+
+/**
+ * Say what came back: a gateway that timed out behind a 200, a
+ * model that stopped after its reasoning without an answer, or an
+ * empty choices array all end here and need telling apart.
+ */
+function describeEmptyText(response) {
+  const finishReason = response?.choices?.[0]?.finish_reason ?? 'unknown';
+  const head = JSON.stringify(response ?? null).slice(0, 200);
+  return `Invalid API response structure: the answer carried no text (finish_reason ${finishReason}; the response begins ${head})`;
+}
+
+/**
+ * The generateText() request. The transport sends it, streams it and reads
+ * the answer; what differs between the providers is here.
+ *
+ * @param {string} prompt
+ * @param {object} options - see DocumentAnalyzer.generateText()
+ * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS, still clamped
+ * @param {boolean} [options.reasoning] - Switches the model's thinking off
+ *   or on for this one call; nothing new is sent when the caller says
+ *   nothing. Off also sends reasoning_effort where the model family takes it
+ * @returns {Promise<Object>} The chat completions request
+ */
+async function buildGenerationRequest(prompt, options) {
+  const model = modelOverride(options) || config.custom.model;
+  const maxContextTokens = Number(config.tokenLimit) || 128000;
+  const desiredCompletionTokens =
+    hasNumber(options.maxTokens) && options.maxTokens > 0
+      ? Math.floor(options.maxTokens)
+      : Number(config.responseTokens) || 1000;
+  const promptTokens = await calculateTokens(prompt, model);
+  const availableCompletionTokens = Math.max(
+    1,
+    maxContextTokens - promptTokens - 64
+  );
+  const maxCompletionTokens = Math.max(
+    1,
+    Math.min(desiredCompletionTokens, availableCompletionTokens)
+  );
+
+  if (maxCompletionTokens < desiredCompletionTokens) {
+    console.log(
+      `[DEBUG] Clamped max_tokens for custom generateText from ${desiredCompletionTokens} to ${maxCompletionTokens} ` +
+        `(context limit: ${maxContextTokens}, prompt tokens: ${promptTokens})`
+    );
+  }
+
+  const reasoning = reasoningEnabled(options);
+  const messages = [];
+  if (hasSystemPrompt(options)) {
+    messages.push({ role: 'system', content: options.systemPrompt });
+  }
+  messages.push({
+    role: 'user',
+    content:
+      reasoning === false && QWEN_MODEL_PATTERN.test(model)
+        ? `${prompt}\n${QWEN_NO_THINK_SWITCH}`
+        : prompt,
+  });
+
+  const request = {
+    model: model,
+    messages,
+    temperature: hasNumber(options.temperature)
+      ? options.temperature
+      : config.aiTemperatureGeneration,
+    max_tokens: maxCompletionTokens,
+  };
+  // The key every OpenAI-compatible server that applies a chat template
+  // understands; one that does not know it passes it through and the
+  // model decides for itself, as it did before.
+  if (reasoning !== null) {
+    request.chat_template_kwargs = { enable_thinking: reasoning };
+  }
+  // A hosted reasoning model reads neither of the two above and thinks
+  // anyway; the families that take reasoning_effort are told in their own
+  // dialect. Only when the caller asked for no thinking, and only for
+  // those names — another model answers 400 for the field.
+  if (reasoning === false) {
+    const effort = reasoningEffortForCompatible(model);
+    if (effort) request.reasoning_effort = effort;
+  }
+
+  return request;
+}
+
+/** Only Custom words a timeout as such (rows e/m, kept as they are). */
+function describeFailure(error) {
+  if (!isTimeoutError(error)) {
+    return error.message;
+  }
+  console.error(
+    `[TIMEOUT][AI] Custom provider request timed out: ${error.message}`
+  );
+  return buildTimeoutErrorMessage('AI');
+}
+
+async function checkStatus(provider) {
+  try {
+    provider.initialize();
+
+    if (!provider.client) {
+      throw new Error('Custom OpenAI client not initialized - missing API key');
+    }
+
+    const model = config.custom.model;
+
+    // Use token-free endpoint where supported by OpenAI-compatible providers.
+    await provider.client.models.list();
+
+    return { status: 'ok', model: model };
+  } catch (error) {
+    console.error(`Error generating text with Custom OpenAI: ${error.message}`);
+    console.debug(error);
+    return { status: 'error', error: error.message };
+  }
+}
+
+module.exports = new DocumentAnalyzer({
+  label: 'Custom OpenAI',
+  transport: new ChatCompletionsTransport({
+    label: 'Custom OpenAI',
+    createClient,
+    describeMissingContent,
+    generation: {
+      notReadyMessage: 'Custom OpenAI client not initialized - missing API key',
+      remedy: TRUNCATION_REMEDY,
+      buildRequest: buildGenerationRequest,
+      describeEmptyText,
+    },
+  }),
+  tokenBudget: tiktokenBudget,
+  promptLayout: chatPromptLayout,
+  resolveModel: () => config.custom.model,
+  describeFailure,
+  checkStatus,
+  clientNotReadyMessage: 'Custom OpenAI client not initialized',
+});
diff --git a/tests/test-native-install-log-paths.js b/tests/test-native-install-log-paths.js
index 277f6a41..58679078 100644
--- a/tests/test-native-install-log-paths.js
+++ b/tests/test-native-install-log-paths.js
@@ -24,11 +24,8 @@ function assertMatches(content, regex, message) {
 function run() {
   console.log('\n=== Native Install Log Path Checks ===');
 
-  // The chat providers share the response log of the DocumentAnalyzer.
-  const filesUsingResponseLogPath = [
-    'services/llm/documentAnalyzer.js',
-    'services/customService.js',
-  ];
+  // Every provider writes the response log through the DocumentAnalyzer.
+  const filesUsingResponseLogPath = ['services/llm/documentAnalyzer.js'];
 
   filesUsingResponseLogPath.forEach((relativePath) => {
     const fullPath = path.join(process.cwd(), relativePath);

From 5cfd7e2a7363ccbb1f4bef656fc2a8deee12541f Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:09:50 +0200
Subject: [PATCH 46/53] Move the Ollama provider onto the DocumentAnalyzer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 6 of services/ManualServiceRefactoring.md, last of three structural
commits. Ollama lays its request out differently from the chat providers —
the instructions in the prompt in front of the document, a short system
prompt of its own, a character budget, num_ctx and a format schema — which
is why the refactor-step-6 branch left its class as it was. All of that is
collaborator or configuration of the shared analyzer now, so the four
providers run one analyzeDocument().

Changes
- services/llm/providers/ollamaProvider.js (new): the composition root —
  the axios client, the output schema (responseSchema), charBudget,
  ollamaPromptLayout, the CONTENT_MAX_LENGTH cut (prepareContent, kept as
  it is), num_ctx sizing (contextWindow) and the /api/ps probe moved
  verbatim. The seams the tests call or stub stay on the singleton as
  one-line delegations: apiUrl, model, documentAnalysisSchema,
  _buildPrompt(), _callOllamaAPI(), _calculatePromptTokenCount(),
  _calculateNumCtx(), _buildRequestHeaders(), _assertNotTruncated(),
  _extractOllamaMetrics(). num_ctx is sized through the two _calculate*()
  methods on both paths, so a stub reaches the request; generateText() gets
  its { responseTokens, numCtx } from the analyzer as before.
- services/llm/transports/ollamaGenerateTransport.js: complete() throws "No
  response data from Ollama API" for an answer without `response`, after
  the truncation check, as _processOllamaResponse() did (row l unchanged).
- services/ollamaService.js: a one-line re-export. The axios client is
  still created on first load, which is what ollama-temperature-wiring's
  require.cache mock relies on.
- tests/test-llm-provider-parity.js: also checks the structure of "Done
  means" — each service a DocumentAnalyzer, its legacy module nothing but
  the re-export of its provider.

Testing
The 46 provider-related tests plus llm-document-analyzer and
native-install-log-paths: 45 pass, the same 3 environmental failures as on
the merge commit. All tests the handoff names as seams pass unmodified
(ollama-temperature-wiring, ollama-upstream-error, ollama-token-metrics,
llm-transport-request-mapping, prompt-existing-data-serialization,
restricted-document-types-placeholder). Request probe: the outgoing request
bodies and the results of all four providers are byte-identical to
refactor-step-6-checkpoint-4-parity in eight scenarios (plain, custom
prompt, external API data, USE_PROMPT_TAGS, USE_EXISTING_DATA,
CONTENT_MAX_LENGTH, cut to TOKEN_LIMIT, refusal). ESLint and Prettier clean
on the touched files.

Impact
Same requests and results. Ollama's debug logs follow the other providers:
data/logs/prompt.txt gets the system prompt and the prompt before the
request instead of prompt plus answer after it, and the answer goes to
data/logs/response.txt.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/llm/providers/ollamaProvider.js      | 246 ++++++++
 .../llm/transports/ollamaGenerateTransport.js |   5 +
 services/ollamaService.js                     | 567 +-----------------
 tests/test-llm-provider-parity.js             |  35 +-
 4 files changed, 288 insertions(+), 565 deletions(-)
 create mode 100644 services/llm/providers/ollamaProvider.js

diff --git a/services/llm/providers/ollamaProvider.js b/services/llm/providers/ollamaProvider.js
new file mode 100644
index 00000000..4002c47c
--- /dev/null
+++ b/services/llm/providers/ollamaProvider.js
@@ -0,0 +1,246 @@
+'use strict';
+
+/**
+ * The Ollama composition root: the DocumentAnalyzer wired to the
+ * /api/generate transport, the character budget and Ollama's prompt layout.
+ * What is Ollama's own lives here — the axios client, the output schema, the
+ * num_ctx sizing, the CONTENT_MAX_LENGTH cut and the /api/ps status probe.
+ * services/ollamaService.js re-exports the singleton.
+ *
+ * The singleton keeps the seams the tests call or stub: apiUrl, model,
+ * documentAnalysisSchema, _buildPrompt(), _callOllamaAPI(),
+ * _calculatePromptTokenCount(), _calculateNumCtx(), _extractOllamaMetrics().
+ * The context is sized through _calculatePromptTokenCount() and
+ * _calculateNumCtx(), so a stub there reaches the request.
+ */
+
+const axios = require('axios');
+const config = require('../../../config/config');
+const { DocumentAnalyzer } = require('../documentAnalyzer');
+const {
+  OllamaGenerateTransport,
+} = require('../transports/ollamaGenerateTransport');
+const charBudget = require('../tokenBudget/charBudget');
+const { buildAnalysisPrompt } = require('../promptBuilder');
+const { ollamaPromptLayout, ollamaPrompt } = require('../promptLayout');
+
+/** The output schema Ollama is held to (`format`). */
+const DOCUMENT_ANALYSIS_SCHEMA = {
+  type: 'object',
+  properties: {
+    title: { type: 'string' },
+    correspondent: { type: 'string' },
+    tags: {
+      type: 'array',
+      items: { type: 'string' },
+    },
+    document_type: { type: 'string' },
+    document_date: { type: 'string' },
+    language: { type: 'string' },
+    custom_fields: {
+      type: 'object',
+      additionalProperties: true,
+    },
+  },
+  required: [
+    'title',
+    'correspondent',
+    'tags',
+    'document_type',
+    'document_date',
+    'language',
+  ],
+};
+
+/** Tokens the external API data may take up in the prompt. */
+const EXTERNAL_API_DATA_MAX_TOKENS = 500;
+
+/** CONTENT_MAX_LENGTH, Ollama's own cut before the budget (kept as is). */
+function truncateContent(content) {
+  try {
+    if (process.env.CONTENT_MAX_LENGTH) {
+      console.log(
+        'Truncating content to max length:',
+        process.env.CONTENT_MAX_LENGTH
+      );
+      return content.substring(0, process.env.CONTENT_MAX_LENGTH);
+    }
+  } catch (error) {
+    console.error(`Error truncating content: ${error.message}`);
+    console.debug(error);
+  }
+  return content;
+}
+
+/** The external API data as prompt text, cut the character-budget way. */
+function externalApiDataForPrompt(apiData) {
+  const dataString =
+    typeof apiData === 'object'
+      ? JSON.stringify(apiData, null, 2)
+      : String(apiData);
+  return charBudget.count(dataString) > EXTERNAL_API_DATA_MAX_TOKENS
+    ? charBudget.cut(dataString, EXTERNAL_API_DATA_MAX_TOKENS)
+    : dataString;
+}
+
+async function checkStatus(provider) {
+  // use ollama status endpoint
+  try {
+    const response = await provider.client.get(`${provider.apiUrl}/api/ps`, {
+      headers: provider._buildRequestHeaders(),
+    });
+    if (response.status === 200) {
+      const data = response.data;
+      // Ensure data is an array and has at least one model
+      let modelName = null;
+      if (Array.isArray(data.models) && data.models.length > 0) {
+        modelName = data.models[0].name;
+      }
+      console.log('Ollama model name:', modelName);
+      return { status: 'ok', model: modelName };
+    }
+  } catch (error) {
+    console.error(`Error checking Ollama service status: ${error.message}`);
+    console.debug(error);
+  }
+  return { status: 'error' };
+}
+
+const transport = new OllamaGenerateTransport({
+  client: axios.create({
+    timeout: 1800000, // 30 minutes timeout
+  }),
+  apiUrl: config.ollama.apiUrl,
+  model: config.ollama.model,
+});
+
+const ollamaProvider = new DocumentAnalyzer({
+  label: 'Ollama',
+  transport,
+  tokenBudget: charBudget,
+  promptLayout: ollamaPromptLayout,
+  resolveModel: () => transport.model,
+  responseSchema: DOCUMENT_ANALYSIS_SCHEMA,
+  contextWindow: (promptText, responseTokens) =>
+    ollamaProvider._calculateNumCtx(
+      ollamaProvider._calculatePromptTokenCount(promptText),
+      responseTokens
+    ),
+  prepareContent: truncateContent,
+  checkStatus,
+  clientNotReadyMessage: 'Ollama client not initialized',
+});
+
+Object.defineProperties(ollamaProvider, {
+  apiUrl: {
+    get() {
+      return this.transport.apiUrl;
+    },
+    set(apiUrl) {
+      this.transport.apiUrl = apiUrl;
+    },
+    enumerable: true,
+  },
+  model: {
+    get() {
+      return this.transport.model;
+    },
+    set(model) {
+      this.transport.model = model;
+    },
+    enumerable: true,
+  },
+  documentAnalysisSchema: {
+    get() {
+      return this.responseSchema;
+    },
+    set(schema) {
+      this.responseSchema = schema;
+    },
+    enumerable: true,
+  },
+});
+
+Object.assign(ollamaProvider, {
+  /**
+   * The analysis prompt for a document: the instructions, then the document.
+   * @param {string} content - The document text
+   * @param {string[]} [existingTags]
+   * @param {string[]} [existingCorrespondent]
+   * @param {string[]} [existingDocumentTypes]
+   * @param {Object} [options] - May carry externalApiData
+   * @returns {string}
+   */
+  _buildPrompt(
+    content,
+    existingTags = [],
+    existingCorrespondent = [],
+    existingDocumentTypes = [],
+    options = {}
+  ) {
+    const { systemPrompt } = buildAnalysisPrompt({
+      existingTags,
+      existingCorrespondentList: Array.isArray(existingCorrespondent)
+        ? existingCorrespondent
+        : [],
+      existingDocumentTypesList: existingDocumentTypes,
+      externalApiData: options.externalApiData
+        ? externalApiDataForPrompt(options.externalApiData)
+        : null,
+    });
+    return ollamaPrompt(systemPrompt, content);
+  },
+
+  /** Estimated tokens of a prompt, two characters each. */
+  _calculatePromptTokenCount(prompt) {
+    return charBudget.estimateTokens(prompt);
+  },
+
+  /** The context window to request: prompt plus answer, at most TOKEN_LIMIT. */
+  _calculateNumCtx(promptTokenCount, expectedResponseTokens) {
+    const numCtx = charBudget.numCtx(promptTokenCount, expectedResponseTokens);
+
+    console.log('Prompt Token Count:', promptTokenCount);
+    console.log('Expected Response Tokens:', expectedResponseTokens);
+    console.log('Dynamic calculated num_ctx:', numCtx);
+
+    return numCtx;
+  },
+
+  /** Request headers, with bearer auth when an API key is set. */
+  _buildRequestHeaders() {
+    return this.transport.buildHeaders();
+  },
+
+  /**
+   * Send one analysis request as it is.
+   * @param {string} prompt - Prompt text
+   * @param {string} systemPrompt - System prompt
+   * @param {number} numCtx - Context window size
+   * @param {Object} schema - Response schema
+   * @returns {Promise<Object>} The raw /api/generate response data
+   */
+  async _callOllamaAPI(prompt, systemPrompt, numCtx, schema) {
+    const { raw } = await this.transport.complete({
+      systemPrompt,
+      userContent: prompt,
+      temperature: config.aiTemperatureAnalysis,
+      maxResponseTokens: Number(config.responseTokens),
+      numCtx,
+      schema,
+    });
+    return raw;
+  },
+
+  /** See OllamaGenerateTransport.assertNotTruncated(). */
+  _assertNotTruncated(responseData, numPredict, numCtx, extra) {
+    this.transport.assertNotTruncated(responseData, numPredict, numCtx, extra);
+  },
+
+  /** Token usage of an /api/generate response. */
+  _extractOllamaMetrics(responseData) {
+    return this.transport.extractMetrics(responseData);
+  },
+});
+
+module.exports = ollamaProvider;
diff --git a/services/llm/transports/ollamaGenerateTransport.js b/services/llm/transports/ollamaGenerateTransport.js
index 610d6b93..fdc1df66 100644
--- a/services/llm/transports/ollamaGenerateTransport.js
+++ b/services/llm/transports/ollamaGenerateTransport.js
@@ -205,6 +205,11 @@ class OllamaGenerateTransport {
     }
 
     this.assertNotTruncated(response.data, numPredict, numCtx);
+    // Worded unlike the chat providers' "Invalid API response structure", so
+    // it matches no OCR marker (row l, kept as it is).
+    if (!response.data.response) {
+      throw new Error('No response data from Ollama API');
+    }
     return {
       payload: response.data.response,
       metrics: this.extractMetrics(response.data),
diff --git a/services/ollamaService.js b/services/ollamaService.js
index c9c108ed..f9b147e9 100644
--- a/services/ollamaService.js
+++ b/services/ollamaService.js
@@ -1,564 +1,3 @@
-const { writePromptToFile } = require('./serviceUtils');
-const { hasNumber } = require('./aiGenerateOptions');
-const axios = require('axios');
-const config = require('../config/config');
-const { cacheThumbnail } = require('./thumbnailCache');
-const { buildAnalysisPrompt } = require('./llm/promptBuilder');
-const {
-  parseAnalysisResponse,
-  isInsufficientContentAnswer,
-} = require('./llm/responseParser');
-const charBudget = require('./llm/tokenBudget/charBudget');
-const {
-  OllamaGenerateTransport,
-} = require('./llm/transports/ollamaGenerateTransport');
-const {
-  buildSuccess,
-  buildFailure,
-  buildInsufficientContent,
-  assertAssignableMetadata,
-  withDocumentDefaults,
-} = require('./llm/analysisResult');
-const { buildCustomFieldsBlock } = require('./llm/customFieldsTemplate');
-
-/**
- * Service for document analysis using Ollama
- */
-class OllamaService {
-  /**
-   * Initialize the Ollama service
-   */
-  constructor() {
-    this.transport = new OllamaGenerateTransport({
-      client: axios.create({
-        timeout: 1800000, // 30 minutes timeout
-      }),
-      apiUrl: config.ollama.apiUrl,
-      model: config.ollama.model,
-    });
-
-    // JSON schema for document analysis output
-    this.documentAnalysisSchema = {
-      type: 'object',
-      properties: {
-        title: { type: 'string' },
-        correspondent: { type: 'string' },
-        tags: {
-          type: 'array',
-          items: { type: 'string' },
-        },
-        document_type: { type: 'string' },
-        document_date: { type: 'string' },
-        language: { type: 'string' },
-        custom_fields: {
-          type: 'object',
-          additionalProperties: true,
-        },
-      },
-      required: [
-        'title',
-        'correspondent',
-        'tags',
-        'document_type',
-        'document_date',
-        'language',
-      ],
-    };
-  }
-
-  /** The axios client. It lives on the transport; tests replace it here. */
-  get client() {
-    return this.transport.client;
-  }
-
-  set client(client) {
-    this.transport.client = client;
-  }
-
-  get apiUrl() {
-    return this.transport.apiUrl;
-  }
-
-  set apiUrl(apiUrl) {
-    this.transport.apiUrl = apiUrl;
-  }
-
-  get model() {
-    return this.transport.model;
-  }
-
-  set model(model) {
-    this.transport.model = model;
-  }
-
-  /** Usage of the last generateText() call; read by the AI review features. */
-  get lastGenerateTextUsage() {
-    return this.transport.lastGenerateTextUsage;
-  }
-
-  set lastGenerateTextUsage(usage) {
-    this.transport.lastGenerateTextUsage = usage;
-  }
-
-  /**
-   * Analyzes a document to extract metadata and tags using an AI model.
-   * @param {string} content - The content of the document to analyze.
-   * @param {Array} existingTags - Array of pre-existing tags to consider during analysis.
-   * @param {Array} existingCorrespondentList - Array of pre-existing correspondents.
-   * @param {Array} existingDocumentTypesList - Array of pre-existing document types.
-   * @param {string|number} id - Unique identifier for the document.
-   * @param {string} [customPrompt] - Custom prompt to guide the analysis process.
-   * @param {Object} [options] - Additional configuration options.
-   * @param {Object} [options.externalApiData] - External data to include in the analysis.
-   * @returns {Promise<Object>} An object containing analysis results, metrics, and status.
-   * @returns {Object} document - Parsed response from the AI analysis.
-   * @returns {Object} metrics - Performance metrics from the analysis process.
-   * @returns {boolean} truncated - Indicates if the content was truncated during processing.
-   * @returns {string} error - Error message if an exception occurred.
-   * @returns {string} errorCode - Error code if an exception occurred.
-   */
-  async analyzeDocument(
-    content,
-    existingTags = [],
-    existingCorrespondentList = [],
-    existingDocumentTypesList = [],
-    id,
-    customPrompt = null,
-    options = {}
-  ) {
-    try {
-      // Truncate content if needed
-      content = this._truncateContent(content);
-
-      // Cache the thumbnail for the UI; never let it affect the analysis.
-      await cacheThumbnail(id);
-
-      // Get external API data if available and validate it
-      let externalApiData = options.externalApiData || null;
-      let validatedExternalApiData = null;
-
-      if (externalApiData) {
-        try {
-          validatedExternalApiData =
-            this._validateAndTruncateExternalApiData(externalApiData);
-          console.log('[DEBUG] External API data validated and included');
-        } catch (error) {
-          console.warn(
-            '[WARNING] External API data validation failed:',
-            error.message
-          );
-          validatedExternalApiData = null;
-        }
-      }
-
-      // Build prompt
-      let prompt;
-      let promptPrefix;
-      if (!customPrompt) {
-        promptPrefix = this._buildPrompt(
-          '',
-          existingTags,
-          existingCorrespondentList,
-          existingDocumentTypesList,
-          options
-        );
-      } else {
-        promptPrefix =
-          buildAnalysisPrompt({ customPrompt }).systemPrompt + '\n\n';
-        console.log('[DEBUG] Ollama Service started with custom prompt');
-      }
-
-      // Fit the document into what is left of the context window. Without
-      // this an oversized prompt silently changed the meaning of the request:
-      // Ollama clamps num_ctx to the model's real limit, the long document
-      // filled the window, and the answer was cut off mid-JSON — reported as
-      // ai_response_truncated on exactly the long documents. With the content
-      // fitted, prompt + reserved answer always fit the configured
-      // TOKEN_LIMIT.
-      const originalContentLength = String(content).length;
-      const { availableTokens } = await charBudget.reserve(promptPrefix);
-      content = await charBudget.fit(content, availableTokens);
-
-      if (!customPrompt) {
-        prompt = this._buildPrompt(
-          content,
-          existingTags,
-          existingCorrespondentList,
-          existingDocumentTypesList,
-          options
-        );
-      } else {
-        prompt = promptPrefix + JSON.stringify(content);
-      }
-      const contentWasTruncated =
-        String(content).length < originalContentLength;
-
-      // Generate custom fields for the prompt
-      const customFieldsStr = this._generateCustomFieldsTemplate();
-
-      // Generate system prompt
-      const systemPrompt = this._generateSystemPrompt(customFieldsStr);
-
-      // Calculate context window size
-      const promptTokenCount = this._calculatePromptTokenCount(prompt);
-      const numCtx = this._calculateNumCtx(
-        promptTokenCount,
-        Number(config.responseTokens)
-      );
-
-      console.log(
-        `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting`
-      );
-      console.log(
-        `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}`
-      );
-
-      // Call Ollama API
-      const response = await this._callOllamaAPI(
-        prompt,
-        systemPrompt,
-        numCtx,
-        this.documentAnalysisSchema
-      );
-
-      // Process response
-      const metrics = this._extractOllamaMetrics(response);
-      let parsedResponse;
-      try {
-        parsedResponse = this._processOllamaResponse(response);
-      } catch (error) {
-        if (
-          error.message.startsWith('Invalid JSON response from API') &&
-          isInsufficientContentAnswer(response.response)
-        ) {
-          console.warn(`Document ${id} has insufficient content for analysis`);
-          return buildInsufficientContent(id, metrics);
-        }
-        throw error;
-      }
-
-      // Log the prompt and response
-      await this._logPromptAndResponse(systemPrompt, prompt, parsedResponse);
-
-      // Return results in consistent format
-      return buildSuccess(parsedResponse, metrics, contentWasTruncated);
-    } catch (error) {
-      console.error(`Error analyzing document with Ollama: ${error.message}`);
-      console.debug(error);
-      return buildFailure(error);
-    }
-  }
-
-  /**
-   * Truncate content to maximum length if specified
-   * @param {string} content - Content to truncate
-   * @returns {string} Truncated content
-   */
-  _truncateContent(content) {
-    try {
-      if (process.env.CONTENT_MAX_LENGTH) {
-        console.log(
-          'Truncating content to max length:',
-          process.env.CONTENT_MAX_LENGTH
-        );
-        return content.substring(0, process.env.CONTENT_MAX_LENGTH);
-      }
-    } catch (error) {
-      console.error(`Error truncating content: ${error.message}`);
-      console.debug(error);
-    }
-    return content;
-  }
-
-  /**
-   * Builds and returns a system prompt string combining configuration, content, and metadata.
-   * @param {string} content - The main content of the document to be processed.
-   * @param {string[]} existingTags - Array of existing tags associated with the document.
-   * @param {string[]} existingCorrespondent - Array of existing correspondents related to the document.
-   * @param {string[]} existingDocumentTypes - Array of existing document types.
-   * @param {Object} options - Optional configuration object. May include externalApiData for additional context.
-   * @return {string} The constructed system prompt string combining configuration, content, and metadata.
-   */
-  _buildPrompt(
-    content,
-    existingTags = [],
-    existingCorrespondent = [],
-    existingDocumentTypes = [],
-    options = {}
-  ) {
-    // Validate that existingCorrespondent is an array and handle if it's not
-    const correspondentList = Array.isArray(existingCorrespondent)
-      ? existingCorrespondent
-      : [];
-
-    // Get validated external API data if available
-    let validatedExternalApiData = null;
-    if (options.externalApiData) {
-      try {
-        validatedExternalApiData = this._validateAndTruncateExternalApiData(
-          options.externalApiData
-        );
-        console.log('[DEBUG] External API data validated and included');
-      } catch (error) {
-        console.warn(
-          '[WARNING] External API data validation failed:',
-          error.message
-        );
-        validatedExternalApiData = null;
-      }
-    }
-
-    const { systemPrompt } = buildAnalysisPrompt({
-      existingTags,
-      existingCorrespondentList: correspondentList,
-      existingDocumentTypesList: existingDocumentTypes,
-      externalApiData: validatedExternalApiData,
-    });
-
-    return `${systemPrompt}
-        ${JSON.stringify(content)}
-        `;
-  }
-
-  /**
-   * Validate and truncate external API data to prevent token overflow
-   * @param {any} apiData - The external API data to validate
-   * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500)
-   * @returns {string} - Validated and potentially truncated data string
-   */
-  _validateAndTruncateExternalApiData(apiData, maxTokens = 500) {
-    if (!apiData) {
-      return null;
-    }
-
-    const dataString =
-      typeof apiData === 'object'
-        ? JSON.stringify(apiData, null, 2)
-        : String(apiData);
-
-    // Calculate tokens for the data (conservative 2 chars/token for non-English)
-    const dataTokens = Math.ceil(dataString.length / 2);
-
-    if (dataTokens > maxTokens) {
-      console.warn(
-        `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating`
-      );
-      // Simple truncation based on character count
-      const maxChars = maxTokens * 2;
-      return dataString.substring(0, maxChars);
-    }
-
-    console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`);
-    return dataString;
-  }
-
-  /**
-   * Generate custom fields template for prompts
-   * @returns {string} Custom fields template as a string
-   */
-  _generateCustomFieldsTemplate() {
-    return buildCustomFieldsBlock();
-  }
-
-  /**
-   * Generate system prompt for document analysis
-   * @param {string} customFieldsStr - Custom fields as a string
-   * @returns {string} System prompt
-   */
-  _generateSystemPrompt(customFieldsStr) {
-    let systemPromptTemplate = `
-            You are a document analyzer. Your task is to analyze documents and extract relevant information. You do not ask back questions. 
-            YOU MUSTNOT: Ask for additional information or clarification, or ask questions about the document, or ask for additional context.
-            YOU MUSTNOT: Return a response without the desired JSON format.
-            YOU MUST: Return the result EXCLUSIVELY as a JSON object. The Tags, Title and Document_Type MUST be in the language that is used in the document.:
-            IMPORTANT: The custom_fields are optional and can be left out if not needed, only try to fill out the values if you find a matching information in the document.
-            Do not change the value of field_name, only fill out the values. If the field is about money only add the number without currency and always use a . for decimal places.
-            {
-                "title": "xxxxx",
-                "correspondent": "xxxxxxxx",
-                "tags": ["Tag1", "Tag2", "Tag3", "Tag4"],
-                "document_type": "Invoice/Contract/...",
-                "document_date": "YYYY-MM-DD",
-                "language": "en/de/es/...",
-                %CUSTOMFIELDS%
-            }
-            ALWAYS USE THE INFORMATION TO FILL OUT THE JSON OBJECT. DO NOT ASK BACK QUESTIONS.
-        `;
-
-    return systemPromptTemplate.replace('%CUSTOMFIELDS%', customFieldsStr);
-  }
-
-  /**
-   * Calculate prompt token count
-   * @param {string} prompt - Prompt text
-   * @returns {number} Estimated token count
-   */
-  _calculatePromptTokenCount(prompt) {
-    return charBudget.estimateTokens(prompt);
-  }
-
-  /**
-   * Calculate context window size for Ollama
-   * @param {number} promptTokenCount - Token count for prompt
-   * @param {number} expectedResponseTokens - Expected response token count
-   * @returns {number} Context window size
-   */
-  _calculateNumCtx(promptTokenCount, expectedResponseTokens) {
-    const numCtx = charBudget.numCtx(promptTokenCount, expectedResponseTokens);
-
-    console.log('Prompt Token Count:', promptTokenCount);
-    console.log('Expected Response Tokens:', expectedResponseTokens);
-    console.log('Dynamic calculated num_ctx:', numCtx);
-
-    return numCtx;
-  }
-
-  /** Request headers, with bearer auth when an API key is set. */
-  _buildRequestHeaders() {
-    return this.transport.buildHeaders();
-  }
-
-  /**
-   * Send the analysis request.
-   * @param {string} prompt - Prompt text
-   * @param {string} systemPrompt - System prompt
-   * @param {number} numCtx - Context window size
-   * @param {Object} schema - Response schema
-   * @returns {Promise<Object>} The raw /api/generate response data
-   */
-  async _callOllamaAPI(prompt, systemPrompt, numCtx, schema) {
-    const { raw } = await this.transport.complete({
-      systemPrompt,
-      userContent: prompt,
-      temperature: config.aiTemperatureAnalysis,
-      // The same number _calculateNumCtx() already reserved room for. It used
-      // to be a hardcoded 256 while the reservation followed the setting, so
-      // the context held space for an answer the model was then forbidden to
-      // write.
-      maxResponseTokens: Number(config.responseTokens),
-      numCtx,
-      schema,
-    });
-    return raw;
-  }
-
-  /** See OllamaGenerateTransport.assertNotTruncated(). */
-  _assertNotTruncated(responseData, numPredict, numCtx, extra) {
-    this.transport.assertNotTruncated(responseData, numPredict, numCtx, extra);
-  }
-
-  /**
-   * Process Ollama API response
-   *
-   * The answer arrives as an object when Ollama applied the format schema,
-   * and as text otherwise; responseParser handles both. Every exit either
-   * returns something the caller can use or throws. Handing back
-   * { tags: [], correspondent: null } for an unreadable answer is
-   * indistinguishable from a model that genuinely found nothing, so the
-   * document would be written back untouched and marked processed.
-   *
-   * @param {Object} responseData - Ollama API response data
-   * @returns {Object} Parsed response
-   */
-  _processOllamaResponse(responseData) {
-    if (!responseData.response) {
-      throw new Error('No response data from Ollama API');
-    }
-    try {
-      const { parsed } = parseAnalysisResponse(responseData.response);
-      // On the parsed answer: the defaults would turn a missing `tags` into
-      // [] and let every answer pass.
-      assertAssignableMetadata(parsed);
-      return withDocumentDefaults(parsed);
-    } catch (error) {
-      console.error('Error parsing Ollama response:', error.message);
-      throw error;
-    }
-  }
-
-  /**
-   * Extract token usage metrics from an Ollama /api/generate response.
-   * @param {Object} responseData - Ollama API response data
-   * @returns {{promptTokens:number, completionTokens:number, totalTokens:number}}
-   */
-  _extractOllamaMetrics(responseData) {
-    return this.transport.extractMetrics(responseData);
-  }
-
-  /**
-   * Logs the prompt and response to a file. Combines the system prompt, user
-   * prompt, and response into a formatted string and writes it to a file.
-   * @param {string} systemPrompt - The system message provided to the model.
-   * @param {string} prompt - The user's input prompt.
-   * @param {Object} response - The model's generated response object.
-   * @return {Promise<void>} A promise that resolves when the prompt and
-   *          response are written to the file.
-   */
-  async _logPromptAndResponse(systemPrompt, prompt, response) {
-    const content =
-      '================================================================================' +
-      prompt +
-      '\n\n' +
-      JSON.stringify(response) +
-      '\n\n' +
-      '================================================================================\n\n';
-
-    await writePromptToFile(systemPrompt, content);
-  }
-
-  /**
-   * Generate text based on a prompt
-   * @param {string} prompt - The prompt to generate text from
-   * @param {object} [options] - Optional overrides for this one call
-   * @param {string} [options.systemPrompt] - Replaces the generic system prompt
-   * @param {number} [options.temperature] - Overrides AI_TEMPERATURE_GENERATION
-   * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS (num_predict)
-   * @param {AbortSignal} [options.signal] - Cancels the request in flight
-   * @param {Function} [options.onProgress] - Streams the answer and reports it
-   *   as it grows; see GenerateTextProgress in aiGenerateOptions
-   * @param {boolean} [options.reasoning] - Sends `think` for this one call and
-   *   overrules OLLAMA_THINK; without it the setting decides, as before
-   * @returns {Promise<string>} - The generated text
-   */
-  async generateText(prompt, options = {}) {
-    const responseTokens =
-      hasNumber(options.maxTokens) && options.maxTokens > 0
-        ? Math.floor(options.maxTokens)
-        : Number(config.responseTokens);
-
-    // Calculate context window size based on prompt length
-    const promptTokenCount = this._calculatePromptTokenCount(prompt);
-    const numCtx = this._calculateNumCtx(promptTokenCount, responseTokens);
-
-    return this.transport.generate(prompt, options, { responseTokens, numCtx });
-  }
-
-  /**
-   * Check if the Ollama service is running
-   * @returns {Promise<boolean>} - True if the service is running, false otherwise
-   */
-  async checkStatus() {
-    // use ollama status endpoint
-    try {
-      const response = await this.client.get(`${this.apiUrl}/api/ps`, {
-        headers: this._buildRequestHeaders(),
-      });
-      if (response.status === 200) {
-        const data = response.data;
-        // Ensure data is an array and has at least one model
-        let modelName = null;
-        if (Array.isArray(data.models) && data.models.length > 0) {
-          modelName = data.models[0].name;
-        }
-        console.log('Ollama model name:', modelName);
-        return { status: 'ok', model: modelName };
-      }
-    } catch (error) {
-      console.error(`Error checking Ollama service status: ${error.message}`);
-      console.debug(error);
-    }
-    return { status: 'error' };
-  }
-}
-
-module.exports = new OllamaService();
+// The Ollama provider is a composition root over the shared DocumentAnalyzer
+// (services/llm/). This path stays so its callers need no change.
+module.exports = require('./llm/providers/ollamaProvider');
diff --git a/tests/test-llm-provider-parity.js b/tests/test-llm-provider-parity.js
index 3750f06b..e8ec9830 100644
--- a/tests/test-llm-provider-parity.js
+++ b/tests/test-llm-provider-parity.js
@@ -1,5 +1,6 @@
 /**
- * The same answer gives the same result on all four providers.
+ * The same answer gives the same result on all four providers, and the
+ * four are one DocumentAnalyzer each behind a re-export.
  *
  * Step 6 of the LLM service refactoring (services/ManualServiceRefactoring.md,
  * "Done means"): once the four providers share one DocumentAnalyzer, an
@@ -16,6 +17,7 @@
 
 const assert = require('assert');
 const fs = require('fs').promises;
+const path = require('path');
 
 process.env.AI_PROVIDER = 'openai';
 process.env.OPENAI_API_KEY = 'test-key';
@@ -36,6 +38,8 @@ const {
   getThumbnailCachePath,
 } = require('../services/thumbnailCachePaths');
 
+const { DocumentAnalyzer } = require('../services/llm/documentAnalyzer');
+
 const PROVIDERS = {
   OpenAI: require('../services/openaiService'),
   Azure: require('../services/azureService'),
@@ -43,6 +47,14 @@ const PROVIDERS = {
   Ollama: require('../services/ollamaService'),
 };
 
+/** The module each provider's legacy path re-exports. */
+const ROOTS = {
+  OpenAI: ['openaiService', 'openaiProvider'],
+  Azure: ['azureService', 'azureProvider'],
+  Custom: ['customService', 'customProvider'],
+  Ollama: ['ollamaService', 'ollamaProvider'],
+};
+
 /** Own id so the seeded thumbnail cannot collide with a real cached document. */
 const DOCUMENT_ID = 'testLlmProviderParity';
 
@@ -144,6 +156,27 @@ const check = async (label, fn) => {
     Buffer.from([0x89, 0x50, 0x4e, 0x47])
   );
 
+  /* The structure behind the parity: one analyzer, four composition roots,
+     and the old module paths as plain re-exports. */
+  for (const [label, [service, provider]] of Object.entries(ROOTS)) {
+    await check(`${label}: a DocumentAnalyzer behind a re-export`, async () => {
+      assert.ok(PROVIDERS[label] instanceof DocumentAnalyzer);
+      assert.strictEqual(
+        PROVIDERS[label],
+        require(`../services/llm/providers/${provider}`)
+      );
+      const source = await fs.readFile(
+        path.join(__dirname, '..', 'services', `${service}.js`),
+        'utf8'
+      );
+      const code = source.replace(/^\/\/.*$/gm, '').trim();
+      assert.strictEqual(
+        code,
+        `module.exports = require('./llm/providers/${provider}');`
+      );
+    });
+  }
+
   for (const [name, text] of ANSWERS) {
     await check(`${name}: byte-identical on all four`, async () => {
       const results = {};

From 4c511aa209aba708771f03e564d5c8b6446b6cac Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:20:30 +0200
Subject: [PATCH 47/53] Record step 6 as done and hand over for step 7

Background
Step 6 of the LLM service refactoring is complete: the decisions of
2026-10-11 are implemented, the four providers are composition roots over one
DocumentAnalyzer, and llm-provider-parity pins byte-identical results. The
plan document is where whoever continues looks first.

Changes
- services/ManualServiceRefactoring.md:
  - Step 6 marked done: the order (decisions, parity, structure), where the
    analyzer's constructor differs from the sketch and why (promptLayout,
    count()/cut(), analysisTemperature, responseSchema, contextWindow,
    prepareContent, describeFailure), the request probe, the logging
    changes, what was kept as decided, and what was taken from the
    refactor-step-6 branch (PR #1) and what not.
  - Drift table: rows a, c and k resolved; row g says where checkStatus
    lives now. The note on the pinned drift cases says row k flipped.
  - Handoff notes rewritten for step 7 onwards, with the seams as they are
    now.
  - Follow-ups: the Anthropic provider from upstream v2026.10.01 (not one of
    the four; has 7k, lacks the field defaults and 7c), the unwritten
    contracts.js, and the CachedNameIdEnum note updated now that the
    generateText() contract is settled.

Testing
Documentation only; Prettier clean. Full suite at this commit,
node scripts/run-tests.js --all on Windows, Node v24.19.0: 112 passed,
7 skipped, 22 failed. The 22 are the same tests that fail on the merge
commit 8dd6121 (environmental: temp-dir EPERM, native crashes, CRLF working
copy, the shell's ANTHROPIC_BASE_URL); the 3 more passing tests are the new
llm-azure-temperature-wiring, llm-provider-parity and llm-document-analyzer.
OpenAPI regenerated, unchanged.

Impact
None at runtime.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/ManualServiceRefactoring.md | 165 +++++++++++++++------------
 1 file changed, 92 insertions(+), 73 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index d8a1721e..75c39838 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -179,27 +179,28 @@ features.
 The refactor collapses four implementations into one, which forces a choice wherever they differ
 today. These are the actual bugs the work will surface:
 
-| #   | Drift                                                                                                                       | Location                                                                                                                                                                         | Proposed resolution                                                                                                                                                                                                                                         |
-| --- | --------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| a   | Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`                                   | `azureService.js:286`                                                                                                                                                            | Use the config value. Fixes a real bug; `test-ai-temperature-config.js` only covers config parsing, not the Azure wiring, so add a wiring test.                                                                                                             |
-| b   | ~~`return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined`~~ **resolved**          | was `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                          | Fixed ahead of the refactor together with row j — same statement block, same root cause. See the note below the table.                                                                                                                                      |
-| c   | "Insufficient content" early return exists only in OpenAI                                                                   | `openaiService.js:310-332`                                                                                                                                                       | Keep, apply to all providers. The phrase feeds OCR fallback (Finding 4).                                                                                                                                                                                    |
-| d   | ~~Three different JSON parsers~~ **resolved**                                                                               | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | Done in step 3: `services/llm/responseParser.js` — custom's brace matcher, `<think>` stripping, Ollama's repair and object passthrough.                                                                                                                     |
-| e   | Only custom normalizes timeouts                                                                                             | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                                                                                                                        |
-| f   | `generateText` option handling — **mostly resolved upstream**                                                               | `services/aiGenerateOptions.js`, used by all four `generateText()` implementations                                                                                               | Upstream unified how options are read (completion cap, temperature, system prompt, model, abort, reasoning, progress). Keep `aiGenerateOptions.js` as the shared reader; the transports only map the result onto their own knobs.                           |
-| g   | `checkStatus` differs by nature                                                                                             | openai/custom `models.list()`; Azure REST GET; `ollamaService.js:1210` `/api/ps`                                                                                                 | Stays in the transport. Not business logic.                                                                                                                                                                                                                 |
-| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`                                    | `openaiService.js:46-60`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop.                                                                                          |
-| i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt                              | all four                                                                                                                                                                         | Still out of scope as a _placement_ question, but the four copies are now one call to `services/thumbnailCache.js`. Moving it out of `analyzeDocument` is a one-line change once a caller wants it.                                                         |
-| j   | ~~`getThumbnailCachePath(id)` runs **outside** the try block, so an invalid id makes `analyzeDocument` throw~~ **resolved** | was `openaiService.js:63`, `azureService.js:52`, `customService.js:147`                                                                                                          | Fixed ahead of the refactor. See the note below the table.                                                                                                                                                                                                  |
-| k   | An answer with neither tags nor a correspondent is a **failure** on three providers and a **success** on Ollama             | `openaiService.js:316-329`, `azureService.js`, `customService.js` vs. `ollamaService.js:223`                                                                                     | Ollama logs "No tags or correspondent found…" and returns an all-null `document` with populated `metrics` and **no `error`**. So the scan loop writes nulls back to Paperless and never falls back to OCR. Adopt the throwing behaviour of the other three. |
-| l   | An empty provider payload is worded "No response data from Ollama API", which matches **no** OCR marker                     | `ollamaService.js:821` vs. `openaiService.js`, `azureService.js`, `customService.js`                                                                                             | The other three say "Invalid API response structure", which `shouldQueueForOcrOnAiError()` matches. Unify the wording on the shared phrase, or add this one to the marker list.                                                                             |
-| m   | Custom classifies `socket hang up` as a timeout and replaces the message                                                    | `customService.js:331`                                                                                                                                                           | Consequence of row e: `isTimeoutError()` swallows the original text into "AI response timeout reached…". Decide deliberately whether a dropped connection is a timeout when the handling is unified.                                                        |
+| #   | Drift                                                                                                                            | Location                                                                                                                                                                         | Proposed resolution                                                                                                                                                                                                               |
+| --- | -------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| a   | ~~Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`~~ **resolved**                       | was `azureService.js:286`                                                                                                                                                        | Done in step 6: analysis follows `config.aiTemperatureAnalysis`, `generateText()` defaults to `config.aiTemperatureGeneration`; pinned by `llm-azure-temperature-wiring`.                                                         |
+| b   | ~~`return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined`~~ **resolved**               | was `openaiService.js:87`, `azureService.js:76`, `customService.js:171`                                                                                                          | Fixed ahead of the refactor together with row j — same statement block, same root cause. See the note below the table.                                                                                                            |
+| c   | ~~"Insufficient content" early return exists only in OpenAI~~ **resolved**                                                       | was `openaiService.js:310-332`                                                                                                                                                   | Done in step 6: `isInsufficientContentAnswer()` in `responseParser.js`, applied by the `DocumentAnalyzer` after a failed parse, so all four report `ai_insufficient_content`.                                                     |
+| d   | ~~Three different JSON parsers~~ **resolved**                                                                                    | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | Done in step 3: `services/llm/responseParser.js` — custom's brace matcher, `<think>` stripping, Ollama's repair and object passthrough.                                                                                           |
+| e   | Only custom normalizes timeouts                                                                                                  | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                                                                                              |
+| f   | `generateText` option handling — **mostly resolved upstream**                                                                    | `services/aiGenerateOptions.js`, used by all four `generateText()` implementations                                                                                               | Upstream unified how options are read (completion cap, temperature, system prompt, model, abort, reasoning, progress). Keep `aiGenerateOptions.js` as the shared reader; the transports only map the result onto their own knobs. |
+| g   | `checkStatus` differs by nature                                                                                                  | openai/custom `models.list()`; Azure REST GET; `ollamaService.js:1210` `/api/ps`                                                                                                 | Not business logic. Each composition root hands its probe to the analyzer unchanged (step 6); nothing calls it, so step 8 decides whether it stays.                                                                               |
+| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`                                         | `openaiService.js:46-60`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop.                                                                |
+| i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt                                   | all four                                                                                                                                                                         | Still out of scope as a _placement_ question, but the four copies are now one call to `services/thumbnailCache.js`. Moving it out of `analyzeDocument` is a one-line change once a caller wants it.                               |
+| j   | ~~`getThumbnailCachePath(id)` runs **outside** the try block, so an invalid id makes `analyzeDocument` throw~~ **resolved**      | was `openaiService.js:63`, `azureService.js:52`, `customService.js:147`                                                                                                          | Fixed ahead of the refactor. See the note below the table.                                                                                                                                                                        |
+| k   | ~~An answer with neither tags nor a correspondent is a **failure** on three providers and a **success** on Ollama~~ **resolved** | was `ollamaService.js:223`                                                                                                                                                       | Done in step 6: the analyzer runs `assertAssignableMetadata()` on the parsed answer, before `withDocumentDefaults()`, for all four.                                                                                               |
+| l   | An empty provider payload is worded "No response data from Ollama API", which matches **no** OCR marker                          | `ollamaService.js:821` vs. `openaiService.js`, `azureService.js`, `customService.js`                                                                                             | The other three say "Invalid API response structure", which `shouldQueueForOcrOnAiError()` matches. Unify the wording on the shared phrase, or add this one to the marker list.                                                   |
+| m   | Custom classifies `socket hang up` as a timeout and replaces the message                                                         | `customService.js:331`                                                                                                                                                           | Consequence of row e: `isTimeoutError()` swallows the original text into "AI response timeout reached…". Decide deliberately whether a dropped connection is a timeout when the handling is unified.                              |
 
 Rows k and l were found by writing the step-1 safety net, not by reading the code — both are
 silent: the analysis "succeeds", the log looks plausible, and the document simply never reaches
 OCR. Both are pinned as-is in the new tests and marked as known drift, so the case starts failing
 the moment the refactor unifies the path. That is deliberate: it forces whoever unifies it to
-update the expectation rather than to not notice.
+update the expectation rather than to not notice. Row k was unified in step 6 and its case flipped;
+row l is still pinned as drift.
 
 **Rows b and j were fixed before the refactor started.** They shared one root cause — a best-effort
 side effect written as a blocking precondition — and sat within three lines of each other, so
@@ -577,40 +578,51 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
      view, no service, no test. It stays in the four services untouched; step 6 decides whether the
      composition roots expose it at all or whether it is dropped with the dead branches of step 8.
 
-6. **Introduce `documentAnalyzer.js`** and the four composition roots. Reduce
-   `services/{openai,azure,custom,ollama}Service.js` to re-export shims. Add `llm-provider-parity`.
-   Fix Finding 7a here (Azure temperature) with its test.
-
-   **Decisions, taken by the maintainer on 2026-10-11** — these are the behaviour changes of this
-   step; each needs its own test that fails before and passes after:
-   - **7a — Azure temperature follows the config on both paths.** Analysis sends
-     `config.aiTemperatureAnalysis` (was a hardcoded 0.3); `generateText()` defaults to
-     `config.aiTemperatureGeneration` (was a hardcoded 0.7). Planned test:
-     `llm-azure-temperature-wiring`. Update the case in `llm-transport-request-mapping` that pins
-     0.3 ("Azure: analysis still sends 0.3").
-   - **7k — an answer without assignable metadata is a failure on all four.** Ollama adopts
-     `assertAssignableMetadata()` and so the OCR fallback. Order matters: the check runs on the
-     **parsed answer, before** `withDocumentDefaults()` — the defaults turn a missing `tags` into
-     `[]` and would make every answer pass. Note what the check really is: `tags` must be an array
-     and `correspondent` a string or null; `{ tags: [], correspondent: null }` passes on all four,
-     as it does on the chat providers today. The pinned drift case in
-     `llm-analysis-result-contract` ("NOT reported as a failure (known drift)") flips to the
-     failure assertion and Ollama joins the loop above it; in `llm-ocr-fallback-phrases` Ollama joins the "an answer without metadata routes to OCR" loop it is filtered out of today.
-   - **Field defaults for all four.** `withDocumentDefaults()` shapes every success: the seven
-     fields (`tags` an array, the rest a value or null), unknown fields dropped. Safe for the
-     write-back: `documentProcessingService.buildUpdateData()` falls back to the document's own
-     title/date and skips a null correspondent, type, language and custom fields. The chat cases in
-     `llm-analysis-result-contract`, `llm-response-parser` (`sameDocument()`) and
-     `llm-token-budget` that compare the document exactly need the defaults in their expectation.
-   - **7c — refusal detection for all four.** OpenAI's "i'm sorry" / "i cannot" / "insufficient"
-     check after a failed parse, with `buildInsufficientContent()`, moves into the analyzer. The OCR
-     reason for those documents changes from the invalid-JSON reason to "insufficient content" on
-     azure, custom and Ollama. Check that `llm-response-parser`'s unreadable answers contain none
-     of the three phrases (they do not today), and extend the OpenAI-only refusal case of `llm-ocr-fallback-phrases` (reason `ai_insufficient_content`) to all four.
-   - **Row g — `checkStatus()`** has no caller. The composition roots delegate it unchanged; whether
-     it goes is decided with step 8.
-   - **Not decided, keep as is:** row e/m (timeout normalization only on Custom), row l ("No
-     response data from Ollama API" matches no OCR marker), Ollama's `CONTENT_MAX_LENGTH` cut.
+6. ~~**Introduce `documentAnalyzer.js`** and the four composition roots~~ **Done.** Built on upstream
+   v2026.10.01 (merged first, `8dd6121`), in this order, every commit green and tagged
+   `refactor-step-6-checkpoint-<n>-…`:
+   - **The decisions first, on the four separate services**, each with a test that failed before
+     and passes after (decided by the maintainer on 2026-10-11):
+     - **7a** — Azure analysis sends `config.aiTemperatureAnalysis` (was 0.3), `generateText()`
+       defaults to `config.aiTemperatureGeneration` (was 0.7); `llm-azure-temperature-wiring`.
+     - **7k** — an answer without assignable metadata is a failure on all four, checked on the
+       parsed answer before `withDocumentDefaults()`; `{ tags: [], correspondent: null }` still
+       passes. Ollama now reaches the OCR queue for it.
+     - **Field defaults** — every success is shaped by `withDocumentDefaults()`: the seven fields,
+       unknown ones dropped. Safe for the write-back (`buildUpdateData()` falls back to the
+       document's own title/date and skips nulls).
+     - **7c** — `isInsufficientContentAnswer()` (`responseParser.js`): a refusal that does not parse
+       returns `buildInsufficientContent()` on all four (OCR reason `ai_insufficient_content`).
+       `llm-response-parser` checks that its unreadable answers contain none of the phrases.
+   - **Then the parity pinned**: `llm-provider-parity` sends twelve answers to all four providers
+     and compares the JSON of the results byte for byte. It passed before the structural commits
+     and had to stay green through them; it also checks that each service is a `DocumentAnalyzer`
+     behind a one-line re-export.
+   - **Then the structure, in three commits** (OpenAI + Azure, Custom, Ollama).
+     `services/llm/documentAnalyzer.js` holds `analyzeDocument()`; `generateText()` and
+     `checkStatus()` delegate. The constructor differs from the sketch under Target Architecture
+     where the code demanded it: a `promptLayout` (`services/llm/promptLayout.js`) says how
+     instructions and document become one request — the chat providers send the instructions as
+     the system message, Ollama puts them in the prompt and sends a system prompt of its own; the
+     token budgets gained `count()`/`cut()` for the external API data; the optional
+     `analysisTemperature` (OpenAI leaves it out for o3-mini), `responseSchema`, `contextWindow`
+     (Ollama's `num_ctx`, sized through the singleton's `_calculate*()` seams on both paths),
+     `prepareContent` (Ollama's `CONTENT_MAX_LENGTH`) and `describeFailure` (Custom's timeout
+     wording, rows e/m) carry what is a provider's own. `checkStatus` is the provider's old probe,
+     moved unchanged (row g). The four `services/*Service.js` are one-line re-exports.
+     A request probe compared the outgoing request bodies and results of all four providers with
+     the state before the structural commits in eight scenarios (plain, custom prompt, external
+     API data, `USE_PROMPT_TAGS`, `USE_EXISTING_DATA`, `CONTENT_MAX_LENGTH`, cut to `TOKEN_LIMIT`,
+     refusal): byte-identical. What changed is logging only: Custom writes `prompt.txt` like the
+     other chat providers, Ollama writes `prompt.txt` before the request and `response.txt` after
+     it, and a failed analysis names its provider.
+   - **Kept as is, as decided:** row e/m (timeout normalization only on Custom), row l ("No
+     response data from Ollama API", now raised by the Ollama transport, still matches no OCR
+     marker), Ollama's `CONTENT_MAX_LENGTH` cut.
+   - `tests/test-llm-document-analyzer.js` pins the analyzer with fake collaborators. The
+     `refactor-step-6` branch (PR #1) was used where it held up: its 7a/7k/defaults/7c test changes
+     and the refusal helper. Its analyzer covered OpenAI only and its parity test compared method
+     signatures, so neither was taken over.
 
 7. **Delete `services/manualService.js`** and repoint `POST /manual/analyze`
    (`routes/setup.js:7477`) at `AIServiceFactory.getService()`, collapsing its four-way switch.
@@ -632,34 +644,32 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
 
 ---
 
-## Handoff notes for step 6 onwards
+## Handoff notes for step 7 onwards
 
-Written when steps 1-5 were done (tag `refactor-step-5-done`), for whoever continues — human or
-agent.
+Written when step 6 was done (tag `refactor-step-6-done`), for whoever continues — human or agent.
 
-- **Read first:** `/CLAUDE.md`, this plan (Findings, Target Architecture, the done notes of steps
-  1-5), `services/llm/` (`promptBuilder`, `responseParser`, `analysisResult`, `tokenBudget/`,
-  `transports/`) and one provider service, e.g. `services/azureService.js`.
+- **Read first:** `/CLAUDE.md`, this plan (Findings, the done notes of steps 1-6),
+  `services/llm/documentAnalyzer.js`, `services/llm/promptLayout.js` and one composition root,
+  e.g. `services/llm/providers/azureProvider.js`.
 - **Seams the tests rely on — keep them on the four exported singletons:** `service.client` is
-  replaced directly by many tests; `lastGenerateTextUsage` is read and set; Ollama's `apiUrl`,
-  `model`, `documentAnalysisSchema`, `_buildPrompt()`, `_callOllamaAPI(prompt, system, numCtx,
-schema)`, `_extractOllamaMetrics()`, `_calculatePromptTokenCount()` and `_calculateNumCtx()`
-  are called or stubbed. `ollama-temperature-wiring` mocks axios through `require.cache` for
-  `services/ollamaService.js`, so the axios client must still be created by that module.
-  `ollama-upstream-error` patches the global `setTimeout` for the retry delay.
+  replaced directly by many tests; `lastGenerateTextUsage` is read and set; `service.transport` is
+  inspected and its `generate` stubbed. Ollama's `apiUrl`, `model`, `documentAnalysisSchema`,
+  `_buildPrompt()`, `_callOllamaAPI(prompt, system, numCtx, schema)`, `_extractOllamaMetrics()`,
+  `_calculatePromptTokenCount()` and `_calculateNumCtx()` are called or stubbed; they live on the
+  singleton in `ollamaProvider.js`. `ollama-temperature-wiring` mocks axios through
+  `require.cache` and loads `services/ollamaService.js` first in a fresh process, so the client is
+  created with the mock. `ollama-upstream-error` patches the global `setTimeout`.
 - **Rules:** TDD; new tests are plain node scripts in `tests/`, registered in `TESTS` and an
   `AREAS` entry of `scripts/run-tests.js`; ESLint and Prettier clean on every touched file; one
   green commit per sub-step, commit body with Background / Changes / Testing / Impact / Upstream
   Status; English only in the repo.
-- **Verify like this:** `node scripts/run-tests.js --all` takes about five minutes. Some tests
-  fail for reasons unrelated to the code, depending on the machine (native modules, file
-  permissions, line endings in the working copy, a changelog/version mismatch inherited from
-  upstream). Never judge a run by its count: run the suite on the parent commit as well and compare
-  the lists of failed tests — the change is fine when the lists are identical apart from tests it
-  added. Back every new test with a mutation check: break the code it guards, see it fail, restore.
-- **Done means:** the four services are re-export shims of `services/llm/providers/*`; the tests
-  above pass unmodified except where a decision of step 6 says otherwise; `llm-provider-parity`
-  shows byte-identical `document` objects for the same payload on all four.
+- **Verify like this:** `node scripts/run-tests.js --all` takes about ten minutes. Some tests
+  fail for reasons unrelated to the code, depending on the machine (native crashes, temp-dir
+  permissions, line endings, environment variables of the shell). Never judge a run by its count:
+  run the suite on the parent commit as well and compare the lists of failed tests. Back every new
+  test with a mutation check: break the code it guards, see it fail, restore.
+- **Step 7 can now use the factory:** every provider has the same `analyzeDocument()` signature
+  and result, so `/manual/analyze` can call `AIServiceFactory.getService()` without a switch.
 
 ## Out of Scope / Follow-ups
 
@@ -673,11 +683,20 @@ schema)`, `_extractOllamaMetrics()`, `_calculatePromptTokenCount()` and `_calcul
 - **`CachedNameIdEnum`'s semantic match calls a method no provider has.** `_findSemanticMatch()`
   (`services/CachedNameIdEnum.js:176`) calls `ai_service.generateAnswer()`, which exists only in
   test stubs. The call throws, is caught and returns "no match", so tag/correspondent/document type
-  lookup silently falls back to its text matchers. Either point it at `generateText(prompt, options)`
-  with a JSON schema once step 6 has settled that contract, or drop the AI tier.
+  lookup silently falls back to its text matchers. Step 6 settled the contract — `generateText(prompt,
+options)` is the same on all four, the Anthropic provider included — so either point it there
+  (it would then cost an AI call per unmatched name), or drop the AI tier.
 - **A structured-output (`response_format`/`format`) path for the chat transports** to match what
   Ollama already does. Would likely remove most of `responseParser`, but it is a behavior change
   needing its own validation against real providers.
+- **The Anthropic provider** (`services/anthropicService.js`, upstream v2026.10.01) arrived after
+  the plan and is not one of the four. It keeps its own `analyzeDocument()` over the Messages API,
+  with its own prompt and parser (`services/analysisPrompt.js`), structured output and prompt
+  caching. It already fails an answer without metadata with the same message (7k), but passes the
+  parsed answer on without the field defaults and has no refusal detection (7c). Making it a fifth
+  composition root needs a Messages transport and a decision on those two differences.
+- **`services/llm/contracts.js`** with the typedefs of "The delegation contracts" was not written;
+  the constructor JSDoc of `DocumentAnalyzer` and the transports document the shapes instead.
 - **This plan document's location.** `/CLAUDE.md` asks for records in the commit body rather than
   Markdown files, and `services/` holds services. Move it to `docs/` or fold it into the PR
   description before the refactor lands.

From 0a7a2f8960424576dcff29dc5730b92591639012 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:38:39 +0200
Subject: [PATCH 48/53] Record the step 7 decisions in the refactoring plan
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 7 of the LLM service refactoring repoints POST /manual/analyze at the
AI service factory. The plan left two questions to the maintainer — whether
token metrics are recorded for every provider, and what happens to the `id`
default — and its line references predate the upstream v2026.10.01 merge.
Both were decided on 2026-10-15 so whoever carries out step 7 does not have
to guess.

Changes
- services/ManualServiceRefactoring.md, step 7:
  - Decisions: metrics for every provider, recorded when the analysis
    returned metrics and the request carried an id (as the Anthropic branch
    does); no API change — `id` stays optional and is passed through as it
    is.
  - The route sits at routes/setup.js:8273 and has five branches since
    upstream added Anthropic, whose factory branch is the template.
  - Two findings to fix on the way: the OpenAI branch reads
    metrics.promptTokens unguarded, so a failed analysis answers 500 with a
    TypeError; and the @swagger text describes a flat response and
    OpenAI-only metrics, while the route returns the AnalysisResult.
    Correcting the text to what the route returns is no API change.
- Handoff notes point at the recorded decisions.

Testing
Documentation only; Prettier clean. The route, its swagger block and its
only in-repo caller (views/partials/scripts/manual-scripts.ejs) were read
for the references.

Impact
None at runtime.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/ManualServiceRefactoring.md | 37 +++++++++++++++++++---------
 1 file changed, 25 insertions(+), 12 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 75c39838..3c03c72f 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -625,17 +625,29 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
      signatures, so neither was taken over.
 
 7. **Delete `services/manualService.js`** and repoint `POST /manual/analyze`
-   (`routes/setup.js:7477`) at `AIServiceFactory.getService()`, collapsing its four-way switch.
-   Clean up what used to trigger Finding 7j while here: all four branches pass `id || []`
-   (`routes/setup.js:7501`, `:7516`, `:7525`, `:7534`), and `String([])` is
-   `''`, which `getThumbnailCachePath()` rejects. That no longer costs the analysis — the request now
-   succeeds with a warning — but `id || []` is still a nonsense default; pass the id through as-is,
-   or require it. Note that
-   `/manual/analyze` currently calls
-   `documentModel.addOpenAIMetrics()` only on the OpenAI branch — decide explicitly whether metrics
-   are recorded for every provider, and if the
-   response shape changes, update the `@swagger` JSDoc and run `node scripts/regen-openapi.js`
-   (the CI drift check will fail otherwise).
+   (`routes/setup.js:8273` after the v2026.10.01 merge) at `AIServiceFactory.getService()`,
+   collapsing its provider switch — five branches since upstream added Anthropic, whose branch
+   already goes through the factory and is the template. Clean up what used to trigger Finding 7j
+   while here: every branch passes `id || []`, and `String([])` is `''`, which
+   `getThumbnailCachePath()` rejects. That no longer costs the analysis — the request succeeds with
+   a warning — but `id || []` is still a nonsense default.
+
+   **Decisions, taken by the maintainer on 2026-10-15:**
+   - **Metrics for every provider.** `documentModel.addOpenAIMetrics()` is called for every
+     provider, not only OpenAI — when the analysis returned `metrics` and the request carried an
+     `id`, as the Anthropic branch does today. Without an id there is no document to record them
+     against.
+   - **No API change.** `id` stays optional and is passed through as it is (no `|| []`); no new
+     required field, no new status code.
+
+   Also fix on the way: the OpenAI branch reads `analyzeDocument.metrics.promptTokens` without a
+   guard, so a failed analysis (`metrics: null`) answers 500 with a TypeError instead of returning
+   the failure record; the guard above removes it. The `@swagger` JSDoc of the route describes the
+   response wrongly — flat `correspondent`/`title`/`tags`/`documentType` and metrics "when using
+   OpenAI", while the route returns the AnalysisResult `{ document, metrics, truncated }` (plus
+   `error`/`errorCode` on failure). Correcting the text to what the route already returns is no API
+   change; run `node scripts/regen-openapi.js` afterwards (the CI drift check fails otherwise).
+
 8. **Verify Finding 7h** (unreachable `initialize()` branches in the former `openaiService`) and drop
    the dead branches.
 9. **Full verification.** `node scripts/run-tests.js --all`; ESLint/Prettier on every changed file;
@@ -669,7 +681,8 @@ Written when step 6 was done (tag `refactor-step-6-done`), for whoever continues
   run the suite on the parent commit as well and compare the lists of failed tests. Back every new
   test with a mutation check: break the code it guards, see it fail, restore.
 - **Step 7 can now use the factory:** every provider has the same `analyzeDocument()` signature
-  and result, so `/manual/analyze` can call `AIServiceFactory.getService()` without a switch.
+  and result, so `/manual/analyze` can call `AIServiceFactory.getService()` without a switch. The
+  decisions it needs are recorded under step 7.
 
 ## Out of Scope / Follow-ups
 

From 5775203f0525cb56e9c51f0c7b88f39360f3c31b Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:54:43 +0200
Subject: [PATCH 49/53] Route POST /manual/analyze through the AI service
 factory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 7 of the LLM service refactoring (services/ManualServiceRefactoring.md).
The route switched on AI_PROVIDER over five provider branches that had
drifted apart: only OpenAI and Anthropic recorded token metrics, the OpenAI
branch read metrics.promptTokens unguarded, so a failed analysis
(metrics: null) answered 500 with a TypeError instead of the failure record,
and every branch passed `id || []`, an id the thumbnail cache rejects. Since
step 6 every provider has the same analyzeDocument() signature and result,
so the switch is no longer needed. Decisions taken by the maintainer on
2026-10-15: metrics for every provider, no API change.

Changes
- routes/setup.js: POST /manual/analyze calls
  AIServiceFactory.getService().analyzeDocument() with the id as sent, and
  records the metrics through documentModel.addOpenAIMetrics() whenever the
  analysis returned metrics and the request carried an id — the rule the
  Anthropic branch already followed. The four now unused provider imports
  are gone.
- The provider is now chosen the way the scan loop chooses it
  (config.aiProvider, OpenAI when unset), so the "AI provider not configured"
  500 for an unknown AI_PROVIDER no longer exists.
- @swagger: the 200 response references the AnalysisResult schema instead of
  a flat correspondent/title/tags/documentType object with OpenAI-only
  metrics; `existingTags` is documented as accepted but not read (the route
  takes the lists from Paperless-ngx), and the 500 text no longer names the
  removed branch. schemas.js AnalysisResult gains `truncated` and
  `errorCode`, and marks `metrics` and `correspondent` nullable, which is
  what a failed analysis returns. OPENAPI/openapi.json regenerated.
- tests/test-manual-analyze-route.js (new, registered as
  manual-analyze-route in the processing area): mounts the real router
  (tests/helpers/mount-router) and replaces only each provider's
  analyzeDocument() and the metrics store.
- tests/test-thumbnail-optional-analysis.js: two comments put `id || []` in
  the past tense.

Testing
- manual-analyze-route, 9 cases: before the change four failed (no metrics
  on Ollama, Custom and Azure; `[]` instead of undefined without an id; 500
  on a failed OpenAI analysis), after it all pass. Mutation-checked:
  restoring `id || []`, dropping the metrics guard and dropping the id guard
  each fail a case.
- On Windows the test ends red after its cases pass, because the harness
  cannot remove its temp directory (EPERM) — the same environmental failure
  as duplicates-endpoints.
- thumbnail-optional-analysis, llm-analysis-result-contract,
  llm-provider-parity, scan-now-user-id-independence, env-export and
  setup-route-security green. ESLint and Prettier clean on every touched
  file; regen-openapi leaves no further diff.

Impact
The manual analysis page (views/partials/scripts/manual-scripts.ejs) reads
result.document as before. Ollama, Custom and Azure users now see their
manual analyses in the token metrics. A failed OpenAI analysis shows its
error instead of a TypeError.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 OPENAPI/openapi.json                      |  72 ++-----
 routes/setup.js                           | 128 +++----------
 schemas.js                                |  15 ++
 scripts/run-tests.js                      |   2 +
 tests/test-manual-analyze-route.js        | 217 ++++++++++++++++++++++
 tests/test-thumbnail-optional-analysis.js |   6 +-
 6 files changed, 282 insertions(+), 158 deletions(-)
 create mode 100644 tests/test-manual-analyze-route.js

diff --git a/OPENAPI/openapi.json b/OPENAPI/openapi.json
index 4205f5f7..85d854dc 100644
--- a/OPENAPI/openapi.json
+++ b/OPENAPI/openapi.json
@@ -325,6 +325,7 @@
       },
       "AnalysisResult": {
         "type": "object",
+        "description": "What analyzeDocument() returns on every AI provider. A failed analysis is\nreturned, not thrown: `document` is then `{ tags: [], correspondent: null }`,\n`metrics` is null and `error` says why.\n",
         "properties": {
           "document": {
             "type": "object",
@@ -348,6 +349,7 @@
               },
               "correspondent": {
                 "type": "string",
+                "nullable": true,
                 "description": "Suggested correspondent name",
                 "example": "ABC Corporation"
               },
@@ -389,6 +391,8 @@
           },
           "metrics": {
             "type": "object",
+            "nullable": true,
+            "description": "Token usage of the analysis; null when it failed",
             "properties": {
               "promptTokens": {
                 "type": "integer",
@@ -407,10 +411,20 @@
               }
             }
           },
+          "truncated": {
+            "type": "boolean",
+            "description": "Whether the document content was cut to fit the token limit",
+            "example": false
+          },
           "error": {
             "type": "string",
             "description": "Error message if analysis failed",
             "example": null
+          },
+          "errorCode": {
+            "type": "string",
+            "description": "Machine-readable reason of a failed analysis, when the provider gave one",
+            "example": null
           }
         }
       },
@@ -6715,7 +6729,7 @@
     "/manual/analyze": {
       "post": {
         "summary": "Analyze document content manually",
-        "description": "Analyzes document content using the configured AI provider and returns structured metadata.\nThis endpoint processes the document text to extract relevant information such as tags,\ncorrespondent, and document type based on content analysis.\n\nThe analysis is performed using the AI provider configured in the application settings.\n",
+        "description": "Analyzes document content using the configured AI provider and returns structured metadata.\nThis endpoint processes the document text to extract relevant information such as tags,\ncorrespondent, and document type based on content analysis.\n\nThe analysis is performed by the AI provider configured in the application settings, the\nsame one the scan uses. The existing tags, correspondents and document types are read from\nPaperless-ngx. When the request carries an `id`, the token usage is recorded against it.\n",
         "tags": [
           "Documents",
           "API"
@@ -6742,7 +6756,7 @@
                   },
                   "existingTags": {
                     "type": "array",
-                    "description": "List of existing tags in the system to help with tag matching",
+                    "description": "Accepted but not read; the existing tags are taken from Paperless-ngx",
                     "items": {
                       "type": "string"
                     },
@@ -6754,7 +6768,7 @@
                   },
                   "id": {
                     "type": "string",
-                    "description": "Optional document ID for tracking metrics",
+                    "description": "Optional Paperless-ngx document ID; the token usage is recorded against it",
                     "example": "doc_123"
                   }
                 }
@@ -6764,57 +6778,11 @@
         },
         "responses": {
           "200": {
-            "description": "Document analysis results",
+            "description": "The analysis result. A failed analysis is answered with status 200 as well, with\n`error` set and `metrics` null.\n",
             "content": {
               "application/json": {
                 "schema": {
-                  "type": "object",
-                  "properties": {
-                    "correspondent": {
-                      "type": "string",
-                      "description": "Detected correspondent name",
-                      "example": "Acme Corp"
-                    },
-                    "title": {
-                      "type": "string",
-                      "description": "Suggested document title",
-                      "example": "Acme Corp Invoice - August 2023"
-                    },
-                    "tags": {
-                      "type": "array",
-                      "description": "Suggested tags for the document",
-                      "items": {
-                        "type": "string"
-                      },
-                      "example": [
-                        "Invoice",
-                        "Finance"
-                      ]
-                    },
-                    "documentType": {
-                      "type": "string",
-                      "description": "Detected document type",
-                      "example": "Invoice"
-                    },
-                    "metrics": {
-                      "type": "object",
-                      "description": "Token usage metrics (when using OpenAI)",
-                      "properties": {
-                        "promptTokens": {
-                          "type": "number",
-                          "example": 350
-                        },
-                        "completionTokens": {
-                          "type": "number",
-                          "example": 120
-                        },
-                        "totalTokens": {
-                          "type": "number",
-                          "example": 470
-                        }
-                      }
-                    }
-                  }
+                  "$ref": "#/components/schemas/AnalysisResult"
                 }
               }
             }
@@ -6830,7 +6798,7 @@
             }
           },
           "500": {
-            "description": "Server error or AI provider not configured",
+            "description": "Server error, e.g. Paperless-ngx could not be reached",
             "content": {
               "application/json": {
                 "schema": {
diff --git a/routes/setup.js b/routes/setup.js
index 2d36fe66..c53f4827 100644
--- a/routes/setup.js
+++ b/routes/setup.js
@@ -4,9 +4,6 @@ const rateLimit = require('express-rate-limit');
 const cron = require('node-cron');
 const setupService = require('../services/setupService.js');
 const paperlessService = require('../services/paperlessService.js');
-const openaiService = require('../services/openaiService.js');
-const ollamaService = require('../services/ollamaService.js');
-const azureService = require('../services/azureService.js');
 const documentModel = require('../models/document.js');
 const AIServiceFactory = require('../services/aiServiceFactory');
 const configFile = require('../config/config.js');
@@ -31,7 +28,6 @@ const {
   verifySessionToken,
   SESSION_TOKEN_TYPE,
 } = require('./auth.js');
-const customService = require('../services/customService.js');
 const mistralOcrService = require('../services/mistralOcrService');
 const quickstartService = require('../services/quickstartService');
 const reconciliationService = require('../services/reconciliationService');
@@ -8189,7 +8185,9 @@ router.post(
  *       This endpoint processes the document text to extract relevant information such as tags,
  *       correspondent, and document type based on content analysis.
  *
- *       The analysis is performed using the AI provider configured in the application settings.
+ *       The analysis is performed by the AI provider configured in the application settings, the
+ *       same one the scan uses. The existing tags, correspondents and document types are read from
+ *       Paperless-ngx. When the request carries an `id`, the token usage is recorded against it.
  *     tags:
  *       - Documents
  *       - API
@@ -8210,53 +8208,23 @@ router.post(
  *                 example: "Invoice from Acme Corp. Total amount: $125.00, Due date: 2023-08-15"
  *               existingTags:
  *                 type: array
- *                 description: List of existing tags in the system to help with tag matching
+ *                 description: Accepted but not read; the existing tags are taken from Paperless-ngx
  *                 items:
  *                   type: string
  *                 example: ["Invoice", "Finance", "Acme Corp"]
  *               id:
  *                 type: string
- *                 description: Optional document ID for tracking metrics
+ *                 description: Optional Paperless-ngx document ID; the token usage is recorded against it
  *                 example: "doc_123"
  *     responses:
  *       200:
- *         description: Document analysis results
+ *         description: |
+ *           The analysis result. A failed analysis is answered with status 200 as well, with
+ *           `error` set and `metrics` null.
  *         content:
  *           application/json:
  *             schema:
- *               type: object
- *               properties:
- *                 correspondent:
- *                   type: string
- *                   description: Detected correspondent name
- *                   example: "Acme Corp"
- *                 title:
- *                   type: string
- *                   description: Suggested document title
- *                   example: "Acme Corp Invoice - August 2023"
- *                 tags:
- *                   type: array
- *                   description: Suggested tags for the document
- *                   items:
- *                     type: string
- *                   example: ["Invoice", "Finance"]
- *                 documentType:
- *                   type: string
- *                   description: Detected document type
- *                   example: "Invoice"
- *                 metrics:
- *                   type: object
- *                   description: Token usage metrics (when using OpenAI)
- *                   properties:
- *                     promptTokens:
- *                       type: number
- *                       example: 350
- *                     completionTokens:
- *                       type: number
- *                       example: 120
- *                     totalTokens:
- *                       type: number
- *                       example: 470
+ *               $ref: '#/components/schemas/AnalysisResult'
  *       400:
  *         description: Invalid request parameters
  *         content:
@@ -8264,7 +8232,7 @@ router.post(
  *             schema:
  *               $ref: '#/components/schemas/Error'
  *       500:
- *         description: Server error or AI provider not configured
+ *         description: Server error, e.g. Paperless-ngx could not be reached
  *         content:
  *           application/json:
  *             schema:
@@ -8288,71 +8256,25 @@ router.post('/manual/analyze', express.json(), async (req, res) => {
         .json({ error: 'Valid content string is required' });
     }
 
-    if (process.env.AI_PROVIDER === 'openai') {
-      const analyzeDocument = await openaiService.analyzeDocument(
-        content,
-        existingTagsList,
-        existingCorrespondentList,
-        existingDocumentTypesList,
-        id || []
-      );
+    // The factory picks the provider the scan loop uses. Every provider
+    // answers with the same AnalysisResult; metrics need a document to be
+    // recorded against, and a failed analysis carries none.
+    const analysis = await AIServiceFactory.getService().analyzeDocument(
+      content,
+      existingTagsList,
+      existingCorrespondentList,
+      existingDocumentTypesList,
+      id
+    );
+    if (analysis.metrics && id) {
       await documentModel.addOpenAIMetrics(
         id,
-        analyzeDocument.metrics.promptTokens,
-        analyzeDocument.metrics.completionTokens,
-        analyzeDocument.metrics.totalTokens
-      );
-      return res.json(analyzeDocument);
-    } else if (process.env.AI_PROVIDER === 'ollama') {
-      const analyzeDocument = await ollamaService.analyzeDocument(
-        content,
-        existingTagsList,
-        existingCorrespondentList,
-        existingDocumentTypesList,
-        id || []
-      );
-      return res.json(analyzeDocument);
-    } else if (process.env.AI_PROVIDER === 'custom') {
-      const analyzeDocument = await customService.analyzeDocument(
-        content,
-        existingTagsList,
-        existingCorrespondentList,
-        existingDocumentTypesList,
-        id || []
+        analysis.metrics.promptTokens,
+        analysis.metrics.completionTokens,
+        analysis.metrics.totalTokens
       );
-      return res.json(analyzeDocument);
-    } else if (process.env.AI_PROVIDER === 'azure') {
-      const analyzeDocument = await azureService.analyzeDocument(
-        content,
-        existingTagsList,
-        existingCorrespondentList,
-        existingDocumentTypesList,
-        id || []
-      );
-      return res.json(analyzeDocument);
-    } else if (process.env.AI_PROVIDER === 'anthropic') {
-      // Through the factory, which owns the Claude service like every scan
-      // does. Its tokens are billed like OpenAI's, so they are recorded too.
-      const analyzeDocument =
-        await AIServiceFactory.getService().analyzeDocument(
-          content,
-          existingTagsList,
-          existingCorrespondentList,
-          existingDocumentTypesList,
-          id || []
-        );
-      if (analyzeDocument.metrics && id) {
-        await documentModel.addOpenAIMetrics(
-          id,
-          analyzeDocument.metrics.promptTokens,
-          analyzeDocument.metrics.completionTokens,
-          analyzeDocument.metrics.totalTokens
-        );
-      }
-      return res.json(analyzeDocument);
-    } else {
-      return res.status(500).json({ error: 'AI provider not configured' });
     }
+    return res.json(analysis);
   } catch (error) {
     console.error('Analysis error:', error);
     return res.status(500).json({ error: error.message });
diff --git a/schemas.js b/schemas.js
index 5ea569bf..91c4a0df 100644
--- a/schemas.js
+++ b/schemas.js
@@ -193,6 +193,10 @@
  *
  *     AnalysisResult:
  *       type: object
+ *       description: |
+ *         What analyzeDocument() returns on every AI provider. A failed analysis is
+ *         returned, not thrown: `document` is then `{ tags: [], correspondent: null }`,
+ *         `metrics` is null and `error` says why.
  *       properties:
  *         document:
  *           type: object
@@ -209,6 +213,7 @@
  *               example: ["invoice", "utilities", "2023"]
  *             correspondent:
  *               type: string
+ *               nullable: true
  *               description: Suggested correspondent name
  *               example: ABC Corporation
  *             document_type:
@@ -239,6 +244,8 @@
  *                     example: "123.45"
  *         metrics:
  *           type: object
+ *           nullable: true
+ *           description: Token usage of the analysis; null when it failed
  *           properties:
  *             promptTokens:
  *               type: integer
@@ -252,10 +259,18 @@
  *               type: integer
  *               description: Total tokens used
  *               example: 570
+ *         truncated:
+ *           type: boolean
+ *           description: Whether the document content was cut to fit the token limit
+ *           example: false
  *         error:
  *           type: string
  *           description: Error message if analysis failed
  *           example: null
+ *         errorCode:
+ *           type: string
+ *           description: Machine-readable reason of a failed analysis, when the provider gave one
+ *           example: null
  *
  *     Tag:
  *       type: object
diff --git a/scripts/run-tests.js b/scripts/run-tests.js
index a6b87e34..e5c30c39 100644
--- a/scripts/run-tests.js
+++ b/scripts/run-tests.js
@@ -106,6 +106,7 @@ const TESTS = {
   'llm-transport-request-mapping': 'test-llm-transport-request-mapping.js',
   'llm-azure-temperature-wiring': 'test-llm-azure-temperature-wiring.js',
   'llm-provider-parity': 'test-llm-provider-parity.js',
+  'manual-analyze-route': 'test-manual-analyze-route.js',
   'custom-field-longtext': 'test-custom-field-longtext.js',
   'document-processing-service': 'test-document-processing-service.js',
   'settings-custom-field-type-conflict':
@@ -267,6 +268,7 @@ const AREAS = {
     'llm-transport-request-mapping',
     'llm-azure-temperature-wiring',
     'llm-provider-parity',
+    'manual-analyze-route',
     'rescan-fetch-before-delete',
   ],
   prompts: [
diff --git a/tests/test-manual-analyze-route.js b/tests/test-manual-analyze-route.js
new file mode 100644
index 00000000..8791c850
--- /dev/null
+++ b/tests/test-manual-analyze-route.js
@@ -0,0 +1,217 @@
+/**
+ * Test: POST /manual/analyze in routes/setup.js
+ *
+ * The route used to switch on AI_PROVIDER over the provider singletons, and
+ * the five branches had drifted apart: only OpenAI and Anthropic recorded
+ * token metrics, OpenAI read `metrics.promptTokens` unguarded (so a failed
+ * analysis answered 500 with a TypeError instead of the failure record), and
+ * every branch passed `id || []` — a nonsense id that the thumbnail cache
+ * rejects. Since every provider has the same analyzeDocument() signature and
+ * result (services/ManualServiceRefactoring.md, step 6), the route goes
+ * through AIServiceFactory.getService() like the scan loop does.
+ *
+ * The router runs for real on a throwaway database (tests/helpers/mount-router);
+ * only each provider's analyzeDocument() and the metrics store are replaced.
+ *
+ * Covers:
+ *  1. Every provider is reached through the factory with the name lists and
+ *     the id as sent, and its result is answered unchanged
+ *  2. Every provider's metrics are recorded against the id
+ *  3. No id: analyzeDocument() gets undefined, nothing is recorded
+ *  4. A failed analysis (metrics: null) answers 200 with the failure record
+ *  5. Invalid content is a 400 and reaches no provider
+ *  6. A provider that throws is a 500 with its message
+ */
+
+'use strict';
+
+const assert = require('assert');
+const path = require('path');
+
+const { mountRouter, REPO_ROOT } = require('./helpers/mount-router');
+
+const API_KEY = 'test-api-key';
+const PROVIDERS = ['openai', 'ollama', 'custom', 'azure', 'anthropic'];
+const SERVICE_FILES = {
+  openai: 'openaiService.js',
+  ollama: 'ollamaService.js',
+  custom: 'customService.js',
+  azure: 'azureService.js',
+  anthropic: 'anthropicService.js',
+};
+
+let passed = 0;
+let failed = 0;
+
+async function test(name, fn) {
+  try {
+    await fn();
+    console.log(`✅  ${name}`);
+    passed += 1;
+  } catch (error) {
+    console.error(`❌  ${name}`);
+    console.error(`    ${error.stack || error.message}`);
+    failed += 1;
+  }
+}
+
+function successFor(provider) {
+  return {
+    document: {
+      tags: ['Invoice'],
+      correspondent: `Acme via ${provider}`,
+      title: 'Acme invoice',
+      document_type: 'Invoice',
+      document_date: '2026-01-01',
+      language: 'en',
+      custom_fields: {},
+    },
+    metrics: { promptTokens: 10, completionTokens: 5, totalTokens: 15 },
+    truncated: false,
+  };
+}
+
+const FAILURE = {
+  document: { tags: [], correspondent: null },
+  metrics: null,
+  error: 'Invalid JSON response from API',
+  errorCode: undefined,
+};
+
+async function main() {
+  const harness = await mountRouter({
+    stub: ({ paperlessService }) => {
+      paperlessService.listCorrespondentsNames = async () => [
+        { id: 1, name: 'Acme' },
+      ];
+      paperlessService.listTagNames = async () => [{ id: 2, name: 'Invoice' }];
+      paperlessService.listDocumentTypesNames = async () => [
+        { id: 3, name: 'Bill' },
+      ];
+    },
+  });
+
+  const config = require(path.join(REPO_ROOT, 'config', 'config.js'));
+  const services = {};
+  for (const provider of PROVIDERS) {
+    services[provider] = require(
+      path.join(REPO_ROOT, 'services', SERVICE_FILES[provider])
+    );
+  }
+
+  const recordedMetrics = [];
+  harness.documentModel.addOpenAIMetrics = async (...args) => {
+    recordedMetrics.push(args);
+  };
+
+  const calls = [];
+  /** Points the app at one provider and decides what it answers. */
+  function useProvider(provider, answer) {
+    config.aiProvider = provider;
+    process.env.AI_PROVIDER = provider;
+    calls.length = 0;
+    recordedMetrics.length = 0;
+    for (const name of PROVIDERS) {
+      services[name].analyzeDocument = async (...args) => {
+        calls.push({ provider: name, args });
+        if (answer instanceof Error) throw answer;
+        return answer;
+      };
+    }
+  }
+
+  const analyze = async (body) => {
+    const response = await fetch(harness.base + '/manual/analyze', {
+      method: 'POST',
+      headers: { 'x-api-key': API_KEY, 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    return { status: response.status, body: await response.json() };
+  };
+
+  try {
+    for (const provider of PROVIDERS) {
+      await test(`${provider}: reached through the factory, answered unchanged, metrics recorded`, async () => {
+        const result = successFor(provider);
+        useProvider(provider, result);
+
+        const response = await analyze({ content: 'Invoice text', id: '42' });
+
+        assert.strictEqual(response.status, 200);
+        assert.deepStrictEqual(response.body, result);
+        assert.deepStrictEqual(
+          calls.map((call) => call.provider),
+          [provider]
+        );
+        assert.deepStrictEqual(calls[0].args, [
+          'Invoice text',
+          ['Invoice'],
+          ['Acme'],
+          ['Bill'],
+          '42',
+        ]);
+        assert.deepStrictEqual(recordedMetrics, [['42', 10, 5, 15]]);
+      });
+    }
+
+    await test('without an id the provider gets undefined and nothing is recorded', async () => {
+      for (const provider of PROVIDERS) {
+        useProvider(provider, successFor(provider));
+        const response = await analyze({ content: 'Invoice text' });
+        assert.strictEqual(response.status, 200, provider);
+        assert.strictEqual(calls.length, 1, provider);
+        assert.strictEqual(calls[0].args.length, 5, provider);
+        assert.strictEqual(calls[0].args[4], undefined, provider);
+        assert.deepStrictEqual(recordedMetrics, [], provider);
+      }
+    });
+
+    await test('a failed analysis answers 200 with the failure record', async () => {
+      for (const provider of PROVIDERS) {
+        useProvider(provider, FAILURE);
+        const response = await analyze({ content: 'Invoice text', id: '42' });
+        assert.strictEqual(response.status, 200, provider);
+        assert.deepStrictEqual(
+          response.body,
+          {
+            document: { tags: [], correspondent: null },
+            metrics: null,
+            error: 'Invalid JSON response from API',
+          },
+          provider
+        );
+        assert.deepStrictEqual(recordedMetrics, [], provider);
+      }
+    });
+
+    await test('invalid content is a 400 and reaches no provider', async () => {
+      useProvider('openai', successFor('openai'));
+      for (const content of [undefined, '', 42]) {
+        const response = await analyze({ content, id: '42' });
+        assert.strictEqual(response.status, 400, String(content));
+        assert.deepStrictEqual(response.body, {
+          error: 'Valid content string is required',
+        });
+      }
+      assert.strictEqual(calls.length, 0);
+    });
+
+    await test('a provider that throws is a 500 with its message', async () => {
+      useProvider('azure', new Error('provider exploded'));
+      const response = await analyze({ content: 'Invoice text', id: '42' });
+      assert.strictEqual(response.status, 500);
+      assert.deepStrictEqual(response.body, { error: 'provider exploded' });
+      assert.deepStrictEqual(recordedMetrics, []);
+    });
+  } finally {
+    await harness.close();
+  }
+
+  console.log(`\n${passed} passed, ${failed} failed`);
+  if (failed > 0) process.exitCode = 1;
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});
diff --git a/tests/test-thumbnail-optional-analysis.js b/tests/test-thumbnail-optional-analysis.js
index 898516fc..39c2b5c7 100644
--- a/tests/test-thumbnail-optional-analysis.js
+++ b/tests/test-thumbnail-optional-analysis.js
@@ -8,7 +8,7 @@
  * 1. getThumbnailCachePath() ran outside the try block, so an id it rejects
  *    ('' after sanitizing, null, undefined) made analyzeDocument() *throw*
  *    instead of returning a result. The scan loop reads analysis.error and
- *    never expects a throw; POST /manual/analyze passes `id || []`, which is
+ *    never expects a throw; POST /manual/analyze passed `id || []`, which is
  *    exactly such a value, and answered 500 for every request without an id.
  * 2. A thumbnail Paperless does not have returned `undefined` from
  *    analyzeDocument(), and every caller immediately reads `.document` on it.
@@ -134,8 +134,8 @@ const writtenCachePaths = [];
     for (const { label, service, stub } of PROVIDERS) {
       /* --- ids the cache path helper rejects ----------------------------- */
 
-      // `id || []` is what routes/setup.js hands over when the request omits
-      // one, and String([]) sanitizes to the empty string.
+      // `id || []` is what routes/setup.js used to hand over when the request
+      // omitted one, and String([]) sanitizes to the empty string.
       for (const id of [[], undefined, null, '', '///']) {
         await check(
           `${label}: an unusable id (${JSON.stringify(id)}) still analyses`,

From c22555804991082c9d98b2643f9248cfa3bb96dc Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:55:29 +0200
Subject: [PATCH 50/53] Delete the dead services/manualService.js
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 7 of the LLM service refactoring (services/ManualServiceRefactoring.md,
Finding 1). Nothing in the repository requires services/manualService.js —
no route, view, service, test, Dockerfile or pm2 config. It called a
method that does not exist (_parseResponse), carried German user-facing
strings and unused parameters. POST /manual/analyze, the business service
the file was named after, now goes through AIServiceFactory (previous
commit).

Changes
- services/manualService.js removed (284 lines).

Testing
`git grep manualService` finds only the refactoring plan. The factory and
the routes load; manual-analyze-route passes its 9 cases.

Impact
None at runtime; the file was never loaded.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/manualService.js | 284 --------------------------------------
 1 file changed, 284 deletions(-)
 delete mode 100644 services/manualService.js

diff --git a/services/manualService.js b/services/manualService.js
deleted file mode 100644
index cd6f893c..00000000
--- a/services/manualService.js
+++ /dev/null
@@ -1,284 +0,0 @@
-const {
-  writePromptToFile,
-  extractChatMessageContent,
-} = require('./serviceUtils');
-const axios = require('axios');
-const OpenAI = require('openai');
-const config = require('../config/config');
-const AzureOpenAI = require('openai').AzureOpenAI;
-
-class ManualService {
-  constructor() {
-    if (config.aiProvider === 'custom') {
-      this.openai = new OpenAI({
-        apiKey: config.custom.apiKey,
-        baseUrl: config.custom.apiUrl,
-      });
-    } else if (config.aiProvider === 'azure') {
-      this.openai = new AzureOpenAI({
-        apiKey: config.azure.apiKey,
-        endpoint: config.azure.endpoint,
-        deploymentName: config.azure.deploymentName,
-        apiVersion: config.azure.apiVersion,
-      });
-    } else {
-      this.openai = new OpenAI({ apiKey: config.openai.apiKey });
-      this.ollama = axios.create({
-        timeout: 300000,
-      });
-    }
-  }
-
-  async analyzeDocument(content, existingTags, provider) {
-    try {
-      if (provider === 'openai') {
-        return this._analyzeOpenAI(content, existingTags);
-      } else if (provider === 'ollama') {
-        return this._analyzeOllama(content, existingTags);
-      } else if (provider === 'custom') {
-        return this._analyzeCustom(content, existingTags);
-      } else if (provider === 'azure') {
-        return this._analyzeAzure(content, existingTags);
-      }
-      console.error('Error analyzing document: Invalid provider');
-    } catch (error) {
-      console.error('Error analyzing document:', error);
-    }
-    return { tags: [], correspondent: null };
-  }
-
-  async _analyzeOpenAI(content, existingTags) {
-    try {
-      const existingTagsList = existingTags.map((tag) => tag.name).join(', ');
-      const model = process.env.OPENAI_MODEL;
-      const systemPrompt = process.env.SYSTEM_PROMPT;
-      await writePromptToFile(systemPrompt, content);
-      const response = await this.openai.chat.completions.create({
-        model: model,
-        messages: [
-          {
-            role: 'system',
-            content: systemPrompt,
-          },
-          {
-            role: 'user',
-            content: content,
-          },
-        ],
-        ...(model !== 'o3-mini' && {
-          temperature: config.aiTemperatureAnalysis,
-        }),
-      });
-
-      let jsonContent = extractChatMessageContent(
-        response?.choices?.[0]?.message,
-        'ManualService/OpenAI'
-      );
-      if (!jsonContent) {
-        throw new Error('Invalid API response structure');
-      }
-      jsonContent = jsonContent
-        .replace(/```json\n?/g, '')
-        .replace(/```\n?/g, '')
-        .trim();
-
-      let parsedResponse;
-      try {
-        parsedResponse = JSON.parse(jsonContent);
-      } catch (error) {
-        console.error('Failed to parse JSON response:', error);
-        throw new Error('Invalid JSON response from API');
-      }
-
-      if (
-        !Array.isArray(parsedResponse.tags) ||
-        (typeof parsedResponse.correspondent !== 'string' &&
-          parsedResponse.correspondent !== null)
-      ) {
-        throw new Error('Invalid response structure');
-      }
-
-      return parsedResponse;
-    } catch (error) {
-      console.error('Failed to analyze document with OpenAI:', error);
-      return { tags: [], correspondent: null };
-    }
-  }
-
-  async _analyzeAzure(content, existingTags) {
-    try {
-      const existingTagsList = existingTags.map((tag) => tag.name).join(', ');
-
-      const systemPrompt = process.env.SYSTEM_PROMPT;
-      await writePromptToFile(systemPrompt, content);
-      const response = await this.openai.chat.completions.create({
-        model: process.env.AZURE_DEPLOYMENT_NAME,
-        messages: [
-          {
-            role: 'system',
-            content: systemPrompt,
-          },
-          {
-            role: 'user',
-            content: content,
-          },
-        ],
-        temperature: config.aiTemperatureAnalysis,
-      });
-
-      let jsonContent = extractChatMessageContent(
-        response?.choices?.[0]?.message,
-        'ManualService/Azure'
-      );
-      if (!jsonContent) {
-        throw new Error('Invalid API response structure');
-      }
-      jsonContent = jsonContent
-        .replace(/```json\n?/g, '')
-        .replace(/```\n?/g, '')
-        .trim();
-
-      let parsedResponse;
-      try {
-        parsedResponse = JSON.parse(jsonContent);
-      } catch (error) {
-        console.error('Failed to parse JSON response:', error);
-        throw new Error('Invalid JSON response from API.');
-      }
-
-      if (
-        !Array.isArray(parsedResponse.tags) ||
-        (typeof parsedResponse.correspondent !== 'string' &&
-          parsedResponse.correspondent !== null)
-      ) {
-        throw new Error('Invalid response structure');
-      }
-
-      return parsedResponse;
-    } catch (error) {
-      console.error('Failed to analyze document with OpenAI:', error);
-      return { tags: [], correspondent: null };
-    }
-  }
-
-  async _analyzeCustom(content, existingTags) {
-    try {
-      const existingTagsList = existingTags.map((tag) => tag.name).join(', ');
-
-      const systemPrompt = process.env.SYSTEM_PROMPT;
-      const model = config.custom.model;
-      const response = await this.openai.chat.completions.create({
-        model: model,
-        messages: [
-          {
-            role: 'system',
-            content: systemPrompt,
-          },
-          {
-            role: 'user',
-            content: content,
-          },
-        ],
-        ...(model !== 'o3-mini' && {
-          temperature: config.aiTemperatureAnalysis,
-        }),
-      });
-
-      let jsonContent = extractChatMessageContent(
-        response?.choices?.[0]?.message,
-        'ManualService/Custom'
-      );
-      if (!jsonContent) {
-        throw new Error('Invalid API response structure');
-      }
-      jsonContent = jsonContent
-        .replace(/```json\n?/g, '')
-        .replace(/```\n?/g, '')
-        .trim();
-
-      const parsedResponse = JSON.parse(jsonContent);
-
-      if (
-        !Array.isArray(parsedResponse.tags) ||
-        (typeof parsedResponse.correspondent !== 'string' &&
-          parsedResponse.correspondent !== null)
-      ) {
-        throw new Error('Invalid response structure');
-      }
-
-      return parsedResponse;
-    } catch (error) {
-      console.error('Failed to analyze document with OpenAI:', error);
-      return { tags: [], correspondent: null };
-    }
-  }
-
-  async _analyzeOllama(content, existingTags) {
-    try {
-      const prompt = process.env.SYSTEM_PROMPT;
-
-      const calculateNumCtx = (promptTokenCount, expectedResponseTokens) => {
-        const totalTokenUsage = promptTokenCount + expectedResponseTokens;
-        const maxCtxLimit = Number(config.tokenLimit);
-
-        const numCtx = Math.min(totalTokenUsage, maxCtxLimit);
-
-        console.log('Prompt Token Count:', promptTokenCount);
-        console.log('Expected Response Tokens:', expectedResponseTokens);
-        console.log('Dynamic calculated num_ctx:', numCtx);
-
-        return numCtx;
-      };
-
-      const calculatePromptTokenCount = (prompt) => {
-        return Math.ceil(prompt.length / 4);
-      };
-
-      const expectedResponseTokens = 1024;
-      const promptTokenCount = calculatePromptTokenCount(prompt);
-
-      const numCtx = calculateNumCtx(promptTokenCount, expectedResponseTokens);
-
-      const ollamaHeaders = { 'Content-Type': 'application/json' };
-      const ollamaApiKey =
-        process.env.OLLAMA_API_KEY || config.ollama.apiKey || '';
-      if (ollamaApiKey) {
-        ollamaHeaders.Authorization = `Bearer ${ollamaApiKey}`;
-      }
-
-      const response = await this.ollama.post(
-        `${config.ollama.apiUrl}/api/generate`,
-        {
-          model: config.ollama.model,
-          prompt: prompt,
-          stream: false,
-          options: {
-            temperature: config.aiTemperatureGeneration,
-            top_p: 0.9,
-            repeat_penalty: 1.1,
-            num_ctx: numCtx,
-          },
-        },
-        { headers: ollamaHeaders }
-      );
-
-      if (!response.data || !response.data.response) {
-        console.error('Unexpected Ollama response format:', response);
-        throw new Error('Invalid response from Ollama API');
-      }
-
-      return this._parseResponse(response.data.response);
-    } catch (error) {
-      if (error.code === 'ECONNABORTED') {
-        console.error('Timeout bei der Ollama-Anfrage:', error);
-        throw new Error(
-          'Die Analyse hat zu lange gedauert. Bitte versuchen Sie es erneut.'
-        );
-      }
-      console.error('Error analyzing document with Ollama:', error);
-      throw error;
-    }
-  }
-}
-
-module.exports = ManualService;

From 28f6027eb5d2be18ae4442abe2eebb40929b155d Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 20:15:08 +0200
Subject: [PATCH 51/53] Record step 7 as done and hand over for step 8
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 7 of the LLM service refactoring — POST /manual/analyze through the
AI service factory, services/manualService.js deleted — landed in the two
previous commits. The plan has to say so, and what it found on the way, so
whoever continues with step 8 does not have to reconstruct it.

Changes
- services/ManualServiceRefactoring.md:
  - Step 7 marked done, with what the route does now (metrics for every
    provider when the analysis returned metrics and the request carried an
    id, id passed as sent, no API change), the consequence that the
    provider is picked as the scan loop picks it, so the "AI provider not
    configured" 500 is gone, the swagger corrections, the
    manual-analyze-route test with its mutation checks, and the Windows
    EPERM on the harness cleanup.
  - Finding 1's decision is marked as carried out in step 7.
  - Row h: the factory is now the only non-test module that requires
    openaiService, which is what step 8 needs to know.
  - Handoff notes retitled for step 8 onwards, with the router-test harness
    as a pointer.

Testing
Documentation only; Prettier clean. Full suite on c225558 against its
parent 0a7a2f8: 112 passed / 7 skipped / 23 failed versus 112 / 7 / 22;
the failure lists are identical except for manual-analyze-route, whose
nine cases pass and which fails only on the harness's temp-directory
cleanup (EPERM) on Windows.

Impact
None at runtime.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/ManualServiceRefactoring.md | 71 ++++++++++++++++------------
 1 file changed, 40 insertions(+), 31 deletions(-)

diff --git a/services/ManualServiceRefactoring.md b/services/ManualServiceRefactoring.md
index 3c03c72f..e43ae541 100644
--- a/services/ManualServiceRefactoring.md
+++ b/services/ManualServiceRefactoring.md
@@ -61,7 +61,7 @@ Further evidence that `manualService.js` was abandoned rather than maintained:
 - Its Ollama path never sends the document content at all — `prompt` is just
   `process.env.SYSTEM_PROMPT` (`manualService.js:218`).
 
-**Decision:** delete `services/manualService.js` as part of this work and point
+**Decision (carried out in step 7):** delete `services/manualService.js` as part of this work and point
 `POST /manual/analyze` at `AIServiceFactory.getService()`. Do not port its behavior — it is worse
 than the dedicated services on every axis: no token budgeting, no content truncation, no restriction
 placeholders, no structured output, no retry, no metrics.
@@ -188,7 +188,7 @@ today. These are the actual bugs the work will surface:
 | e   | Only custom normalizes timeouts                                                                                                  | `customService.js:428-436`                                                                                                                                                       | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers.                                                                                                                                                              |
 | f   | `generateText` option handling — **mostly resolved upstream**                                                                    | `services/aiGenerateOptions.js`, used by all four `generateText()` implementations                                                                                               | Upstream unified how options are read (completion cap, temperature, system prompt, model, abort, reasoning, progress). Keep `aiGenerateOptions.js` as the shared reader; the transports only map the result onto their own knobs. |
 | g   | `checkStatus` differs by nature                                                                                                  | openai/custom `models.list()`; Azure REST GET; `ollamaService.js:1210` `/api/ps`                                                                                                 | Not business logic. Each composition root hands its probe to the analyzer unchanged (step 6); nothing calls it, so step 8 decides whether it stays.                                                                               |
-| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`                                         | `openaiService.js:46-60`                                                                                                                                                         | `aiServiceFactory` never routes those providers here and both direct call sites in `routes/setup.js` are guarded by `AI_PROVIDER === 'openai'`. Verify, then drop.                                                                |
+| h   | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom`                                         | `openaiService.js:46-60`                                                                                                                                                         | `aiServiceFactory` never routes those providers here. Since step 7 the factory is the only module that requires `openaiService` outside the tests. Verify, then drop.                                                             |
 | i   | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt                                   | all four                                                                                                                                                                         | Still out of scope as a _placement_ question, but the four copies are now one call to `services/thumbnailCache.js`. Moving it out of `analyzeDocument` is a one-line change once a caller wants it.                               |
 | j   | ~~`getThumbnailCachePath(id)` runs **outside** the try block, so an invalid id makes `analyzeDocument` throw~~ **resolved**      | was `openaiService.js:63`, `azureService.js:52`, `customService.js:147`                                                                                                          | Fixed ahead of the refactor. See the note below the table.                                                                                                                                                                        |
 | k   | ~~An answer with neither tags nor a correspondent is a **failure** on three providers and a **success** on Ollama~~ **resolved** | was `ollamaService.js:223`                                                                                                                                                       | Done in step 6: the analyzer runs `assertAssignableMetadata()` on the parsed answer, before `withDocumentDefaults()`, for all four.                                                                                               |
@@ -624,29 +624,33 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
      and the refusal helper. Its analyzer covered OpenAI only and its parity test compared method
      signatures, so neither was taken over.
 
-7. **Delete `services/manualService.js`** and repoint `POST /manual/analyze`
-   (`routes/setup.js:8273` after the v2026.10.01 merge) at `AIServiceFactory.getService()`,
-   collapsing its provider switch — five branches since upstream added Anthropic, whose branch
-   already goes through the factory and is the template. Clean up what used to trigger Finding 7j
-   while here: every branch passes `id || []`, and `String([])` is `''`, which
-   `getThumbnailCachePath()` rejects. That no longer costs the analysis — the request succeeds with
-   a warning — but `id || []` is still a nonsense default.
-
-   **Decisions, taken by the maintainer on 2026-10-15:**
-   - **Metrics for every provider.** `documentModel.addOpenAIMetrics()` is called for every
-     provider, not only OpenAI — when the analysis returned `metrics` and the request carried an
-     `id`, as the Anthropic branch does today. Without an id there is no document to record them
-     against.
-   - **No API change.** `id` stays optional and is passed through as it is (no `|| []`); no new
-     required field, no new status code.
-
-   Also fix on the way: the OpenAI branch reads `analyzeDocument.metrics.promptTokens` without a
-   guard, so a failed analysis (`metrics: null`) answers 500 with a TypeError instead of returning
-   the failure record; the guard above removes it. The `@swagger` JSDoc of the route describes the
-   response wrongly — flat `correspondent`/`title`/`tags`/`documentType` and metrics "when using
-   OpenAI", while the route returns the AnalysisResult `{ document, metrics, truncated }` (plus
-   `error`/`errorCode` on failure). Correcting the text to what the route already returns is no API
-   change; run `node scripts/regen-openapi.js` afterwards (the CI drift check fails otherwise).
+7. ~~**Delete `services/manualService.js`** and repoint `POST /manual/analyze` at
+   `AIServiceFactory.getService()`~~ **Done**, in two commits:
+   - **The route.** The five-branch switch on `process.env.AI_PROVIDER` is one call to
+     `AIServiceFactory.getService().analyzeDocument()` with the `id` as sent (no `|| []`). As decided
+     by the maintainer on 2026-10-15, `documentModel.addOpenAIMetrics()` runs for every provider
+     whenever the analysis returned `metrics` and the request carried an `id` — the Anthropic
+     branch's rule — and the API is unchanged: `id` stays optional, no new field, no new status.
+     That guard also removes the OpenAI branch's unguarded `metrics.promptTokens`, so a failed
+     analysis answers 200 with its failure record instead of a 500 TypeError. The four provider
+     imports of `routes/setup.js` were unused afterwards and are gone.
+
+     One consequence that was not a decision of its own: the provider is now picked as the scan
+     loop picks it (`config.aiProvider`, OpenAI when unset or unknown), so the old 500 "AI provider
+     not configured" no longer exists. The swagger block was corrected to what the route returns:
+     the 200 references `AnalysisResult` (`schemas.js`, previously unreferenced; it gained
+     `truncated` and `errorCode`, and `metrics`/`correspondent` are nullable), `existingTags` is
+     documented as accepted but not read — the route takes all three lists from Paperless-ngx —
+     and the 500 text no longer names the removed branch. `OPENAPI/openapi.json` regenerated.
+
+     `manual-analyze-route` (9 cases) mounts the real router (`tests/helpers/mount-router`) and
+     replaces only each provider's `analyzeDocument()` and the metrics store. Four cases failed
+     before the change; restoring `id || []`, dropping the metrics guard or the id guard each
+     fails one. On Windows the test ends red after its cases pass, because the harness cannot
+     remove its temp directory (EPERM), like `duplicates-endpoints`.
+
+   - **The dead file.** `services/manualService.js` (284 lines) is deleted; `git grep manualService`
+     finds only this plan.
 
 8. **Verify Finding 7h** (unreachable `initialize()` branches in the former `openaiService`) and drop
    the dead branches.
@@ -656,11 +660,12 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
 
 ---
 
-## Handoff notes for step 7 onwards
+## Handoff notes for step 8 onwards
 
-Written when step 6 was done (tag `refactor-step-6-done`), for whoever continues — human or agent.
+Written when step 6 was done (tag `refactor-step-6-done`) and updated after step 7, for whoever
+continues — human or agent.
 
-- **Read first:** `/CLAUDE.md`, this plan (Findings, the done notes of steps 1-6),
+- **Read first:** `/CLAUDE.md`, this plan (Findings, the done notes of steps 1-7),
   `services/llm/documentAnalyzer.js`, `services/llm/promptLayout.js` and one composition root,
   e.g. `services/llm/providers/azureProvider.js`.
 - **Seams the tests rely on — keep them on the four exported singletons:** `service.client` is
@@ -680,9 +685,13 @@ Written when step 6 was done (tag `refactor-step-6-done`), for whoever continues
   permissions, line endings, environment variables of the shell). Never judge a run by its count:
   run the suite on the parent commit as well and compare the lists of failed tests. Back every new
   test with a mutation check: break the code it guards, see it fail, restore.
-- **Step 7 can now use the factory:** every provider has the same `analyzeDocument()` signature
-  and result, so `/manual/analyze` can call `AIServiceFactory.getService()` without a switch. The
-  decisions it needs are recorded under step 7.
+- **After step 7** no route switches on the provider any more: `POST /manual/analyze` goes through
+  the factory like the scan loop, and `aiServiceFactory.js` is the only non-test module that
+  requires the provider singletons. Step 8 (row h) can therefore reason about
+  `openaiService.initialize()` from the factory alone.
+- **Router tests:** `tests/helpers/mount-router.js` mounts the real `routes/setup.js` on a throwaway
+  database; `tests/test-manual-analyze-route.js` is the smallest example. On Windows its `close()`
+  fails with EPERM on the temp directory — expected there; the cases themselves pass.
 
 ## Out of Scope / Follow-ups
 

From 965cca697341d8682713b74388e1049143d27e18 Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 20:48:08 +0200
Subject: [PATCH 52/53] Drop the unreachable Ollama and Custom clients from the
 OpenAI provider
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background
Step 8 of the LLM service refactoring (services/ManualServiceRefactoring.md,
Finding 7h). The OpenAI composition root's createClient() built an
Ollama client (`<OLLAMA_API_URL>/v1`) for AI_PROVIDER=ollama and a Custom
client for AI_PROVIDER=custom. Neither can be reached: since step 7 the
only non-test module that requires openaiService is aiServiceFactory.js,
which hands it out for `openai` and as the default for an unknown provider
only — `ollama` and `custom` get their own composition roots — and no
production code changes config.aiProvider at runtime.

Changes
- services/llm/providers/openaiProvider.js: createClient() builds a client
  for the openai provider with an API key and returns null otherwise, as
  it already did for every provider other than the two dead ones.
- tests/test-llm-transport-request-mapping.js: one case pins that the
  OpenAI provider builds an api.openai.com client for `openai` and none for
  ollama, custom, azure or an unknown provider.

Testing
The new case failed before the change (an Ollama client for `ollama`) and
passes after it. All llm-* tests, ai-provider-streaming,
entity-match-ai-service, response-truncation-detection, token-limit-config
and thumbnail-optional-analysis green; ai-run-cost fails with the same
Windows EBUSY on its temp database before and after. ESLint and Prettier
clean.

Impact
None at runtime; the removed branches were never taken.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 services/llm/providers/openaiProvider.js    | 18 ++++-----------
 tests/test-llm-transport-request-mapping.js | 25 +++++++++++++++++++++
 2 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/services/llm/providers/openaiProvider.js b/services/llm/providers/openaiProvider.js
index c4eae7c7..1168423d 100644
--- a/services/llm/providers/openaiProvider.js
+++ b/services/llm/providers/openaiProvider.js
@@ -27,21 +27,11 @@ const { chatPromptLayout } = require('../promptLayout');
 const TRUNCATION_REMEDY =
   'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.';
 
+/**
+ * The factory hands this service out for the openai provider only (and as
+ * its default for an unknown one), so there is no client for any other.
+ */
 function createClient() {
-  if (config.aiProvider === 'ollama') {
-    return new OpenAI({
-      baseURL: config.ollama.apiUrl + '/v1',
-      // Ollama ignores the key unless the endpoint enforces auth (e.g. a
-      // reverse proxy); the OpenAI SDK requires a non-empty value.
-      apiKey: config.ollama.apiKey || 'ollama',
-    });
-  }
-  if (config.aiProvider === 'custom') {
-    return new OpenAI({
-      baseURL: config.custom.apiUrl,
-      apiKey: config.custom.apiKey,
-    });
-  }
   if (config.aiProvider === 'openai' && config.openai.apiKey) {
     return new OpenAI({
       apiKey: config.openai.apiKey,
diff --git a/tests/test-llm-transport-request-mapping.js b/tests/test-llm-transport-request-mapping.js
index f1f32a19..8eba74d9 100644
--- a/tests/test-llm-transport-request-mapping.js
+++ b/tests/test-llm-transport-request-mapping.js
@@ -492,6 +492,31 @@ const ANALYSIS_REQUEST = {
     }
   );
 
+  // Finding 7h: the factory hands out openaiService for the openai provider
+  // (and as its default) only, so OpenAI builds no client for Ollama or
+  // Custom — those have composition roots of their own.
+  await check(
+    'OpenAI: builds an OpenAI client for the openai provider only (7h)',
+    () => {
+      const previousProvider = config.aiProvider;
+      const build = (provider) => {
+        config.aiProvider = provider;
+        openaiService.client = null;
+        openaiService.transport.initialize();
+        return openaiService.client;
+      };
+      try {
+        assert.match(build('openai').baseURL, /api\.openai\.com/);
+        for (const provider of ['ollama', 'custom', 'azure', 'unknown']) {
+          assert.strictEqual(build(provider), null, provider);
+        }
+      } finally {
+        config.aiProvider = previousProvider;
+        openaiService.client = null;
+      }
+    }
+  );
+
   await check(
     'Azure: analysis sends AI_TEMPERATURE_ANALYSIS (7a)',
     async () => {

From d511101fadf44fae3095f7a6cc59de2edeb078ff Mon Sep 17 00:00:00 2001
From: OvisMaximus <OvisMaximus@users.noreply.github.com>
Date: Thu, 15 Oct 2026 20:59:29 +0200
Subject: [PATCH 53/53] Move the refactoring plan to docs/ and record the
 refactoring as done

Background
Steps 8 and 9 of the LLM service refactoring finish it. The plan's own
follow-up list asked for the document to leave services/ before the
refactor lands: /CLAUDE.md keeps fix records in commit bodies, and
services/ holds services.

Changes
- services/ManualServiceRefactoring.md moved to
  docs/llm-service-refactoring.md (git mv, history kept); the eleven
  comments in services/llm/tokenBudget/charBudget.js and tests/ that cite it
  point at the new path.
- The document now opens with the status (done; steps 1-8, step 9 except
  the live smoke run) and the line count against the expected ~2000: the
  four services went from 2783 lines to one-line re-exports,
  manualService.js (284) is gone, services/llm/ holds 2636.
- Step 8 marked done with how 7h was verified; step 9 points at a new
  "Final verification" section; the handoff notes became notes for whoever
  works on this code next; the location follow-up is closed.

Testing
- node scripts/run-tests.js --all: 112 passed / 7 skipped / 23 failed, the
  same failure list as after step 7 (28f6027), which equals the list before
  step 7 (0a7a2f8) plus manual-analyze-route (its cases pass; the harness
  cleanup hits EPERM on Windows).
- ESLint, Prettier and stylelint on every file the branch changes against
  upstream main, selected as CI selects them: clean (Prettier checked on the
  index content, since five working-copy files have CRLF line endings).
- regen-openapi + git diff --exit-code OPENAPI/openapi.json: no drift.
- Not done: the manual smoke run against a real Paperless-ngx and provider;
  recorded in the document as the remaining check before a release.

Impact
None at runtime.

Upstream Status
Local only; part of the LLM service refactoring.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../llm-service-refactoring.md                | 80 ++++++++++++-------
 services/llm/tokenBudget/charBudget.js        |  2 +-
 tests/test-llm-analysis-result-contract.js    |  2 +-
 tests/test-llm-azure-temperature-wiring.js    |  2 +-
 tests/test-llm-document-analyzer.js           |  2 +-
 tests/test-llm-ocr-fallback-phrases.js        |  2 +-
 tests/test-llm-prompt-builder.js              |  2 +-
 tests/test-llm-provider-parity.js             |  2 +-
 tests/test-llm-response-parser.js             |  2 +-
 tests/test-llm-token-budget.js                |  2 +-
 tests/test-llm-transport-request-mapping.js   |  2 +-
 tests/test-manual-analyze-route.js            |  2 +-
 12 files changed, 63 insertions(+), 39 deletions(-)
 rename services/ManualServiceRefactoring.md => docs/llm-service-refactoring.md (93%)

diff --git a/services/ManualServiceRefactoring.md b/docs/llm-service-refactoring.md
similarity index 93%
rename from services/ManualServiceRefactoring.md
rename to docs/llm-service-refactoring.md
index e43ae541..a17f649f 100644
--- a/services/ManualServiceRefactoring.md
+++ b/docs/llm-service-refactoring.md
@@ -1,5 +1,17 @@
 # LLM Service Refactoring Plan
 
+**Status: done** (2026-10-15) — steps 1-8, and step 9 except the live smoke run. The four
+provider services are composition roots of one `DocumentAnalyzer` (`services/llm/`),
+`POST /manual/analyze` goes through `AIServiceFactory`, and `services/manualService.js` is gone.
+What is left is listed under Out of Scope / Follow-ups. The plan is kept as the record of why the
+code looks the way it does; the commit bodies hold the details of each step.
+
+Measured against the expected benefits: the four services shrank from 2783 lines to four one-line
+re-exports and `manualService.js` (284 lines) is gone, while `services/llm/` holds 2636 lines. The
+saving is smaller than the ~2000 lines expected: the remaining code carries JSDoc, and the checks
+that now apply to every provider — the refusal check (7c), the metadata check (7k), the field
+defaults — each existed in one service before.
+
 ## The Goal
 
 Four provider services offer the same business service through different AI APIs. Implement the
@@ -652,22 +664,41 @@ reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char
    - **The dead file.** `services/manualService.js` (284 lines) is deleted; `git grep manualService`
      finds only this plan.
 
-8. **Verify Finding 7h** (unreachable `initialize()` branches in the former `openaiService`) and drop
-   the dead branches.
-9. **Full verification.** `node scripts/run-tests.js --all`; ESLint/Prettier on every changed file;
-   `node scripts/regen-openapi.js` + `git diff --exit-code OPENAPI/openapi.json`; a manual smoke run
-   of `/manual/analyze` and a scan cycle against a real provider, since no test exercises a live API.
+8. ~~**Verify Finding 7h**~~ **Done.** Verified unreachable: since step 7 the only non-test module
+   that requires `openaiService` is `aiServiceFactory.js`, which hands it out for `openai` and as
+   the default for an unknown provider only, and no production code changes `config.aiProvider` at
+   runtime. `createClient()` in `services/llm/providers/openaiProvider.js` no longer builds an
+   Ollama (`<OLLAMA_API_URL>/v1`) or a Custom client; it builds one for `openai` with an API key and
+   returns `null` otherwise. One case in `llm-transport-request-mapping` pins that — it failed
+   before the change (an Ollama client for `ollama`).
+9. ~~**Full verification.**~~ **Done, except the live smoke run.** See "Final verification" below.
 
 ---
 
-## Handoff notes for step 8 onwards
-
-Written when step 6 was done (tag `refactor-step-6-done`) and updated after step 7, for whoever
-continues — human or agent.
-
-- **Read first:** `/CLAUDE.md`, this plan (Findings, the done notes of steps 1-7),
-  `services/llm/documentAnalyzer.js`, `services/llm/promptLayout.js` and one composition root,
-  e.g. `services/llm/providers/azureProvider.js`.
+## Final verification
+
+On 2026-10-15, at the commit that moved this document to `docs/`:
+
+- `node scripts/run-tests.js --all`: 112 passed, 7 skipped, 23 failed. The failed tests are the same list as on the
+  commit before step 7 (`0a7a2f8`) plus `manual-analyze-route`, whose nine cases pass and which
+  fails only because `tests/helpers/mount-router.js` cannot remove its temp directory on Windows
+  (EPERM) — the reason `duplicates-endpoints` fails there too. The rest of that list is
+  environmental on the Windows development machine (temp-dir EPERM/EBUSY, native crashes, CRLF
+  in the working copy, the shell's `ANTHROPIC_BASE_URL`).
+- ESLint, Prettier and stylelint on every file the branch changes against upstream `main`, the
+  way CI selects them: clean.
+- `node scripts/regen-openapi.js` + `git diff --exit-code OPENAPI/openapi.json`: no drift.
+- **Not done: a manual smoke run** of `/manual/analyze` and a scan cycle against a real
+  Paperless-ngx and a real provider. No test exercises a live API, and none was available here.
+  Do this before the release that carries the refactoring: one document per configured provider
+  through the scan, one through the manual page, and check the token metrics appear for it.
+
+## Notes for whoever works on this code next
+
+- **Read first:** `/CLAUDE.md`, the Findings and done notes above, `services/llm/documentAnalyzer.js`,
+  `services/llm/promptLayout.js` and one composition root, e.g.
+  `services/llm/providers/azureProvider.js`. A new chat-completions provider is one composition
+  root plus a factory case.
 - **Seams the tests rely on — keep them on the four exported singletons:** `service.client` is
   replaced directly by many tests; `lastGenerateTextUsage` is read and set; `service.transport` is
   inspected and its `generate` stubbed. Ollama's `apiUrl`, `model`, `documentAnalysisSchema`,
@@ -676,22 +707,16 @@ continues — human or agent.
   singleton in `ollamaProvider.js`. `ollama-temperature-wiring` mocks axios through
   `require.cache` and loads `services/ollamaService.js` first in a fresh process, so the client is
   created with the mock. `ollama-upstream-error` patches the global `setTimeout`.
-- **Rules:** TDD; new tests are plain node scripts in `tests/`, registered in `TESTS` and an
-  `AREAS` entry of `scripts/run-tests.js`; ESLint and Prettier clean on every touched file; one
-  green commit per sub-step, commit body with Background / Changes / Testing / Impact / Upstream
-  Status; English only in the repo.
+- **The factory is the only door.** No route or service requires a provider singleton directly;
+  `POST /manual/analyze` and the scan loop both call `AIServiceFactory.getService()`.
+- **Router tests:** `tests/helpers/mount-router.js` mounts the real `routes/setup.js` on a throwaway
+  database; `tests/test-manual-analyze-route.js` is the smallest example. On Windows its `close()`
+  fails with EPERM on the temp directory — expected there; the cases themselves pass.
 - **Verify like this:** `node scripts/run-tests.js --all` takes about ten minutes. Some tests
   fail for reasons unrelated to the code, depending on the machine (native crashes, temp-dir
   permissions, line endings, environment variables of the shell). Never judge a run by its count:
   run the suite on the parent commit as well and compare the lists of failed tests. Back every new
   test with a mutation check: break the code it guards, see it fail, restore.
-- **After step 7** no route switches on the provider any more: `POST /manual/analyze` goes through
-  the factory like the scan loop, and `aiServiceFactory.js` is the only non-test module that
-  requires the provider singletons. Step 8 (row h) can therefore reason about
-  `openaiService.initialize()` from the factory alone.
-- **Router tests:** `tests/helpers/mount-router.js` mounts the real `routes/setup.js` on a throwaway
-  database; `tests/test-manual-analyze-route.js` is the smallest example. On Windows its `close()`
-  fails with EPERM on the temp directory — expected there; the cases themselves pass.
 
 ## Out of Scope / Follow-ups
 
@@ -703,7 +728,7 @@ continues — human or agent.
   the caller is a small change whenever someone wants to make it.
 - **Vision/multimodal analysis**, which is presumably why the thumbnail is cached at all.
 - **`CachedNameIdEnum`'s semantic match calls a method no provider has.** `_findSemanticMatch()`
-  (`services/CachedNameIdEnum.js:176`) calls `ai_service.generateAnswer()`, which exists only in
+  (`services/CachedNameIdEnum.js`) calls `ai_service.generateAnswer()`, which exists only in
   test stubs. The call throws, is caught and returns "no match", so tag/correspondent/document type
   lookup silently falls back to its text matchers. Step 6 settled the contract — `generateText(prompt,
 options)` is the same on all four, the Anthropic provider included — so either point it there
@@ -719,9 +744,8 @@ options)` is the same on all four, the Anthropic provider included — so either
   composition root needs a Messages transport and a decision on those two differences.
 - **`services/llm/contracts.js`** with the typedefs of "The delegation contracts" was not written;
   the constructor JSDoc of `DocumentAnalyzer` and the transports document the shapes instead.
-- **This plan document's location.** `/CLAUDE.md` asks for records in the commit body rather than
-  Markdown files, and `services/` holds services. Move it to `docs/` or fold it into the PR
-  description before the refactor lands.
+- ~~**This plan document's location.**~~ Moved from `services/ManualServiceRefactoring.md` to
+  `docs/llm-service-refactoring.md` at the end of the refactoring; `services/` holds services.
 
 ## Risks
 
diff --git a/services/llm/tokenBudget/charBudget.js b/services/llm/tokenBudget/charBudget.js
index bcbad26c..edd441da 100644
--- a/services/llm/tokenBudget/charBudget.js
+++ b/services/llm/tokenBudget/charBudget.js
@@ -8,7 +8,7 @@
  * ~4 of English, and a ÷4 estimate under-sizes num_ctx for exactly those
  * documents. Everything here uses the same divisor, so the prompt estimate,
  * the context window and the document cut agree by construction (Finding 5
- * of services/ManualServiceRefactoring.md — do not unify it with tiktoken's).
+ * of docs/llm-service-refactoring.md — do not unify it with tiktoken's).
  *
  * Shares its reserve()/fit()/count()/cut() shape with tiktokenBudget.js.
  */
diff --git a/tests/test-llm-analysis-result-contract.js b/tests/test-llm-analysis-result-contract.js
index ca9d46ae..405feb22 100644
--- a/tests/test-llm-analysis-result-contract.js
+++ b/tests/test-llm-analysis-result-contract.js
@@ -1,7 +1,7 @@
 /**
  * The analysis result contract, pinned across all four providers.
  *
- * Safety net for the LLM service refactoring (services/ManualServiceRefactoring.md,
+ * Safety net for the LLM service refactoring (docs/llm-service-refactoring.md,
  * step 1). It describes what analyzeDocument() promises its three callers today —
  * server.js:766-821, services/mistralOcrService.js:988-997, and
  * routes/setup.js:3167-3177 — so that collapsing four implementations into one
diff --git a/tests/test-llm-azure-temperature-wiring.js b/tests/test-llm-azure-temperature-wiring.js
index 4bd591eb..bf8ac446 100644
--- a/tests/test-llm-azure-temperature-wiring.js
+++ b/tests/test-llm-azure-temperature-wiring.js
@@ -2,7 +2,7 @@
  * Azure carries the configured temperature on both its paths.
  *
  * Step 6 decision 7a of the LLM service refactoring
- * (services/ManualServiceRefactoring.md): analysis sends
+ * (docs/llm-service-refactoring.md): analysis sends
  * config.aiTemperatureAnalysis (was a hardcoded 0.3) and generateText()
  * defaults to config.aiTemperatureGeneration (was a hardcoded 0.7). The rest
  * of the request — model, messages, max_tokens, the absence of
diff --git a/tests/test-llm-document-analyzer.js b/tests/test-llm-document-analyzer.js
index 5eba84dd..d71fef8f 100644
--- a/tests/test-llm-document-analyzer.js
+++ b/tests/test-llm-document-analyzer.js
@@ -1,7 +1,7 @@
 /**
  * The DocumentAnalyzer on its own, every collaborator a fake.
  *
- * Step 6 of the LLM service refactoring (services/ManualServiceRefactoring.md):
+ * Step 6 of the LLM service refactoring (docs/llm-service-refactoring.md):
  * the four providers are composition roots over this one class, so the order
  * of its steps is the business logic of all four. The provider suites
  * (llm-analysis-result-contract, llm-provider-parity, ...) pin the outcome
diff --git a/tests/test-llm-ocr-fallback-phrases.js b/tests/test-llm-ocr-fallback-phrases.js
index 181bf4e8..fc8a1677 100644
--- a/tests/test-llm-ocr-fallback-phrases.js
+++ b/tests/test-llm-ocr-fallback-phrases.js
@@ -1,7 +1,7 @@
 /**
  * The OCR fallback phrases, pinned end to end (Finding 4).
  *
- * Safety net for the LLM service refactoring (services/ManualServiceRefactoring.md,
+ * Safety net for the LLM service refactoring (docs/llm-service-refactoring.md,
  * step 1). server.js:774-821 decides whether a failed analysis goes to the OCR
  * queue by matching analysis.error against a fixed list of **substrings** in
  * shouldQueueForOcrOnAiError(), and picks the queue reason with
diff --git a/tests/test-llm-prompt-builder.js b/tests/test-llm-prompt-builder.js
index 15547df2..d23f0b2c 100644
--- a/tests/test-llm-prompt-builder.js
+++ b/tests/test-llm-prompt-builder.js
@@ -1,7 +1,7 @@
 /**
  * The analysis prompt, built once for all four providers.
  *
- * Step 2 of the LLM service refactoring (services/ManualServiceRefactoring.md).
+ * Step 2 of the LLM service refactoring (docs/llm-service-refactoring.md).
  * The %CUSTOMFIELDS% block and the prompt cascade after it — pre-existing
  * data, mustHavePrompt, restriction placeholders, external API data,
  * USE_PROMPT_TAGS and the webhook's custom prompt — used to be copied into
diff --git a/tests/test-llm-provider-parity.js b/tests/test-llm-provider-parity.js
index e8ec9830..da6edbd8 100644
--- a/tests/test-llm-provider-parity.js
+++ b/tests/test-llm-provider-parity.js
@@ -2,7 +2,7 @@
  * The same answer gives the same result on all four providers, and the
  * four are one DocumentAnalyzer each behind a re-export.
  *
- * Step 6 of the LLM service refactoring (services/ManualServiceRefactoring.md,
+ * Step 6 of the LLM service refactoring (docs/llm-service-refactoring.md,
  * "Done means"): once the four providers share one DocumentAnalyzer, an
  * answer must come back byte-identical from each of them — the document, the
  * metrics, the truncation flag and, for a failure, the error and its code.
diff --git a/tests/test-llm-response-parser.js b/tests/test-llm-response-parser.js
index 7af13839..0c4ea180 100644
--- a/tests/test-llm-response-parser.js
+++ b/tests/test-llm-response-parser.js
@@ -1,7 +1,7 @@
 /**
  * The analysis answer, parsed once for all four providers.
  *
- * Step 3 of the LLM service refactoring (services/ManualServiceRefactoring.md).
+ * Step 3 of the LLM service refactoring (docs/llm-service-refactoring.md).
  * Finding 7d: there were three parsers. Custom matched braces and stripped
  * <think> blocks, openai and azure stripped fences and called JSON.parse on
  * the rest, Ollama took everything from the first "{" to the last "}" and
diff --git a/tests/test-llm-token-budget.js b/tests/test-llm-token-budget.js
index e6820bc0..ab9e5117 100644
--- a/tests/test-llm-token-budget.js
+++ b/tests/test-llm-token-budget.js
@@ -1,7 +1,7 @@
 /**
  * The two token budgets, each written once.
  *
- * Step 4 of the LLM service refactoring (services/ManualServiceRefactoring.md).
+ * Step 4 of the LLM service refactoring (docs/llm-service-refactoring.md).
  * openai, azure and custom carried the same reserve-then-truncate block:
  * tiktoken counts for the prompt, the configured answer reserved, the
  * document cut to what is left. It lives in
diff --git a/tests/test-llm-transport-request-mapping.js b/tests/test-llm-transport-request-mapping.js
index 8eba74d9..f5d9671c 100644
--- a/tests/test-llm-transport-request-mapping.js
+++ b/tests/test-llm-transport-request-mapping.js
@@ -1,7 +1,7 @@
 /**
  * The transports: what each API family is sent, and what comes back.
  *
- * Step 5 of the LLM service refactoring (services/ManualServiceRefactoring.md).
+ * Step 5 of the LLM service refactoring (docs/llm-service-refactoring.md).
  * A transport owns exactly one thing — how to talk to one API family — and
  * never shapes an analysis result:
  *
diff --git a/tests/test-manual-analyze-route.js b/tests/test-manual-analyze-route.js
index 8791c850..e41538c5 100644
--- a/tests/test-manual-analyze-route.js
+++ b/tests/test-manual-analyze-route.js
@@ -7,7 +7,7 @@
  * analysis answered 500 with a TypeError instead of the failure record), and
  * every branch passed `id || []` — a nonsense id that the thumbnail cache
  * rejects. Since every provider has the same analyzeDocument() signature and
- * result (services/ManualServiceRefactoring.md, step 6), the route goes
+ * result (docs/llm-service-refactoring.md, step 6), the route goes
  * through AIServiceFactory.getService() like the scan loop does.
  *
  * The router runs for real on a throwaway database (tests/helpers/mount-router);