diff --git a/.gitignore b/.gitignore index 655e1560..259d0aa6 100644 --- a/.gitignore +++ b/.gitignore @@ -32,9 +32,12 @@ dist/ .local/ *.local.* .vscode/ +.idea/ +.ollamassist/ # AI context files (auto-generated) COPILOT.md .DS_Store GEMINI.md CVE/* -.todo/* \ No newline at end of file +.todo/* +.ai/ diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..902d1820 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,12 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Jetbrains IDE +/.idea/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3c3329b7..a2608574 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,12 @@ COPY --chown=node:node schemas.js swagger.js ./ COPY --chown=node:node scripts ./scripts/ COPY docker-entrypoint.sh start-services.sh ./ -RUN chmod +x docker-entrypoint.sh start-services.sh +# An explicit mode rather than +x: the entrypoint re-executes itself after the +# privilege drop, so the unprivileged user has to be able to read it, too. A +# build context that hands the files over without read permission for others +# (e.g. a Samba share with a restrictive create mask) would otherwise produce +# 0711 and a restart loop with "cannot open ./docker-entrypoint.sh". +RUN chmod 755 docker-entrypoint.sh start-services.sh # Configure persistent data volume. # Docker seeds a fresh named volume from the image including ownership and mode, diff --git a/OPENAPI/openapi.json b/OPENAPI/openapi.json index 4205f5f7..85d854dc 100644 --- a/OPENAPI/openapi.json +++ b/OPENAPI/openapi.json @@ -325,6 +325,7 @@ }, "AnalysisResult": { "type": "object", + "description": "What analyzeDocument() returns on every AI provider. A failed analysis is\nreturned, not thrown: `document` is then `{ tags: [], correspondent: null }`,\n`metrics` is null and `error` says why.\n", "properties": { "document": { "type": "object", @@ -348,6 +349,7 @@ }, "correspondent": { "type": "string", + "nullable": true, "description": "Suggested correspondent name", "example": "ABC Corporation" }, @@ -389,6 +391,8 @@ }, "metrics": { "type": "object", + "nullable": true, + "description": "Token usage of the analysis; null when it failed", "properties": { "promptTokens": { "type": "integer", @@ -407,10 +411,20 @@ } } }, + "truncated": { + "type": "boolean", + "description": "Whether the document content was cut to fit the token limit", + "example": false + }, "error": { "type": "string", "description": "Error message if analysis failed", "example": null + }, + "errorCode": { + "type": "string", + "description": "Machine-readable reason of a failed analysis, when the provider gave one", + "example": null } } }, @@ -6715,7 +6729,7 @@ "/manual/analyze": { "post": { "summary": "Analyze document content manually", - "description": "Analyzes document content using the configured AI provider and returns structured metadata.\nThis endpoint processes the document text to extract relevant information such as tags,\ncorrespondent, and document type based on content analysis.\n\nThe analysis is performed using the AI provider configured in the application settings.\n", + "description": "Analyzes document content using the configured AI provider and returns structured metadata.\nThis endpoint processes the document text to extract relevant information such as tags,\ncorrespondent, and document type based on content analysis.\n\nThe analysis is performed by the AI provider configured in the application settings, the\nsame one the scan uses. The existing tags, correspondents and document types are read from\nPaperless-ngx. When the request carries an `id`, the token usage is recorded against it.\n", "tags": [ "Documents", "API" @@ -6742,7 +6756,7 @@ }, "existingTags": { "type": "array", - "description": "List of existing tags in the system to help with tag matching", + "description": "Accepted but not read; the existing tags are taken from Paperless-ngx", "items": { "type": "string" }, @@ -6754,7 +6768,7 @@ }, "id": { "type": "string", - "description": "Optional document ID for tracking metrics", + "description": "Optional Paperless-ngx document ID; the token usage is recorded against it", "example": "doc_123" } } @@ -6764,57 +6778,11 @@ }, "responses": { "200": { - "description": "Document analysis results", + "description": "The analysis result. A failed analysis is answered with status 200 as well, with\n`error` set and `metrics` null.\n", "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "correspondent": { - "type": "string", - "description": "Detected correspondent name", - "example": "Acme Corp" - }, - "title": { - "type": "string", - "description": "Suggested document title", - "example": "Acme Corp Invoice - August 2023" - }, - "tags": { - "type": "array", - "description": "Suggested tags for the document", - "items": { - "type": "string" - }, - "example": [ - "Invoice", - "Finance" - ] - }, - "documentType": { - "type": "string", - "description": "Detected document type", - "example": "Invoice" - }, - "metrics": { - "type": "object", - "description": "Token usage metrics (when using OpenAI)", - "properties": { - "promptTokens": { - "type": "number", - "example": 350 - }, - "completionTokens": { - "type": "number", - "example": 120 - }, - "totalTokens": { - "type": "number", - "example": 470 - } - } - } - } + "$ref": "#/components/schemas/AnalysisResult" } } } @@ -6830,7 +6798,7 @@ } }, "500": { - "description": "Server error or AI provider not configured", + "description": "Server error, e.g. Paperless-ngx could not be reached", "content": { "application/json": { "schema": { diff --git a/build.sh b/build.sh old mode 100755 new mode 100644 diff --git a/config/config.js b/config/config.js index 96c412f9..e2c74424 100644 --- a/config/config.js +++ b/config/config.js @@ -498,6 +498,11 @@ module.exports = { globalRateLimitMax: parseInt(process.env.GLOBAL_RATE_LIMIT_MAX || '1000', 10), predefinedMode: process.env.PROCESS_PREDEFINED_DOCUMENTS, ignoreTags: process.env.IGNORE_TAGS || '', + // Predefined-scan trigger tags (TAGS in predefined mode) are removed from a + // document after it has been processed, so re-adding the tag is what queues + // a re-run. Off by default: without it the tag stays, matching legacy + // behaviour. + removeTriggerTags: parseEnvBoolean(process.env.REMOVE_TRIGGER_TAGS, 'no'), // The context window the provider is told to respect. Read through the // strict parser: a NaN here used to travel all the way into // truncateToTokenLimit(), which then sent the model an empty document. diff --git a/docs/llm-service-refactoring.md b/docs/llm-service-refactoring.md new file mode 100644 index 00000000..a17f649f --- /dev/null +++ b/docs/llm-service-refactoring.md @@ -0,0 +1,758 @@ +# LLM Service Refactoring Plan + +**Status: done** (2026-10-15) — steps 1-8, and step 9 except the live smoke run. The four +provider services are composition roots of one `DocumentAnalyzer` (`services/llm/`), +`POST /manual/analyze` goes through `AIServiceFactory`, and `services/manualService.js` is gone. +What is left is listed under Out of Scope / Follow-ups. The plan is kept as the record of why the +code looks the way it does; the commit bodies hold the details of each step. + +Measured against the expected benefits: the four services shrank from 2783 lines to four one-line +re-exports and `manualService.js` (284 lines) is gone, while `services/llm/` holds 2636 lines. The +saving is smaller than the ~2000 lines expected: the remaining code carries JSDoc, and the checks +that now apply to every provider — the refusal check (7c), the metadata check (7k), the field +defaults — each existed in one service before. + +## The Goal + +Four provider services offer the same business service through different AI APIs. Implement the +business logic **once** and keep the API-specific parts in thin, injected adapters. + +In scope: `openaiService.js`, `azureService.js`, `customService.js`, `ollamaService.js`, and the +removal of `manualService.js` (see Finding 1 — it is dead code). + +Rules for this work: + +- Apply TDD. +- Adhere to the conventions in `/CLAUDE.md` (English only, singleton services, `{ success, data }` + API shape, OpenAPI kept in sync, commit body as the single source of truth). +- Prefer **delegation over inheritance**. No abstract base class, no `extends`. +- New tests must be symmetrical to the existing tests and registered in the test runner. + +### Expected Benefits + +- Single point of business logic (prompt building, token budgeting, parsing, result shaping). +- Easier maintenance and extension — a new provider is one adapter plus one composition root. +- Better test coverage: injected collaborators make the seams testable without `require.cache` tricks. +- Centralized, consistent error handling and result shaping. +- Removal of ~2000 lines of near-duplicate code across four services. + +--- + +## Validation Against the Source Code + +The following was verified against the working tree. Each finding constrains the design that follows. +Line numbers were re-checked after the merge of upstream v2026.09.03 (merge `9d337d2`). + +### 1. `manualService.js` is dead code + +A repo-wide search across all file types finds **no reference** to `services/manualService.js` +(no `require`, no route, no view, no test). Refactoring it in isolation changes nothing at runtime. + +The duplication that the "manual" business service actually suffers from lives in +`routes/setup.js`, which switches on `process.env.AI_PROVIDER` inline over the four _dedicated_ +singletons: + +- `routes/setup.js:7477-7543` — `POST /manual/analyze`, four-way switch (`:7495-7537`) calling + `analyzeDocument()`. + +The second switch, `POST /manual/playground` over `analyzePlayground()`, is gone: upstream removed +the Playground (#359), and the merge took that over, including the local `analyzePlayground()` in +`ollamaService.js`. + +Further evidence that `manualService.js` was abandoned rather than maintained: + +- `services/manualService.js:270` calls `this._parseResponse(...)`, a method that **does not exist** + on `ManualService` (it exists only on `OllamaService`). The Ollama path throws `TypeError` on any + non-object response. +- `existingTagsList` is computed and never used in all four methods + (`manualService.js:52`, `:110`, `:166`) — would fail `no-unused-vars` under `eslint.config.mjs`. +- `manualService.js:273-276` contains German user-facing strings + (`'Die Analyse hat zu lange gedauert...'`), violating the English-only rule in `/CLAUDE.md`. +- It is the only service exporting a **class** (`module.exports = ManualService;`) instead of a + singleton, contrary to the documented service-layer convention. +- Its Ollama path never sends the document content at all — `prompt` is just + `process.env.SYSTEM_PROMPT` (`manualService.js:218`). + +**Decision (carried out in step 7):** delete `services/manualService.js` as part of this work and point +`POST /manual/analyze` at `AIServiceFactory.getService()`. Do not port its behavior — it is worse +than the dedicated services on every axis: no token budgeting, no content truncation, no restriction +placeholders, no structured output, no retry, no metrics. + +### 2. Scale and shape of the real duplication + +| Service | Lines | Relationship | +| ------------------ | ----- | ----------------------------------------------- | +| `openaiService.js` | 550 | ~90 % identical to azure/custom | +| `azureService.js` | 568 | ~90 % identical to openai/custom | +| `customService.js` | 698 | ~90 % identical, plus the strongest JSON parser | +| `ollamaService.js` | 1231 | genuinely different transport and token model | + +The largest duplicated block is **prompt construction**: the +`CUSTOM_FIELDS` → `customFieldsTemplate` → `%CUSTOMFIELDS%` block appears **six times** +(`openaiService.js:125-160`, `azureService.js:143-178`, `customService.js:219-254`, +`ollamaService.js:128-160` in `analyzeDocument()`, `:348-383` in `_buildPrompt()` and `:499-534` in +`_generateCustomFieldsTemplate()`), followed each time by the same +`useExistingData` / `mustHavePrompt` / `RestrictionPromptService` / `USE_PROMPT_TAGS` / +`customPrompt` cascade. + +A seam drawn only at the API call would leave that block untouched. Prompt construction therefore +has to be a first-class collaborator of its own, not a side effect of the transport. + +### 3. The result contract that must be preserved + +Produced by all four services and consumed by three call sites: + +```js +{ document: { tags, correspondent, title, document_type, document_date, language, custom_fields }, + metrics: { promptTokens, completionTokens, totalTokens } | null, + truncated: boolean, + error?: string, + errorCode?: string } +``` + +Consumers: `server.js:854-890` (`processDocument()`), `services/mistralOcrService.js:1027-1052` +(`_runAiAnalysis()`), `routes/setup.js:3135-3165` (`processDocument()`) and `POST /manual/analyze` +(`routes/setup.js:7495-7537`), which returns the result as JSON. The first three hand `document` to +`documentProcessingService.buildUpdateData()`, which since the merge is the single write-back for +scan, queue and OCR. Any narrowing of `document` drops title, document type, date, language or +custom fields silently — `buildUpdateData()` simply stops setting those fields. + +### 4. The error contract is load-bearing and phrase-matched — it must be preserved verbatim + +`analyzeDocument()` **never throws**; it returns the failure in `error`/`errorCode`. +`server.js:870-890` (and `routes/setup.js:3160`) then routes the document to the OCR queue based on +`shouldQueueForOcrOnAiError()` / `classifyOcrQueueReasonFromAiError()` +(`services/serviceUtils.js:847-900`), which match on **exact phrases**: + +- `'insufficient content for ai analysis'` +- `'invalid response structure'` +- `'could not determine assignable metadata'` +- `'invalid json response from api'` +- `'invalid api response structure'` + +Rewording any of these silently disables OCR fallback. Guarded by the `ocr-fallback-ai-errors`, +`response-truncation-detection` and `llm-ocr-fallback-phrases` tests. The +`error.code = 'ai_response_truncated'` marker (`serviceUtils.js:834`, `ollamaService.js:791`) has the +same status. + +This already happened once: `38a5352` replaced Custom's "Invalid API response structure" with +detailed messages ("API response is missing choices."), and `llm-ocr-fallback-phrases` went red. +`355ac29` keeps the detail behind the phrase ("Invalid API response structure: missing choices."). + +Consequence for the design: `analyzeDocument()` keeps its non-throwing contract. A version that +throws on, say, a missing `SYSTEM_PROMPT` would take down the scan loop instead of queueing the +document. + +### 5. The Ollama context arithmetic must be preserved exactly + +Two separate functions, easily conflated: + +```js +// ollamaService.js:568-570 — conservative 2 chars/token, deliberate and documented +_calculatePromptTokenCount(prompt) { + return Math.ceil(prompt.length / 2); +} + +// ollamaService.js:581-588 — tokens, with the answer reserved +_calculateNumCtx(promptTokenCount, expectedResponseTokens) { + return Math.min(promptTokenCount + expectedResponseTokens, Number(config.tokenLimit)); +} +``` + +The `/2` divisor exists because non-English tokenization (CJK, German) produces roughly two +characters per token instead of ~4; `manualService.js:234` used `/4` and would truncate those +documents. Centralizing the estimators must **preserve `/2` for Ollama**, not unify the divisors. + +`_fitContentToContext()` (`ollamaService.js:290-320`) uses the same `/2` estimate by construction, so +the two agree. Whatever holds this arithmetic after the refactor has to keep that property. + +### 6. The transport seam needs a wide signature + +Ollama's transport needs `(prompt, systemPrompt, numCtx, schema)` (`ollamaService.js:678`), sends +`format: schema` for structured output, computes `num_ctx`/`num_predict`, carries a bounded retry for +transient failures (`:643-735`, `_wrapOllamaRequestError()` at `:664`), and inspects `done_reason` +for truncation (`:739-795`). Its response may arrive as a **JS object** rather than a string +(`_processOllamaResponse()`, `:803`). + +A single-string `callAPI(prompt)` cannot carry any of that, which is why `CompletionRequest` below is +a record rather than a string. + +`generateText(prompt, options)` widened the seam further with the merge. The Duplicates and +"Simplify tags" features (`entityMatchAiService.js`, `tagSimplifyService.js`) call it with options +that `services/aiGenerateOptions.js` reads the same way for all four providers: `systemPrompt`, +`temperature` (where `0` is a value), a completion cap, a model override, an abort signal, +reasoning on/off and a progress handler that streams partial text (`ai-provider-streaming` test). +Upstream already unified the option parsing; what is still per provider is how each option reaches +the API. The transport contract has to carry all of it, or the generation path regresses for those +features. + +### 7. behavioral drift between the four services — each item needs an explicit decision + +The refactor collapses four implementations into one, which forces a choice wherever they differ +today. These are the actual bugs the work will surface: + +| # | Drift | Location | Proposed resolution | +| --- | -------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| a | ~~Azure hardcodes `temperature: 0.3` while openai/custom use `config.aiTemperatureAnalysis`~~ **resolved** | was `azureService.js:286` | Done in step 6: analysis follows `config.aiTemperatureAnalysis`, `generateText()` defaults to `config.aiTemperatureGeneration`; pinned by `llm-azure-temperature-wiring`. | +| b | ~~`return;` (undefined!) when a thumbnail is missing — callers then read `.document` of `undefined`~~ **resolved** | was `openaiService.js:87`, `azureService.js:76`, `customService.js:171` | Fixed ahead of the refactor together with row j — same statement block, same root cause. See the note below the table. | +| c | ~~"Insufficient content" early return exists only in OpenAI~~ **resolved** | was `openaiService.js:310-332` | Done in step 6: `isInsufficientContentAnswer()` in `responseParser.js`, applied by the `DocumentAnalyzer` after a failed parse, so all four report `ai_insufficient_content`. | +| d | ~~Three different JSON parsers~~ **resolved** | `customService.js:35-126` (brace-matched, strongest) vs. `openaiService.js:299-334` / `azureService.js:286-298` (naive strip) vs. `ollamaService.js:977-1053` (regex + sanitize) | Done in step 3: `services/llm/responseParser.js` — custom's brace matcher, `` stripping, Ollama's repair and object passthrough. | +| e | Only custom normalizes timeouts | `customService.js:428-436` | Apply `isTimeoutError`/`buildTimeoutErrorMessage` for all providers. | +| f | `generateText` option handling — **mostly resolved upstream** | `services/aiGenerateOptions.js`, used by all four `generateText()` implementations | Upstream unified how options are read (completion cap, temperature, system prompt, model, abort, reasoning, progress). Keep `aiGenerateOptions.js` as the shared reader; the transports only map the result onto their own knobs. | +| g | `checkStatus` differs by nature | openai/custom `models.list()`; Azure REST GET; `ollamaService.js:1210` `/api/ps` | Not business logic. Each composition root hands its probe to the analyzer unchanged (step 6); nothing calls it, so step 8 decides whether it stays. | +| h | Unreachable branches in `openaiService.initialize()` for providers `ollama` and `custom` | `openaiService.js:46-60` | `aiServiceFactory` never routes those providers here. Since step 7 the factory is the only module that requires `openaiService` outside the tests. Verify, then drop. | +| i | Thumbnail caching sits inside `analyzeDocument` although the thumbnail never enters the prompt | all four | Still out of scope as a _placement_ question, but the four copies are now one call to `services/thumbnailCache.js`. Moving it out of `analyzeDocument` is a one-line change once a caller wants it. | +| j | ~~`getThumbnailCachePath(id)` runs **outside** the try block, so an invalid id makes `analyzeDocument` throw~~ **resolved** | was `openaiService.js:63`, `azureService.js:52`, `customService.js:147` | Fixed ahead of the refactor. See the note below the table. | +| k | ~~An answer with neither tags nor a correspondent is a **failure** on three providers and a **success** on Ollama~~ **resolved** | was `ollamaService.js:223` | Done in step 6: the analyzer runs `assertAssignableMetadata()` on the parsed answer, before `withDocumentDefaults()`, for all four. | +| l | An empty provider payload is worded "No response data from Ollama API", which matches **no** OCR marker | `ollamaService.js:821` vs. `openaiService.js`, `azureService.js`, `customService.js` | The other three say "Invalid API response structure", which `shouldQueueForOcrOnAiError()` matches. Unify the wording on the shared phrase, or add this one to the marker list. | +| m | Custom classifies `socket hang up` as a timeout and replaces the message | `customService.js:331` | Consequence of row e: `isTimeoutError()` swallows the original text into "AI response timeout reached…". Decide deliberately whether a dropped connection is a timeout when the handling is unified. | + +Rows k and l were found by writing the step-1 safety net, not by reading the code — both are +silent: the analysis "succeeds", the log looks plausible, and the document simply never reaches +OCR. Both are pinned as-is in the new tests and marked as known drift, so the case starts failing +the moment the refactor unifies the path. That is deliberate: it forces whoever unifies it to +update the expectation rather than to not notice. Row k was unified in step 6 and its case flipped; +row l is still pinned as drift. + +**Rows b and j were fixed before the refactor started.** They shared one root cause — a best-effort +side effect written as a blocking precondition — and sat within three lines of each other, so +fixing one alone would have left the other a line away. Writing the test first showed the drift was +also wider than row j claimed: Ollama's `!id` guard misses truthy-but-unusable ids such as `[]` and +`'///'`, and its `_handleThumbnailCaching` swallowed a failing fetch but still lost the analysis, so +all four providers were affected, not three. + +The fix consolidates the four copies into `services/thumbnailCache.js`, whose `cacheThumbnail(id)` +is best-effort by construction: it never throws and never short-circuits its caller, it returns a +boolean, and an unusable id, a missing thumbnail or an unreachable Paperless each produce a warning +and `false`. Ollama's `_handleThumbnailCaching()` is gone. `tests/test-thumbnail-optional-analysis.js` +(32 cases, registered under `processing`) pins the contract for all four providers: whatever the +thumbnail does, `analyzeDocument` returns a full analysis. + +### 8. Moving tests into `tests/services/` would break the CI drift guard + +`findUnregisteredTests()` (`scripts/run-tests.js:233-249`) reads `tests/` **non-recursively** and +matches `/^test-.*\.js$/`. Any test moved into `tests/services/` silently leaves the guard, which is +exactly the failure mode that guard exists to prevent. Registration itself would still work +(`path.join(__dirname, '..', 'tests', 'services/test-x.js')` is portable), but the discovery walk +must be made recursive in the same change. + +Additionally, moving every test that covers production code in `services/` would relocate a large set +of files unrelated to this refactor (`restriction-service`, `updated-service`, +`reconciliation-service`, the `ollama-*` and `ocr-*` families, …). That is churn that hides the +refactor in the diff. **Deferred to a separate PR** (see Out of Scope). + +--- + +## Target Architecture + +### Design principle: composition, not inheritance + +One **concrete** business-logic class holds injected collaborators. There is no base class and +nothing `extends` anything. A provider is a _composition root_: it wires collaborators together and +exports the singleton facade. + +``` +services/llm/ + documentAnalyzer.js the single business-logic implementation (concrete, never subclassed) + promptBuilder.js systemPrompt + user content assembly (replaces 6 duplicates) + customFieldsTemplate.js the %CUSTOMFIELDS% block + responseParser.js raw text|object -> document object + analysisResult.js buildSuccess() / buildFailure() — the one place the result shape lives + tokenBudget/ + tiktokenBudget.js openai/azure/custom: calculateTotalPromptTokens + truncateToTokenLimit + charBudget.js ollama: /2 estimate + _fitContentToContext + transports/ + chatCompletionsTransport.js OpenAI SDK, AzureOpenAI SDK, custom base URL — messages[] + ollamaGenerateTransport.js axios /api/generate, format schema, num_ctx/num_predict, retry + providers/ + openaiProvider.js composition root + singleton facade + azureProvider.js + customProvider.js + ollamaProvider.js +``` + +The existing module paths stay as one-line re-exports +(`services/openaiService.js` → `module.exports = require('./llm/providers/openaiProvider');`) so that +`aiServiceFactory.js`, `routes/setup.js`, `server.js` and `mistralOcrService.js` need no change and +the diff stays reviewable. The four re-export shims can be collapsed in a later cleanup. + +### The delegation contracts + +Written as JSDoc typedefs in `services/llm/contracts.js`; enforced by tests, not by a base class. + +```js +/** + * @typedef {Object} CompletionRequest + * @property {string} systemPrompt + * @property {string} userContent + * @property {string} model + * @property {number} temperature + * @property {number} maxResponseTokens + * @property {Object|null} schema structured-output schema; null for chat transports + * @property {'analysis'|'generation'} purpose + */ + +/** + * @typedef {Object} CompletionResult + * @property {string|Object} payload raw assistant text, or the object Ollama may return + * @property {{promptTokens:number, completionTokens:number, totalTokens:number}|null} metrics + */ + +/** + * A transport owns exactly one thing: how to talk to one API family. + * It throws on transport failure and on provider-reported truncation + * (error.code === 'ai_response_truncated'); it never shapes an AnalysisResult. + * + * @typedef {Object} LlmTransport + * @property {string} label 'OpenAI' | 'AzureOpenAI' | 'Custom OpenAI' | 'Ollama' + * @property {() => void} initialize lazy client construction, idempotent + * @property {() => boolean} isReady + * @property {(req: CompletionRequest) => Promise} complete + * @property {() => Promise<{status:string, model?:string, error?:string}>} checkStatus + */ + +/** + * @typedef {Object} TokenBudget + * @property {(systemPrompt:string, extraPrompts?:string[], model?:string) => Promise<{totalPromptTokens:number, reservedTokens:number, availableTokens:number}>} reserve + * @property {(content:string, availableTokens:number, model?:string) => Promise} fit + */ +``` + +`maxResponseTokens` is what each transport maps to its own knob: `max_tokens` for the chat +transports, `num_predict` plus a `num_ctx` sized by `min(promptTokens + maxResponseTokens, tokenLimit)` +for Ollama. That mapping is the only place the Ollama context arithmetic lives, and it keeps +Finding 5's formula intact. + +### DocumentAnalyzer — the single business logic + +```js +class DocumentAnalyzer { + /** + * @param {Object} deps + * @param {LlmTransport} deps.transport + * @param {TokenBudget} deps.tokenBudget + * @param {Object} deps.promptBuilder + * @param {Object} deps.responseParser + * @param {() => string} deps.resolveModel provider-specific model lookup + * @param {Object} [deps.thumbnailCache] injected for testability + */ + constructor({ transport, tokenBudget, promptBuilder, responseParser, resolveModel, thumbnailCache }) { ... } + + async analyzeDocument(content, existingTags = [], existingCorrespondentList = [], + existingDocumentTypesList = [], id, customPrompt = null, options = {}) { ... } + + async generateText(prompt, options = {}) { ... } // options as read by aiGenerateOptions.js + checkStatus() { return this.transport.checkStatus(); } +} +``` + +Every collaborator arrives through the constructor. That is what makes the tests cheap: a fake +transport is an object literal with a `complete()`, so no `require.cache` surgery and no network. + +### Provider composition root (example) + +```js +// services/llm/providers/azureProvider.js +const DocumentAnalyzer = require('../documentAnalyzer'); +const ChatCompletionsTransport = require('../transports/chatCompletionsTransport'); +const tiktokenBudget = require('../tokenBudget/tiktokenBudget'); +const promptBuilder = require('../promptBuilder'); +const responseParser = require('../responseParser'); +const config = require('../../../config/config'); + +module.exports = new DocumentAnalyzer({ + transport: new ChatCompletionsTransport({ + label: 'AzureOpenAI', + createClient: () => + new (require('openai').AzureOpenAI)({ + apiKey: config.azure.apiKey, + endpoint: config.azure.endpoint, + deploymentName: config.azure.deploymentName, + apiVersion: config.azure.apiVersion, + }), + checkStatus: azureDeploymentProbe, // the REST GET from azureService.js:574 + }), + tokenBudget: tiktokenBudget, + promptBuilder, + responseParser, + resolveModel: () => process.env.AZURE_DEPLOYMENT_NAME, +}); +``` + +Adding a fifth provider is: one `createClient`, one `resolveModel`, one `checkStatus` probe. No new +business logic, no subclass. + +### Why not the base-class variant + +An abstract base class with a `callAPI()` override per provider forces Ollama's four extra +parameters, its object-shaped response, its retry loop and its char-based token model either into the +base class (where they are dead weight for three providers) or into an override that reimplements the +template method (which is the duplication we started with). Injecting a transport keeps Ollama's +peculiarities inside `ollamaGenerateTransport.js` and out of everyone else's way. + +--- + +## Test Strategy + +House style: plain node scripts under `tests/`, `require('assert')`, one `main()`, `[PASS]`/`[FAIL]` +and `process.exitCode = 1`. No test framework is introduced — the runner executes each file with +`spawnSync(process.execPath, [filePath])` (`scripts/run-tests.js:471`), so framework globals are not +available. Model for service tests: `tests/test-ollama-temperature-wiring.js`, including its +`require.cache` injection helper (`:26-39`). + +The seam under test is the real one: +`analyzeDocument(content, existingTags, existingCorrespondentList, existingDocumentTypesList, id, customPrompt, options)`. + +Because collaborators are injected, most tests need no module mocking at all — construct a +`DocumentAnalyzer` with a fake transport: + +```js +const fakeTransport = { + label: 'Fake', + initialize() {}, + isReady: () => true, + calls: [], + async complete(request) { + this.calls.push(request); + return { + payload: JSON.stringify({ + tags: ['tag1'], + correspondent: 'John Doe', + title: 't', + document_type: 'Invoice', + document_date: '2026-01-01', + language: 'en', + }), + metrics: { promptTokens: 10, completionTokens: 5, totalTokens: 15 }, + }; + }, + async checkStatus() { + return { status: 'ok' }; + }, +}; +``` + +Planned tests (each registered in `TESTS` in `scripts/run-tests.js` and in an `AREAS` entry): + +| Test name | Asserts | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `llm-response-parser` | code fences stripped; `…` stripped; brace-matched extraction from surrounding prose; trailing-comma/unquoted-key sanitize fallback; unparseable input throws with the exact phrase `Invalid JSON response from API` | +| `llm-analysis-result-contract` | success returns `{document, metrics, truncated}`; failure returns `{document:{tags:[],correspondent:null}, metrics:null, error, errorCode}` and **never throws** | +| `llm-ocr-fallback-phrases` | every phrase in `shouldQueueForOcrOnAiError` is still produced by the refactored paths — the regression guard for Finding 4 | +| `llm-prompt-builder` | `%CUSTOMFIELDS%` rendering for date/boolean/other; `useExistingData` branch; `USE_PROMPT_TAGS` override; `customPrompt` override; restriction placeholders delegated to `RestrictionPromptService`; external API data appended | +| `llm-token-budget` | `tiktokenBudget.reserve` throws `Token limit exceeded: …` when `availableTokens <= 0`; `charBudget` uses the `/2` divisor and `min(promptTokens + responseTokens, tokenLimit)` for `num_ctx` — Finding 5 | +| `llm-transport-request-mapping` | chat transports send `messages[]` with system+user and omit `temperature` for `o3-mini`; Ollama transport sends `format`, `num_ctx`, `num_predict`, `think:false` unless `OLLAMA_THINK=true` | +| `llm-azure-temperature-wiring` | Azure analysis requests carry `config.aiTemperatureAnalysis`, not a hardcoded `0.3` — Finding 7a | +| ~~`llm-missing-thumbnail-continues`~~ | **already delivered** as `tests/test-thumbnail-optional-analysis.js` (Findings 7b/7j). Keep it green through the refactor instead of writing a new one; it exercises `analyzeDocument` end to end, so the composition roots must keep calling `cacheThumbnail(id)`. | +| `llm-provider-parity` | all four composition roots expose `analyzeDocument`/`generateText`/`checkStatus` and, given the same fake transport payload, produce byte-identical `document` objects | + +Existing tests that must stay green without modification (they are the real acceptance criteria): +`ocr-fallback-ai-errors`, `response-truncation-detection`, `ollama-response-limit`, +`ollama-upstream-error`, `ollama-temperature-wiring`, `ollama-token-metrics`, +`prompt-existing-data-serialization`, `restriction-service`, `restricted-document-types-placeholder`, +`document-type-restriction`, `ai-temperature-config`, `ai-provider-streaming`. +(`playground-deprecation` was retired upstream together with the Playground, #359.) + +Repo hygiene for every step: + +- Register each new test in `TESTS`; run `node scripts/run-tests.js --all` — the registry drift check + only runs on `--all`. +- `npx eslint `, `npx prettier --check ` — CI lints the files touched by the PR. + +--- + +## Implementation Steps + +Each step ends with a green `node scripts/run-tests.js --all`. Steps 1-2 are pure preparation and +carry no behavioral risk; the risk starts at step 4. + +1. ~~**Freeze the contract in tests first (TDD).**~~ **Done.** `llm-analysis-result-contract` + (32 cases) and `llm-ocr-fallback-phrases` (15 cases) cover all four providers and are green + against the **current** services — which is what makes them a regression net rather than a + description of the new code. Both are mutation-verified: rewording an OCR marker in + `serviceUtils.js` fails five cases, dropping `errorCode: error.code` from `openaiService.js` + fails one. They stub only the provider client and pre-seed the thumbnail cache, so no network + and no `require.cache` tricks. They also produced Findings 7k, 7l and 7m. + + Two things they deliberately do **not** pin: the wording of a failure message (Custom and Ollama + both rewrite it — rows e/m and the retry wrapper), and the phrases the OCR matcher owns, which + are copied into the test rather than imported. Importing that list would make the test agree + with whatever the source says, which is the opposite of pinning it. + +2. ~~**Extract `customFieldsTemplate.js` and `promptBuilder.js`**~~ **Done.** + `services/llm/customFieldsTemplate.js` (`buildCustomFieldsBlock()`) and + `services/llm/promptBuilder.js` (`buildAnalysisPrompt()` → `{ systemPrompt, promptTags }`) replace + the six copies. `llm-prompt-builder` (27 cases) pins the cascade and checks that every provider + sends what the builder built. `ollamaService._buildPrompt()` stays as a thin wrapper, because + `prompt-existing-data-serialization` and `restricted-document-types-placeholder` call it. + + Verified by recording the requests of all four providers for eight prompt scenarios before and + after: the 24 requests of openai/azure/custom are byte-identical. Ollama differs in two + deliberate ways. Its two fixed text blocks are now indented with 8 spaces instead of 12, like + the other three; the wording is unchanged. And it sends the external API data instead of + `[object Promise]`: its `_validateAndTruncateExternalApiData()` was `async` but never awaited + by either caller. It awaited nothing itself, so it is synchronous now. + + Not changed on purpose: a `CUSTOM_FIELDS` that parses but has no `custom_fields` array still + throws inside `buildCustomFieldsBlock()` (and so fails the analysis), as all six copies did. + Ollama still ignores `PROMPT_TAGS` as user content; the chat providers only use it for the + token count. + +3. ~~**Extract `responseParser.js`**~~ **Done.** `services/llm/responseParser.js` + (`parseAnalysisResponse()` → `{ parsed, normalized }`, plus `stripResponseWrapping()`) replaces + custom's `_extractFirstJsonValue`/`_parseJsonResponse`, the inline strip-and-parse of openai and + azure, and Ollama's `_parseResponse`/`_sanitizeJsonString`. Order: an object passes through, + `` blocks and fences (any case) are stripped, the text is parsed as it stands, then each + **top-level** `{...}` is tried as written and repaired. An unreadable object is skipped whole, so + a nested `custom_fields` entry is never mistaken for the answer — on Ollama that would have been + a silent success (row k). Every failure starts with "Invalid JSON response from API". + `llm-response-parser` (57 cases) sends seven readable and three unreadable answers through the + parser and through all four providers. + + Against the services before this step 12 provider cases failed, which is Finding 7d measured: + OpenAI and Azure could not read an upper-case fence, prose around the object, a trailing comma + or prose plus braces in a string; Custom could not read a trailing comma; Ollama could not read + a `` block or prose after the object, and its repair branch **dropped `document_type` + and `custom_fields`** from an answer it did repair. All of those now read the same everywhere. + + Deliberately not moved: the field defaults Ollama applies (`tags` → `[]`, the rest → `null`) + stay in `_processOllamaResponse()`, now for both the object and the text path. They are result + shaping and belong to `analysisResult.js` in step 4, so the parser returns the object as read + rather than the "document object" the target architecture lists. OpenAI's "insufficient + content" early return still runs only on a failed parse, on the stripped text. Row l ("No + response data from Ollama API") is untouched and still pinned as drift. + +4. ~~**Extract `analysisResult.js`** and the two `tokenBudget` strategies~~ **Done.** + `services/llm/tokenBudget/tiktokenBudget.js` replaces the reserve-then-truncate block that was + byte-identical in openai, azure and custom. `services/llm/tokenBudget/charBudget.js` holds + Ollama's arithmetic (Finding 5): `estimateTokens()` with the `/2` divisor, `numCtx()` as + `min(prompt + answer, TOKEN_LIMIT)`, and `reserve()`/`fit()` in place of + `_fitContentToContext()`, which is gone. Both budgets expose `reserve()` → `{ totalPromptTokens, +reservedTokens, availableTokens }` and `fit(content, availableTokens)`; the char budget's + `reserve()` takes the prompt prefix alone, since it has no model and no extra messages to count. + `_calculatePromptTokenCount()`/`_calculateNumCtx()` stay as one-line wrappers because + `ollama-temperature-wiring` stubs them; `generateText()` reaches the char budget through them. + + `services/llm/analysisResult.js` is the one place the result shape lives: `buildSuccess()`, + `buildFailure(error, message?)` (Custom passes its normalized timeout text), + `buildInsufficientContent()` (OpenAI only, row c), `assertAssignableMetadata()` (the three chat + providers; Ollama's row k is untouched and still pinned) and `withDocumentDefaults()` (Ollama's + field defaults; whether the chat providers adopt them is the parity decision of step 6). + + `llm-token-budget` (23 cases) pins both budgets and that every provider spends the one it was + given. Its provider half was green against the services before this step, which is the + point of a pure extraction. `llm-analysis-result-contract` gained a case for OpenAI's + insufficient-content placeholder, which was pinned nowhere before. Mutation-checked: + divisor `/4`, a dropped answer reservation, a provider that fits to a larger budget, a dropped + `errorCode`, a dropped placeholder field and a dropped `custom_fields` default each fail a case. + + One log fixed on the way: Ollama's "exceeds the context budget" debug line was written in single + quotes and printed `${text.length}` literally. + +5. **Introduce the transports.** `chatCompletionsTransport.js` covering openai/azure/custom, and + `ollamaGenerateTransport.js` carrying the retry, `format`, `num_ctx`/`num_predict` and + `done_reason` truncation check moved verbatim from `ollamaService.js`. Cover with + `llm-transport-request-mapping`. Landed in two commits, because `generateText()` brings + streaming, abort, three reasoning dialects and per-provider defaults the analysis call has + none of: + - ~~**5a — the analysis request.**~~ **Done.** Each service owns a `transport`; `client` + (and Ollama's `apiUrl`/`model`) are accessors onto it, so the many tests that replace + `service.client` keep working unchanged. The chat transport takes a `createClient()` + factory — OpenAI's still carries the unreachable branches of row h, removed in step 8 — and + an optional `describeMissingContent()`, which is how Custom keeps its detailed + "Invalid API response structure: missing choices[0]." (355ac29). Per-provider temperature + stays with the provider: OpenAI passes `undefined` for o3-mini, which the transport omits; + Azure still passes 0.3 and is pinned as such until step 6 (row a). The Ollama transport gets + the axios client handed in, because `ollama-temperature-wiring` mocks axios for the service + module; `_callOllamaAPI()`, `_assertNotTruncated()`, `_extractOllamaMetrics()` and + `_buildRequestHeaders()` are one-line delegations, the first and third because tests call + them. `num_ctx` is still computed by the service and handed in; moving it behind the + transport is step 6. `llm-transport-request-mapping` (31 cases) pins both transports with + fake clients and every provider's analysis request; six mutations each fail a case. + - ~~**5b — `generateText()`**~~ **Done.** The chat transport's `generate()` holds what the + three `generateText()` copies shared: abort and progress handling, the streamed and the plain + call, `lastGenerateTextUsage`, reasoning stripping, the cut-off check with `partialText`, and + the abort normalization. What differs stays with the provider, as one + `_buildGenerationRequest()` each — moved verbatim: OpenAI caps `max_tokens` only on request; + Azure always caps at RESPONSE_TOKENS and defaults to temperature 0.7; Custom clamps to the + context window and speaks three reasoning dialects (Qwen's `/no_think`, + `chat_template_kwargs`, `reasoning_effort`). Hooks: `streamUsage()` (Azure's API-version + gate for `stream_options`) and `describeEmptyText()` (Custom's "the answer carried no + text …"). Ollama's `generate()` and its NDJSON stream reader moved into + `ollamaGenerateTransport.js` verbatim; the service still sizes `num_predict`/`num_ctx` + through `_calculatePromptTokenCount()`/`_calculateNumCtx()` and hands them in. + `lastGenerateTextUsage` is an accessor onto the transport on all four services. + `llm-transport-request-mapping` grew to 48 cases. Of eight mutations on the generation path, + three survived the existing suite — Custom's empty-answer message, Ollama's `reasoning` + override of OLLAMA_THINK, and Ollama's abort signal — and are pinned now. + + **Not moved: `checkStatus()`** (row g). Nothing in the repository calls it — no route, no + view, no service, no test. It stays in the four services untouched; step 6 decides whether the + composition roots expose it at all or whether it is dropped with the dead branches of step 8. + +6. ~~**Introduce `documentAnalyzer.js`** and the four composition roots~~ **Done.** Built on upstream + v2026.10.01 (merged first, `8dd6121`), in this order, every commit green and tagged + `refactor-step-6-checkpoint--…`: + - **The decisions first, on the four separate services**, each with a test that failed before + and passes after (decided by the maintainer on 2026-10-11): + - **7a** — Azure analysis sends `config.aiTemperatureAnalysis` (was 0.3), `generateText()` + defaults to `config.aiTemperatureGeneration` (was 0.7); `llm-azure-temperature-wiring`. + - **7k** — an answer without assignable metadata is a failure on all four, checked on the + parsed answer before `withDocumentDefaults()`; `{ tags: [], correspondent: null }` still + passes. Ollama now reaches the OCR queue for it. + - **Field defaults** — every success is shaped by `withDocumentDefaults()`: the seven fields, + unknown ones dropped. Safe for the write-back (`buildUpdateData()` falls back to the + document's own title/date and skips nulls). + - **7c** — `isInsufficientContentAnswer()` (`responseParser.js`): a refusal that does not parse + returns `buildInsufficientContent()` on all four (OCR reason `ai_insufficient_content`). + `llm-response-parser` checks that its unreadable answers contain none of the phrases. + - **Then the parity pinned**: `llm-provider-parity` sends twelve answers to all four providers + and compares the JSON of the results byte for byte. It passed before the structural commits + and had to stay green through them; it also checks that each service is a `DocumentAnalyzer` + behind a one-line re-export. + - **Then the structure, in three commits** (OpenAI + Azure, Custom, Ollama). + `services/llm/documentAnalyzer.js` holds `analyzeDocument()`; `generateText()` and + `checkStatus()` delegate. The constructor differs from the sketch under Target Architecture + where the code demanded it: a `promptLayout` (`services/llm/promptLayout.js`) says how + instructions and document become one request — the chat providers send the instructions as + the system message, Ollama puts them in the prompt and sends a system prompt of its own; the + token budgets gained `count()`/`cut()` for the external API data; the optional + `analysisTemperature` (OpenAI leaves it out for o3-mini), `responseSchema`, `contextWindow` + (Ollama's `num_ctx`, sized through the singleton's `_calculate*()` seams on both paths), + `prepareContent` (Ollama's `CONTENT_MAX_LENGTH`) and `describeFailure` (Custom's timeout + wording, rows e/m) carry what is a provider's own. `checkStatus` is the provider's old probe, + moved unchanged (row g). The four `services/*Service.js` are one-line re-exports. + A request probe compared the outgoing request bodies and results of all four providers with + the state before the structural commits in eight scenarios (plain, custom prompt, external + API data, `USE_PROMPT_TAGS`, `USE_EXISTING_DATA`, `CONTENT_MAX_LENGTH`, cut to `TOKEN_LIMIT`, + refusal): byte-identical. What changed is logging only: Custom writes `prompt.txt` like the + other chat providers, Ollama writes `prompt.txt` before the request and `response.txt` after + it, and a failed analysis names its provider. + - **Kept as is, as decided:** row e/m (timeout normalization only on Custom), row l ("No + response data from Ollama API", now raised by the Ollama transport, still matches no OCR + marker), Ollama's `CONTENT_MAX_LENGTH` cut. + - `tests/test-llm-document-analyzer.js` pins the analyzer with fake collaborators. The + `refactor-step-6` branch (PR #1) was used where it held up: its 7a/7k/defaults/7c test changes + and the refusal helper. Its analyzer covered OpenAI only and its parity test compared method + signatures, so neither was taken over. + +7. ~~**Delete `services/manualService.js`** and repoint `POST /manual/analyze` at + `AIServiceFactory.getService()`~~ **Done**, in two commits: + - **The route.** The five-branch switch on `process.env.AI_PROVIDER` is one call to + `AIServiceFactory.getService().analyzeDocument()` with the `id` as sent (no `|| []`). As decided + by the maintainer on 2026-10-15, `documentModel.addOpenAIMetrics()` runs for every provider + whenever the analysis returned `metrics` and the request carried an `id` — the Anthropic + branch's rule — and the API is unchanged: `id` stays optional, no new field, no new status. + That guard also removes the OpenAI branch's unguarded `metrics.promptTokens`, so a failed + analysis answers 200 with its failure record instead of a 500 TypeError. The four provider + imports of `routes/setup.js` were unused afterwards and are gone. + + One consequence that was not a decision of its own: the provider is now picked as the scan + loop picks it (`config.aiProvider`, OpenAI when unset or unknown), so the old 500 "AI provider + not configured" no longer exists. The swagger block was corrected to what the route returns: + the 200 references `AnalysisResult` (`schemas.js`, previously unreferenced; it gained + `truncated` and `errorCode`, and `metrics`/`correspondent` are nullable), `existingTags` is + documented as accepted but not read — the route takes all three lists from Paperless-ngx — + and the 500 text no longer names the removed branch. `OPENAPI/openapi.json` regenerated. + + `manual-analyze-route` (9 cases) mounts the real router (`tests/helpers/mount-router`) and + replaces only each provider's `analyzeDocument()` and the metrics store. Four cases failed + before the change; restoring `id || []`, dropping the metrics guard or the id guard each + fails one. On Windows the test ends red after its cases pass, because the harness cannot + remove its temp directory (EPERM), like `duplicates-endpoints`. + + - **The dead file.** `services/manualService.js` (284 lines) is deleted; `git grep manualService` + finds only this plan. + +8. ~~**Verify Finding 7h**~~ **Done.** Verified unreachable: since step 7 the only non-test module + that requires `openaiService` is `aiServiceFactory.js`, which hands it out for `openai` and as + the default for an unknown provider only, and no production code changes `config.aiProvider` at + runtime. `createClient()` in `services/llm/providers/openaiProvider.js` no longer builds an + Ollama (`/v1`) or a Custom client; it builds one for `openai` with an API key and + returns `null` otherwise. One case in `llm-transport-request-mapping` pins that — it failed + before the change (an Ollama client for `ollama`). +9. ~~**Full verification.**~~ **Done, except the live smoke run.** See "Final verification" below. + +--- + +## Final verification + +On 2026-10-15, at the commit that moved this document to `docs/`: + +- `node scripts/run-tests.js --all`: 112 passed, 7 skipped, 23 failed. The failed tests are the same list as on the + commit before step 7 (`0a7a2f8`) plus `manual-analyze-route`, whose nine cases pass and which + fails only because `tests/helpers/mount-router.js` cannot remove its temp directory on Windows + (EPERM) — the reason `duplicates-endpoints` fails there too. The rest of that list is + environmental on the Windows development machine (temp-dir EPERM/EBUSY, native crashes, CRLF + in the working copy, the shell's `ANTHROPIC_BASE_URL`). +- ESLint, Prettier and stylelint on every file the branch changes against upstream `main`, the + way CI selects them: clean. +- `node scripts/regen-openapi.js` + `git diff --exit-code OPENAPI/openapi.json`: no drift. +- **Not done: a manual smoke run** of `/manual/analyze` and a scan cycle against a real + Paperless-ngx and a real provider. No test exercises a live API, and none was available here. + Do this before the release that carries the refactoring: one document per configured provider + through the scan, one through the manual page, and check the token metrics appear for it. + +## Notes for whoever works on this code next + +- **Read first:** `/CLAUDE.md`, the Findings and done notes above, `services/llm/documentAnalyzer.js`, + `services/llm/promptLayout.js` and one composition root, e.g. + `services/llm/providers/azureProvider.js`. A new chat-completions provider is one composition + root plus a factory case. +- **Seams the tests rely on — keep them on the four exported singletons:** `service.client` is + replaced directly by many tests; `lastGenerateTextUsage` is read and set; `service.transport` is + inspected and its `generate` stubbed. Ollama's `apiUrl`, `model`, `documentAnalysisSchema`, + `_buildPrompt()`, `_callOllamaAPI(prompt, system, numCtx, schema)`, `_extractOllamaMetrics()`, + `_calculatePromptTokenCount()` and `_calculateNumCtx()` are called or stubbed; they live on the + singleton in `ollamaProvider.js`. `ollama-temperature-wiring` mocks axios through + `require.cache` and loads `services/ollamaService.js` first in a fresh process, so the client is + created with the mock. `ollama-upstream-error` patches the global `setTimeout`. +- **The factory is the only door.** No route or service requires a provider singleton directly; + `POST /manual/analyze` and the scan loop both call `AIServiceFactory.getService()`. +- **Router tests:** `tests/helpers/mount-router.js` mounts the real `routes/setup.js` on a throwaway + database; `tests/test-manual-analyze-route.js` is the smallest example. On Windows its `close()` + fails with EPERM on the temp directory — expected there; the cases themselves pass. +- **Verify like this:** `node scripts/run-tests.js --all` takes about ten minutes. Some tests + fail for reasons unrelated to the code, depending on the machine (native crashes, temp-dir + permissions, line endings, environment variables of the shell). Never judge a run by its count: + run the suite on the parent commit as well and compare the lists of failed tests. Back every new + test with a mutation check: break the code it guards, see it fail, restore. + +## Out of Scope / Follow-ups + +- **Relocating tests into `tests/services/`.** Needs `findUnregisteredTests()` made recursive first + (Finding 8) and would bury this refactor in rename noise. Separate PR. +- **Thumbnail caching inside `analyzeDocument`** (Finding 7i): the thumbnail is fetched, written to + disk and never used in the prompt. Behavior preserved here; worth its own issue. It is now a + single `cacheThumbnail(id)` call per provider rather than four inline copies, so hoisting it to + the caller is a small change whenever someone wants to make it. +- **Vision/multimodal analysis**, which is presumably why the thumbnail is cached at all. +- **`CachedNameIdEnum`'s semantic match calls a method no provider has.** `_findSemanticMatch()` + (`services/CachedNameIdEnum.js`) calls `ai_service.generateAnswer()`, which exists only in + test stubs. The call throws, is caught and returns "no match", so tag/correspondent/document type + lookup silently falls back to its text matchers. Step 6 settled the contract — `generateText(prompt, +options)` is the same on all four, the Anthropic provider included — so either point it there + (it would then cost an AI call per unmatched name), or drop the AI tier. +- **A structured-output (`response_format`/`format`) path for the chat transports** to match what + Ollama already does. Would likely remove most of `responseParser`, but it is a behavior change + needing its own validation against real providers. +- **The Anthropic provider** (`services/anthropicService.js`, upstream v2026.10.01) arrived after + the plan and is not one of the four. It keeps its own `analyzeDocument()` over the Messages API, + with its own prompt and parser (`services/analysisPrompt.js`), structured output and prompt + caching. It already fails an answer without metadata with the same message (7k), but passes the + parsed answer on without the field defaults and has no refusal detection (7c). Making it a fifth + composition root needs a Messages transport and a decision on those two differences. +- **`services/llm/contracts.js`** with the typedefs of "The delegation contracts" was not written; + the constructor JSDoc of `DocumentAnalyzer` and the transports document the shapes instead. +- ~~**This plan document's location.**~~ Moved from `services/ManualServiceRefactoring.md` to + `docs/llm-service-refactoring.md` at the end of the refactoring; `services/` holds services. + +## Risks + +| Risk | Mitigation | +| ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| OCR fallback silently stops working because an error phrase changed | `llm-ocr-fallback-phrases` written in step 1, before any code moves | +| A caller reads a field that the unified `document` object no longer carries | `llm-provider-parity` compares full objects; `server.js:854`, `mistralOcrService.js:1027`, `routes/setup.js:3135` reviewed explicitly | +| Ollama's token arithmetic subtly changes and long documents start truncating | `charBudget` keeps the `/2` divisor and the real `num_ctx` formula; `ollama-response-limit` and `ollama-token-metrics` guard it | +| No test exercises a real provider API | Step 9 keeps a manual smoke run against at least one live provider in the acceptance criteria | +| The refactor lands as one unreviewable diff | Steps 2-6 are individually green and individually landable | diff --git a/routes/setup.js b/routes/setup.js index 7d41ea02..c53f4827 100644 --- a/routes/setup.js +++ b/routes/setup.js @@ -4,9 +4,6 @@ const rateLimit = require('express-rate-limit'); const cron = require('node-cron'); const setupService = require('../services/setupService.js'); const paperlessService = require('../services/paperlessService.js'); -const openaiService = require('../services/openaiService.js'); -const ollamaService = require('../services/ollamaService.js'); -const azureService = require('../services/azureService.js'); const documentModel = require('../models/document.js'); const AIServiceFactory = require('../services/aiServiceFactory'); const configFile = require('../config/config.js'); @@ -31,7 +28,6 @@ const { verifySessionToken, SESSION_TOKEN_TYPE, } = require('./auth.js'); -const customService = require('../services/customService.js'); const mistralOcrService = require('../services/mistralOcrService'); const quickstartService = require('../services/quickstartService'); const reconciliationService = require('../services/reconciliationService'); @@ -4774,23 +4770,21 @@ router.get('/setup', async (req, res) => { console.warn( '[SECURITY] Attempting to access /setup in degraded state (corrupted database)' ); - return res - .status(500) - .render('setup-error', { + try { + return res.status(500).render('setup-error', { title: 'System Configuration Error', errorMessage: 'The system configuration exists but the database is inaccessible or corrupted. This is an administrative error state. Please check system logs and database integrity.', supportText: 'This may occur if: (1) the database file was deleted or corrupted, (2) file permissions changed, or (3) the database is locked. Restart the application after verifying database and permissions.', - }) - .catch(() => { - // Fallback if setup-error template doesn't exist - res - .status(500) - .send( - '

System Configuration Error

Database is inaccessible. Please contact your administrator.

' - ); }); + } catch { + return res + .status(500) + .send( + '

System Configuration Error

Database is inaccessible. Please contact your administrator.

' + ); + } } // Base configuration object - load this FIRST, before any checks @@ -8191,7 +8185,9 @@ router.post( * This endpoint processes the document text to extract relevant information such as tags, * correspondent, and document type based on content analysis. * - * The analysis is performed using the AI provider configured in the application settings. + * The analysis is performed by the AI provider configured in the application settings, the + * same one the scan uses. The existing tags, correspondents and document types are read from + * Paperless-ngx. When the request carries an `id`, the token usage is recorded against it. * tags: * - Documents * - API @@ -8212,53 +8208,23 @@ router.post( * example: "Invoice from Acme Corp. Total amount: $125.00, Due date: 2023-08-15" * existingTags: * type: array - * description: List of existing tags in the system to help with tag matching + * description: Accepted but not read; the existing tags are taken from Paperless-ngx * items: * type: string * example: ["Invoice", "Finance", "Acme Corp"] * id: * type: string - * description: Optional document ID for tracking metrics + * description: Optional Paperless-ngx document ID; the token usage is recorded against it * example: "doc_123" * responses: * 200: - * description: Document analysis results + * description: | + * The analysis result. A failed analysis is answered with status 200 as well, with + * `error` set and `metrics` null. * content: * application/json: * schema: - * type: object - * properties: - * correspondent: - * type: string - * description: Detected correspondent name - * example: "Acme Corp" - * title: - * type: string - * description: Suggested document title - * example: "Acme Corp Invoice - August 2023" - * tags: - * type: array - * description: Suggested tags for the document - * items: - * type: string - * example: ["Invoice", "Finance"] - * documentType: - * type: string - * description: Detected document type - * example: "Invoice" - * metrics: - * type: object - * description: Token usage metrics (when using OpenAI) - * properties: - * promptTokens: - * type: number - * example: 350 - * completionTokens: - * type: number - * example: 120 - * totalTokens: - * type: number - * example: 470 + * $ref: '#/components/schemas/AnalysisResult' * 400: * description: Invalid request parameters * content: @@ -8266,7 +8232,7 @@ router.post( * schema: * $ref: '#/components/schemas/Error' * 500: - * description: Server error or AI provider not configured + * description: Server error, e.g. Paperless-ngx could not be reached * content: * application/json: * schema: @@ -8290,71 +8256,25 @@ router.post('/manual/analyze', express.json(), async (req, res) => { .json({ error: 'Valid content string is required' }); } - if (process.env.AI_PROVIDER === 'openai') { - const analyzeDocument = await openaiService.analyzeDocument( - content, - existingTagsList, - existingCorrespondentList, - existingDocumentTypesList, - id || [] - ); + // The factory picks the provider the scan loop uses. Every provider + // answers with the same AnalysisResult; metrics need a document to be + // recorded against, and a failed analysis carries none. + const analysis = await AIServiceFactory.getService().analyzeDocument( + content, + existingTagsList, + existingCorrespondentList, + existingDocumentTypesList, + id + ); + if (analysis.metrics && id) { await documentModel.addOpenAIMetrics( id, - analyzeDocument.metrics.promptTokens, - analyzeDocument.metrics.completionTokens, - analyzeDocument.metrics.totalTokens - ); - return res.json(analyzeDocument); - } else if (process.env.AI_PROVIDER === 'ollama') { - const analyzeDocument = await ollamaService.analyzeDocument( - content, - existingTagsList, - existingCorrespondentList, - existingDocumentTypesList, - id || [] + analysis.metrics.promptTokens, + analysis.metrics.completionTokens, + analysis.metrics.totalTokens ); - return res.json(analyzeDocument); - } else if (process.env.AI_PROVIDER === 'custom') { - const analyzeDocument = await customService.analyzeDocument( - content, - existingTagsList, - existingCorrespondentList, - existingDocumentTypesList, - id || [] - ); - return res.json(analyzeDocument); - } else if (process.env.AI_PROVIDER === 'azure') { - const analyzeDocument = await azureService.analyzeDocument( - content, - existingTagsList, - existingCorrespondentList, - existingDocumentTypesList, - id || [] - ); - return res.json(analyzeDocument); - } else if (process.env.AI_PROVIDER === 'anthropic') { - // Through the factory, which owns the Claude service like every scan - // does. Its tokens are billed like OpenAI's, so they are recorded too. - const analyzeDocument = - await AIServiceFactory.getService().analyzeDocument( - content, - existingTagsList, - existingCorrespondentList, - existingDocumentTypesList, - id || [] - ); - if (analyzeDocument.metrics && id) { - await documentModel.addOpenAIMetrics( - id, - analyzeDocument.metrics.promptTokens, - analyzeDocument.metrics.completionTokens, - analyzeDocument.metrics.totalTokens - ); - } - return res.json(analyzeDocument); - } else { - return res.status(500).json({ error: 'AI provider not configured' }); } + return res.json(analysis); } catch (error) { console.error('Analysis error:', error); return res.status(500).json({ error: error.message }); @@ -10239,10 +10159,6 @@ router.post('/settings', express.json(), async (req, res) => { ...updatedConfig, }; - // The route has already validated the submitted values (and Paperless - // reachability when its settings changed). Skip saveConfig's built-in - // validateConfig so saving does not live-test Paperless/AI on every save. - await setupService.saveConfig(mergedConfig, { skipValidation: true }); try { for (const field of processedCustomFields) { await paperlessService.createCustomFieldSafely( @@ -10252,9 +10168,17 @@ router.post('/settings', express.json(), async (req, res) => { ); } } catch (error) { - console.log('[ERROR] Error creating custom fields:', error); + if (error.code === 'CUSTOM_FIELD_TYPE_MISMATCH') { + return res.status(400).json({ error: error.message }); + } + throw error; } + // The route has already validated the submitted values (and Paperless + // reachability when its settings changed). Skip saveConfig's built-in + // validateConfig so saving does not live-test Paperless/AI on every save. + await setupService.saveConfig(mergedConfig, { skipValidation: true }); + res.json({ success: true, message: 'Configuration saved successfully.', diff --git a/schemas.js b/schemas.js index 5ea569bf..91c4a0df 100644 --- a/schemas.js +++ b/schemas.js @@ -193,6 +193,10 @@ * * AnalysisResult: * type: object + * description: | + * What analyzeDocument() returns on every AI provider. A failed analysis is + * returned, not thrown: `document` is then `{ tags: [], correspondent: null }`, + * `metrics` is null and `error` says why. * properties: * document: * type: object @@ -209,6 +213,7 @@ * example: ["invoice", "utilities", "2023"] * correspondent: * type: string + * nullable: true * description: Suggested correspondent name * example: ABC Corporation * document_type: @@ -239,6 +244,8 @@ * example: "123.45" * metrics: * type: object + * nullable: true + * description: Token usage of the analysis; null when it failed * properties: * promptTokens: * type: integer @@ -252,10 +259,18 @@ * type: integer * description: Total tokens used * example: 570 + * truncated: + * type: boolean + * description: Whether the document content was cut to fit the token limit + * example: false * error: * type: string * description: Error message if analysis failed * example: null + * errorCode: + * type: string + * description: Machine-readable reason of a failed analysis, when the provider gave one + * example: null * * Tag: * type: object diff --git a/scripts/run-tests.js b/scripts/run-tests.js index aeb614eb..e5c30c39 100644 --- a/scripts/run-tests.js +++ b/scripts/run-tests.js @@ -31,6 +31,7 @@ function formatStatus(status) { } const TESTS = { + 'cached-name-id-enum': 'test-cached-name-id-enum.js', 'dashboard-stats-cache': 'test-dashboard-stats-cache.js', 'document-type-restriction': 'test-document-type-restriction.js', 'effective-document-count-cache': 'test-effective-document-count-cache.js', @@ -62,6 +63,8 @@ const TESTS = { 'document-metadata-batching': 'test-document-metadata-batching.js', 'pr772-fix': 'test-pr772-fix.js', 'ollama-response-limit': 'test-ollama-response-limit.js', + 'ollama-upstream-error': 'test-ollama-upstream-error.js', + 'predefined-retrigger': 'test-predefined-retrigger.js', 'response-truncation-detection': 'test-response-truncation-detection.js', 'ollama-temperature-wiring': 'test-ollama-temperature-wiring.js', 'quickstart-model-classification': 'test-quickstart-model-classification.js', @@ -80,6 +83,7 @@ const TESTS = { 'setup-remote-guard': 'test-setup-remote-guard.js', 'thumbnail-auth-guard': 'test-thumbnail-auth-guard.js', 'thumbnail-startup-migration': 'test-thumbnail-startup-migration.js', + 'thumbnail-optional-analysis': 'test-thumbnail-optional-analysis.js', 'restriction-service': 'test-restriction-service.js', 'updated-service': 'test-updated-service.js', 'prompt-existing-data-serialization': @@ -93,6 +97,16 @@ const TESTS = { 'ai-temperature-config': 'test-ai-temperature-config.js', 'custom-field-monetary-normalization': 'test-custom-field-monetary-normalization.js', + 'llm-analysis-result-contract': 'test-llm-analysis-result-contract.js', + 'llm-document-analyzer': 'test-llm-document-analyzer.js', + 'llm-ocr-fallback-phrases': 'test-llm-ocr-fallback-phrases.js', + 'llm-prompt-builder': 'test-llm-prompt-builder.js', + 'llm-response-parser': 'test-llm-response-parser.js', + 'llm-token-budget': 'test-llm-token-budget.js', + 'llm-transport-request-mapping': 'test-llm-transport-request-mapping.js', + 'llm-azure-temperature-wiring': 'test-llm-azure-temperature-wiring.js', + 'llm-provider-parity': 'test-llm-provider-parity.js', + 'manual-analyze-route': 'test-manual-analyze-route.js', 'custom-field-longtext': 'test-custom-field-longtext.js', 'document-processing-service': 'test-document-processing-service.js', 'settings-custom-field-type-conflict': @@ -238,6 +252,7 @@ const AREAS = { 'scanner-startup-resilience', 'paperless-unreachable-banner', 'thumbnail-startup-migration', + 'thumbnail-optional-analysis', 'ai-temperature-config', 'custom-field-monetary-normalization', 'custom-field-longtext', @@ -246,6 +261,14 @@ const AREAS = { 'ollama-token-metrics', 'reconciliation-service', 'settings-paperless-url-fallback', + 'llm-analysis-result-contract', + 'llm-document-analyzer', + 'llm-ocr-fallback-phrases', + 'llm-response-parser', + 'llm-transport-request-mapping', + 'llm-azure-temperature-wiring', + 'llm-provider-parity', + 'manual-analyze-route', 'rescan-fetch-before-delete', ], prompts: [ @@ -253,6 +276,8 @@ const AREAS = { 'updated-service', 'prompt-existing-data-serialization', 'restricted-document-types-placeholder', + 'llm-prompt-builder', + 'llm-token-budget', 'token-limit-config', 'truncate-returns-string', ], diff --git a/server.js b/server.js index 9310adc1..7352fb67 100644 --- a/server.js +++ b/server.js @@ -655,6 +655,50 @@ async function saveOpenApiSpec() { } // Document processing functions + +/** + * True when the document carries at least one of the tags configured as the + * predefined-scan trigger (PROCESS_PREDEFINED_DOCUMENTS=yes + TAGS). Only + * meaningful in predefined mode; resolves the tag names to ids through the + * shared tag cache. + * + * @param {Object} doc - document as returned by getAllDocuments() (fields incl. tags) + * @returns {Promise} + */ +async function documentCarriesTriggerTag(doc) { + if (config.predefinedMode !== 'yes') { + return false; + } + + const triggerNames = (process.env.TAGS || '') + .split(',') + .map((tag) => tag.trim()) + .filter(Boolean); + if (triggerNames.length === 0) { + return false; + } + + const docTagIds = new Set( + (Array.isArray(doc.tags) ? doc.tags : []) + .map((tag) => (typeof tag === 'object' ? Number(tag?.id) : Number(tag))) + .filter((id) => Number.isInteger(id)) + ); + if (docTagIds.size === 0) { + return false; + } + + try { + const triggerIds = await paperlessService.resolveTagIdsByName(triggerNames); + return triggerIds.some((id) => docTagIds.has(id)); + } catch (error) { + console.warn( + `[WARN] Could not resolve trigger tags for re-check of document ${doc.id}:`, + error.message + ); + return false; + } +} + async function processDocument( doc, existingTags, @@ -663,7 +707,19 @@ async function processDocument( context = {} ) { const isProcessed = await documentModel.isDocumentProcessed(doc.id); - if (isProcessed) return null; + if (isProcessed) { + // Predefined-scan re-trigger: in "process only specific pre-tagged + // documents" mode, the presence of a configured trigger tag on an already + // processed document is the operator's explicit request to run it again — + // that is how documents are re-queued after fixing a bug. Outside + // predefined mode, or without a trigger tag, the record wins. + if (!(await documentCarriesTriggerTag(doc))) { + return null; + } + console.info( + `Document ${doc.id} carries a configured trigger tag; re-processing despite existing record` + ); + } // A document that waits in the OCR queue, or sits inside an OCR run right // now, gets both its text and its analysis from that path. Analysing it here diff --git a/services/CachedNameIdEnum.js b/services/CachedNameIdEnum.js new file mode 100644 index 00000000..9913f679 --- /dev/null +++ b/services/CachedNameIdEnum.js @@ -0,0 +1,399 @@ +const { safeExtractRelativePath } = require('./serviceUtils'); + +class CachedNameIdEnum { + constructor( + element_type_name, + url_prefix, + similarity_prompt, + cache_ttl_access, + ai_service, + page_size = 100 + ) { + this.ai_service = ai_service; + this.element_type_name = element_type_name; + this.url_prefix = url_prefix; + this.similarity_prompt = similarity_prompt; + this.cache_ttl_access = cache_ttl_access; + // Asked for on every page request. DRF clamps to its own max_page_size + // rather than rejecting, so a server that allows less simply pages more. + this.page_size = page_size; + this.cache_last_fetch_time = 0; + this.cache_refresh_promise = null; + this.elements_by_id = new Map(); + this.elements_by_name = new Map(); + this.element_list = []; + } + + _clear() { + this.elements_by_id.clear(); + this.elements_by_name.clear(); + this.element_list = []; + } + + _add_all(elements) { + this.element_list = this.element_list.concat(elements); + this.element_list.forEach((element) => { + this.elements_by_id.set(element.id, element); + this.elements_by_name.set(element.name.toLowerCase(), element); + }); + } + + async _cache_refill(client) { + this._clear(); + this._add_all(await this._fetchAll(client)); + } + + _result_element_from_element(element) { + return { + id: element.id, + name: element.name, + }; + } + + async _fetchAll(client) { + this._checkClient(client); + let nextUrl = this.url_prefix; + let existing_elements = []; + while (nextUrl) { + // page_size must ride in `params` — a bare property on the axios config + // is silently dropped so that the server answers with its default page size. + const response = await client.get(nextUrl, { + params: { page_size: this.page_size }, + }); + existing_elements = existing_elements.concat(response.data.results); + // Safely extract relative path from next URL to prevent SSRF + if (response.data.next) { + nextUrl = safeExtractRelativePath(response.data.next, client); + if (nextUrl) { + console.debug( + `[DEBUG] Next page URL: ${nextUrl} elements fetched so far: ${existing_elements.length}` + ); + } else { + console.warn(`[WARN] Response has data.next but Next page URL is + empty. elements fetched so far: ${existing_elements.length}`); + } + } else { + nextUrl = null; + this.cache_last_fetch_time = Date.now(); + console.debug( + `[DEBUG] Fetched ${existing_elements.length} ${this.element_type_name}s` + ); + console.debug( + `[DEBUG] Cache contains these ${this.element_type_name}s now: + ${existing_elements.map((element) => element.name).join(', ')}` + ); + } + } + return existing_elements; + } + + async _check_cache_is_not_outdated( + client, + cache_ttl = this.cache_ttl_access() + ) { + if (this.element_list.length === 0) { + await this.flush_cache(client); + return; + } + if (cache_ttl === 0) return; // No timeout configured, refresh only on cache miss + if (cache_ttl > 0) { + // check for timeout and refresh if necessary + const cacheAge = Date.now() - this.cache_last_fetch_time; + if (cacheAge < cache_ttl) return; + const ageSeconds = Math.floor(cacheAge / 1000); + const ttlSeconds = Math.floor(cache_ttl / 1000); + const expireTime = new Date( + this.cache_last_fetch_time + cache_ttl + ).toISOString(); + console.debug( + `[DEBUG] ${this.element_type_name} cache expired (age: ${ageSeconds}s, TTL: ${ttlSeconds}s, expired at: ${expireTime})` + ); + } + // cache_ttl < 0 or expired + await this.flush_cache(client); + } + + async _getByName(name, client, cache_ttl = this.cache_ttl_access()) { + await this._check_cache_is_not_outdated(client, cache_ttl); + let match = this.elements_by_name.get(name.toLowerCase()); + if (match) { + console.debug( + `[DEBUG] Found exact match for ${this.element_type_name} "${match.name}" with ID ${match.id}` + ); + return this._result_element_from_element(match); + } + match = await this._findSimilar(name); + if (match) { + console.debug( + `[DEBUG] Found similar match for ${this.element_type_name} "${name}": "${match.name}" with ID ${match.id}` + ); + return this._result_element_from_element(match); + } + console.debug( + `[DEBUG] No exact nor similar match found for ${this.element_type_name} "${name}"` + ); + return null; + } + + async _getById(id, client, cache_ttl = this.cache_ttl_access()) { + if (id === 0) { + console.error(`[ERROR] Id cannot be 0` + new Error().stack); + return null; + } + await this._check_cache_is_not_outdated(client, cache_ttl); + let match = this.elements_by_id.get(id); + if (match) { + console.log( + `[DEBUG] Found ${this.element_type_name} by ID #${match.id}: "${match.name}"` + ); + return this._result_element_from_element(match); + } + console.log(`[DEBUG] No ${this.element_type_name} with ID #${id} found`); + return null; + } + + async _findSemanticMatch(name) { + let selectPrompt = + `These ${this.element_type_name}s are known: ` + + `${this.element_list.map((e) => `${e.name} (ID: ${e.id})`).join(', ')}. ` + + `Is ${name} contained in this list? ${this.similarity_prompt}`; + + const format = { + type: 'object', + properties: { + match: { + type: ['object', 'null'], + properties: { + id: { type: 'number' }, + name: { type: 'string' }, + }, + required: ['id', 'name'], + }, + }, + required: ['match'], + }; + try { + let aiResult = await this.ai_service.generateAnswer( + selectPrompt, + undefined, + 300, + format + ); + console.debug(`[DEBUG] AI result: ${aiResult}`); + let parsedResult = + typeof aiResult === 'string' ? JSON.parse(aiResult) : aiResult; + if (!parsedResult.match) { + console.debug( + `[DEBUG] No ${this.element_type_name} found similar to "${name}"` + ); + return null; + } + const matchedElement = parsedResult.match; + console.debug( + `[DEBUG] Found similar match for ${this.element_type_name} "${name}": "${matchedElement.name}" with ID ${matchedElement.id}` + ); + return this._result_element_from_element(matchedElement); + } catch (error) { + console.error( + '[ERROR] while searching for existing correspondent:', + error.message + ); + } + + return null; + } + + async _findMatchByFilter(name, filter, filterName) { + let elementResultList = await this.element_list.filter(filter); + if (elementResultList.length === 0) { + console.log( + `[DEBUG] No ${this.element_type_name} found with ${filterName} "${name}"` + ); + return null; + } + if (elementResultList.length > 1) { + console.log( + `[DEBUG] Multiple ${this.element_type_name} found with a ${filterName} "${name}"`, + `[DEBUG] "${elementResultList.map((element) => element.name).join('", "')}"` + ); + return null; + } + const matchedElement = elementResultList[0]; + console.log( + `[DEBUG] Found "${filterName}" match for ${this.element_type_name} "${name}": "${matchedElement.name}" with ID ${matchedElement.id}` + ); + return this._result_element_from_element(matchedElement); + } + + async _findSubString(name) { + return this._findMatchByFilter( + name, + (element) => element.name.toLowerCase().includes(name.toLowerCase()), + 'substring in' + ); + } + + _splitToTerms(name) { + let terms = name + .toLowerCase() + .replace(/[\s-_.:]+/g, ' ') + .split(' '); + terms = terms.filter((term) => term.length > 0); + console.debug(`[debug] ${name} in terms: ${terms.join(', ')}`); + return terms; + } + + _isMatchBySubterms(name, candidate) { + let nameTerms = this._splitToTerms(name); + let candidateTerms = this._splitToTerms(candidate); + const termsToMatch = candidateTerms.length; + let matchNo = 0; + for (let candidateTerm of candidateTerms) { + if (nameTerms.includes(candidateTerm)) { + matchNo++; + } + } + let result = termsToMatch > 0 && matchNo === termsToMatch; + console.debug( + `[debug] ${name} is ${result ? '' : 'not'} in: ${candidateTerms.join(', ')}` + ); + return result; + } + + async _findByTermMatch(name) { + return this._findMatchByFilter( + name, + (element) => this._isMatchBySubterms(name, element.name), + 'has all terms' + ); + } + + async _findSimilar(name) { + // Todo: implement methods to find similar element by matching with single word permutations and with something + // like hemming distance before asking ai to spare tokens + let match = await this._findSubString(name); + if (match) { + return match; + } + match = await this._findByTermMatch(name); + if (match) { + return match; + } + return this._findSemanticMatch(name); + } + + _checkClient(client) { + if (!client) { + throw new Error('No client provided.'); + } + } + + _checkDefined(name, value) { + if (value === undefined || value === null) { + throw new Error(`Mandatory argument ${name} is undefined or null.`); + } + } + + /************************************************************************** + * + * Public API + * + *************************************************************************/ + + flush_cache(client) { + this._checkClient(client); + if (this.cache_refresh_promise) { + return this.cache_refresh_promise; + } + // No race condition: synchronous code is never preempted in Node.js's + // event loop, so no other call can reach here between the check above + // and the assignment below. + this.cache_refresh_promise = this._cache_refill(client).finally(() => { + this.cache_refresh_promise = null; + }); + return this.cache_refresh_promise; + } + + async getByName(client, name) { + this._checkClient(client); + this._checkDefined('name', name); + let match = await this._getByName(name, client); + if (!match) { + console.log( + `[DEBUG] Cache miss for ${this.element_type_name} "${name}", refreshing cache` + ); + match = await this._getByName(name, client, -1); + } + return match; + } + + async getById(client, id) { + this._checkClient(client); + this._checkDefined('id', id); + if (id === 0) { + return this._getById(id, client); + } + let match = await this._getById(id, client); + if (!match && id > Math.max(...this.elements_by_id.keys())) { + console.log( + `[DEBUG] Cache miss for ${this.element_type_name} #${id}, refreshing cache` + ); + match = await this._getById(id, client, -1); + } + return match; + } + + /** + * Insert a single element (e.g., one just created on the server) into the + * cache without a full refresh. + */ + add(element) { + if (!element || element.id == null || !element.name) { + return; + } + this.elements_by_id.set(element.id, element); + this.elements_by_name.set(element.name.toLowerCase(), element); + if (!this.element_list.some((e) => e.id === element.id)) { + this.element_list.push(element); + } + } + + /** + * Drop the cached elements without reading the server; the next access + * refills the cache. + */ + invalidate() { + this._clear(); + } + + async visitAllContainedElements(client, visitor) { + this._checkClient(client); + await this._check_cache_is_not_outdated(client); + let result_set = new Set(); + for (let element of this.element_list.values()) { + visitor(element, result_set); + } + return result_set; + } + + async getCount(client) { + this._checkClient(client); + await this._check_cache_is_not_outdated(client); + return this.element_list.length; + } + + /** + * All cached elements as `{id, name}` objects, refreshing when stale. + */ + async getAll(client) { + this._checkClient(client); + await this._check_cache_is_not_outdated(client); + return this.element_list.map((element) => + this._result_element_from_element(element) + ); + } +} + +module.exports = { + CachedNameIdEnum, +}; diff --git a/services/azureService.js b/services/azureService.js index f1cd9f2f..2ccda9af 100644 --- a/services/azureService.js +++ b/services/azureService.js @@ -1,588 +1,3 @@ -const { - calculateTokens, - calculateTotalPromptTokens, - truncateToTokenLimit, - writePromptToFile, - extractChatMessageContent, - assertCompletionNotTruncated, - toNameList, -} = require('./serviceUtils'); -const { - abortSignal, - abortedGenerationError, - hasNumber, - hasSystemPrompt, - isAbortError, - modelOverride, - progressHandler, - readCompletionUsage, - reasoningEffortForOpenAi, - reasoningEnabled, - runChatCompletionStream, - stripReasoningText, -} = require('./aiGenerateOptions'); -const axios = require('axios'); -const AzureOpenAI = require('openai').AzureOpenAI; -const config = require('../config/config'); -const paperlessService = require('./paperlessService'); -const fs = require('fs').promises; -const path = require('path'); -const { - THUMBNAIL_CACHE_DIR, - getThumbnailCachePath, -} = require('./thumbnailCachePaths'); -const RestrictionPromptService = require('./restrictionPromptService'); -const responseLogPath = path.join( - process.cwd(), - 'data', - 'logs', - 'response.txt' -); -/** What an operator can do about an answer that hit the completion cap. */ -const TRUNCATION_REMEDY = 'Raise Response Tokens (RESPONSE_TOKENS).'; -/** - * The first Azure API version that answers a streamed request with a usage - * chunk. Older versions reject `stream_options` outright, which would turn a - * working deployment into a 400 the moment progress is asked for. - */ -const STREAM_OPTIONS_SINCE = '2024-06-01'; -/** Version aliases that always mean "the newest surface". */ -const ROLLING_API_VERSIONS = ['preview', 'latest', 'v1']; - -/** - * Whether the configured API version understands - * `stream_options: { include_usage: true }`. - * - * Azure versions are dates, so the comparison is a string comparison on the - * leading YYYY-MM-DD; a preview of a supported date counts as supported. - * Anything unrecognized is treated as too old and the token count for that - * deployment stays an estimate, which is the harmless half of being wrong. - * - * @param {string} apiVersion - * @returns {boolean} - */ -function supportsStreamOptions(apiVersion) { - const version = String(apiVersion || '') - .trim() - .toLowerCase(); - if (ROLLING_API_VERSIONS.includes(version)) return true; - const date = /^(\d{4}-\d{2}-\d{2})/.exec(version); - return date ? date[1] >= STREAM_OPTIONS_SINCE : false; -} - -class AzureOpenAIService { - constructor() { - this.client = null; - } - - initialize() { - if (!this.client && config.aiProvider === 'azure') { - this.client = new AzureOpenAI({ - apiKey: config.azure.apiKey, - endpoint: config.azure.endpoint, - deploymentName: config.azure.deploymentName, - apiVersion: config.azure.apiVersion, - }); - } - } - - async analyzeDocument( - content, - existingTags = [], - existingCorrespondentList = [], - existingDocumentTypesList = [], - id, - customPrompt = null, - options = {} - ) { - const cachePath = getThumbnailCachePath(id); - try { - this.initialize(); - const now = new Date(); - const timestamp = now.toLocaleString('de-DE', { - dateStyle: 'short', - timeStyle: 'short', - }); - - if (!this.client) { - throw new Error('AzureOpenAI client not initialized'); - } - - // Handle thumbnail caching - try { - await fs.access(cachePath); - console.log('[DEBUG] Thumbnail already cached'); - } catch { - console.log('Thumbnail not cached, fetching from Paperless'); - - const thumbnailData = await paperlessService.getThumbnailImage(id); - - if (!thumbnailData) { - console.warn('Thumbnail not found'); - return; - } - - await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); - await fs.writeFile(cachePath, thumbnailData); - } - - // Format existing data - callers may hand over entity objects or plain names - const existingTagNames = toNameList(existingTags).join(', '); - const existingCorrespondentNames = toNameList( - existingCorrespondentList - ).join(', '); - const existingDocumentTypeNames = toNameList( - existingDocumentTypesList - ).join(', '); - - // Get external API data if available and validate it - let externalApiData = options.externalApiData || null; - let validatedExternalApiData = null; - - if (externalApiData) { - try { - validatedExternalApiData = - await this._validateAndTruncateExternalApiData(externalApiData); - console.log('[DEBUG] External API data validated and included'); - } catch (error) { - console.warn( - '[WARNING] External API data validation failed:', - error.message - ); - validatedExternalApiData = null; - } - } - - let systemPrompt = ''; - let promptTags = ''; - const model = process.env.AZURE_DEPLOYMENT_NAME; - - // Parse CUSTOM_FIELDS from environment variable - let customFieldsObj; - try { - customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS); - } catch (error) { - console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); - console.debug(error); - customFieldsObj = { custom_fields: [] }; - } - - // Generate custom fields template for the prompt - const customFieldsTemplate = {}; - - customFieldsObj.custom_fields.forEach((field, index) => { - let valueHint; - if (field.data_type === 'date') { - valueHint = - 'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis'; - } else if (field.data_type === 'boolean') { - valueHint = "Fill in 'true' or 'false' based on your analysis"; - } else { - valueHint = 'Fill in the value based on your analysis'; - } - customFieldsTemplate[index] = { - field_name: field.value, - value: valueHint, - }; - }); - - // Convert template to string for replacement and wrap in custom_fields - const customFieldsStr = - '"custom_fields": ' + - JSON.stringify(customFieldsTemplate, null, 2) - .split('\n') - .map((line) => ' ' + line) // Add proper indentation - .join('\n'); - - // Get system prompt and model - if ( - config.useExistingData === 'yes' && - config.restrictToExistingTags === 'no' && - config.restrictToExistingCorrespondents === 'no' - ) { - systemPrompt = - ` - Pre-existing tags: ${existingTagNames}\n\n - Pre-existing correspondents: ${existingCorrespondentNames}\n\n - Pre-existing document types: ${existingDocumentTypeNames}\n\n - ` + - process.env.SYSTEM_PROMPT + - '\n\n' + - config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr); - promptTags = ''; - } else { - const mustHavePrompt = config.mustHavePrompt.replace( - '%CUSTOMFIELDS%', - customFieldsStr - ); - systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt; - promptTags = ''; - } - - // Process placeholder replacements in system prompt - systemPrompt = RestrictionPromptService.processRestrictionsInPrompt( - systemPrompt, - existingTags, - existingCorrespondentList, - existingDocumentTypesList - ); - - // Include validated external API data if available - if (validatedExternalApiData) { - systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`; - } - - if (process.env.USE_PROMPT_TAGS === 'yes') { - promptTags = process.env.PROMPT_TAGS; - systemPrompt = - ` - Take these tags and try to match one or more to the document content.\n\n - ` + config.specialPromptPreDefinedTags; - } - - if (customPrompt) { - console.log( - '[DEBUG] Replace system prompt with custom prompt via WebHook' - ); - systemPrompt = - customPrompt + - '\n\n' + - config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr); - } - - // Calculate tokens AFTER all prompt modifications are complete - const totalPromptTokens = await calculateTotalPromptTokens( - systemPrompt, - process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [], - model - ); - - const maxTokens = Number(config.tokenLimit); - const reservedTokens = totalPromptTokens + Number(config.responseTokens); - const availableTokens = maxTokens - reservedTokens; - - // Validate that we have positive available tokens. A non-finite budget - // is the same failure: it slips past a bare `<= 0` and truncates the - // document down to nothing instead of stopping. - if (!Number.isFinite(availableTokens) || availableTokens <= 0) { - console.warn( - `[WARNING] No available tokens for content. Reserved: ${reservedTokens}, Max: ${maxTokens}` - ); - throw new Error( - 'Token limit exceeded: prompt too large for available token limit' - ); - } - - console.log( - `[DEBUG] Token calculation - Prompt: ${totalPromptTokens}, Reserved: ${reservedTokens}, Available: ${availableTokens}` - ); - console.log( - `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting` - ); - console.log( - `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}` - ); - - const truncatedContent = await truncateToTokenLimit( - content, - availableTokens, - model - ); - - await writePromptToFile(systemPrompt, truncatedContent); - - const response = await this.client.chat.completions.create({ - model: model, - messages: [ - { - role: 'system', - content: systemPrompt, - }, - { - role: 'user', - content: truncatedContent, - }, - ], - temperature: 0.3, - }); - - assertCompletionNotTruncated( - response, - 'AzureOpenAI', - 'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.' - ); - const message = response?.choices?.[0]?.message; - let jsonContent = extractChatMessageContent(message, 'AzureOpenAI'); - if (!jsonContent) { - throw new Error('Invalid API response structure'); - } - - console.log(`[DEBUG] [${timestamp}] AzureOpenAI request sent`); - console.log( - `[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}` - ); - - const usage = response.usage; - const mappedUsage = { - promptTokens: usage.prompt_tokens, - completionTokens: usage.completion_tokens, - totalTokens: usage.total_tokens, - }; - - // Strip ... reasoning tags from models like Qwen3, DeepSeek-R1 - jsonContent = jsonContent.replace(/[\s\S]*?<\/think>/g, ''); - jsonContent = jsonContent - .replace(/```json\n?/g, '') - .replace(/```\n?/g, '') - .trim(); - - let parsedResponse; - try { - parsedResponse = JSON.parse(jsonContent); - } catch (error) { - console.error(`Failed to parse JSON response: ${error.message}`); - console.debug(error); - throw new Error('Invalid JSON response from API', { cause: error }); - } - - try { - await fs.mkdir(path.dirname(responseLogPath), { recursive: true }); - await fs.appendFile(responseLogPath, `${jsonContent}\n`); - } catch (logError) { - console.warn('Failed to write AI response log:', logError.message); - } - - if ( - !parsedResponse || - !Array.isArray(parsedResponse.tags) || - (typeof parsedResponse.correspondent !== 'string' && - parsedResponse.correspondent !== null) - ) { - throw new Error( - 'AI could not determine assignable metadata: no tags or correspondent found' - ); - } - - return { - document: parsedResponse, - metrics: mappedUsage, - truncated: truncatedContent.length < content.length, - }; - } catch (error) { - console.error(`Failed to analyze document: ${error.message}`); - console.debug(error); - return { - document: { tags: [], correspondent: null }, - metrics: null, - error: error.message, - // Undefined for everything that is not one of ours; the scan loop - // falls back to its generic reason then. - errorCode: error.code, - }; - } - } - - /** - * Validate and truncate external API data to prevent token overflow - * @param {any} apiData - The external API data to validate - * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500) - * @returns {string} - Validated and potentially truncated data string - */ - async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) { - if (!apiData) { - return null; - } - - const dataString = - typeof apiData === 'object' - ? JSON.stringify(apiData, null, 2) - : String(apiData); - - // Calculate tokens for the data - const dataTokens = await calculateTokens( - dataString, - process.env.AZURE_DEPLOYMENT_NAME - ); - - if (dataTokens > maxTokens) { - console.warn( - `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating` - ); - return await truncateToTokenLimit( - dataString, - maxTokens, - process.env.AZURE_DEPLOYMENT_NAME - ); - } - - console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`); - return dataString; - } - - /** - * Generate text based on a prompt - * @param {string} prompt - The prompt to generate text from - * @param {object} [options] - Optional overrides for this one call - * @param {string} [options.systemPrompt] - Sent as the system message - * @param {number} [options.temperature] - Overrides the default of 0.7 - * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS - * @param {AbortSignal} [options.signal] - Cancels the request in flight - * @param {Function} [options.onProgress] - Streams the answer and reports it - * as it grows; see GenerateTextProgress in aiGenerateOptions - * @param {boolean} [options.reasoning] - false sends reasoning_effort on the - * deployments that take it and nothing on the others, which have no - * thinking switch; true and null send nothing - * @returns {Promise} - The generated text - */ - async generateText(prompt, options = {}) { - // Read before the try so the catch can tell a stopped request from a - // failed one. - const signal = abortSignal(options); - const onProgress = progressHandler(options); - - try { - this.initialize(); - - if (!this.client) { - throw new Error('AzureOpenAI client not initialized - missing API key'); - } - - // Azure addresses a deployment rather than a model, so an override - // names the deployment to send this one request to. - const model = modelOverride(options) || process.env.AZURE_DEPLOYMENT_NAME; - const messages = []; - if (hasSystemPrompt(options)) { - messages.push({ role: 'system', content: options.systemPrompt }); - } - messages.push({ role: 'user', content: prompt }); - - const request = { - model: model, - messages, - temperature: hasNumber(options.temperature) ? options.temperature : 0.7, - // Was a hardcoded 1000, which quarrelled with the setting the same way - // Ollama's num_predict did — the reservation followed the operator, - // the limit did not. - max_tokens: - hasNumber(options.maxTokens) && options.maxTokens > 0 - ? Math.floor(options.maxTokens) - : Number(config.responseTokens), - }; - // As on OpenAI: the reasoning deployments take reasoning_effort, every - // other one answers 400 for it, so it is sent for those names only and - // only when the caller asked for no thinking. - if (reasoningEnabled(options) === false) { - const effort = reasoningEffortForOpenAi(model); - if (effort) request.reasoning_effort = effort; - } - - this.lastGenerateTextUsage = null; - // The second argument is the SDK's request option bag; a caller that - // brought no signal gets the call it always got. - const requestOptions = signal ? { signal } : null; - - if (onProgress) { - const streamRequest = { ...request, stream: true }; - // Only where the configured API version takes it; an older one would - // answer 400 instead of streaming, so that deployment estimates. - if (supportsStreamOptions(config.azure.apiVersion)) { - streamRequest.stream_options = { include_usage: true }; - } - return await runChatCompletionStream({ - client: this.client, - request: streamRequest, - requestOptions, - signal, - onProgress, - provider: 'AzureOpenAI', - remedy: TRUNCATION_REMEDY, - onUsage: (usage) => { - this.lastGenerateTextUsage = usage; - }, - }); - } - - const response = requestOptions - ? await this.client.chat.completions.create(request, requestOptions) - : await this.client.chat.completions.create(request); - this.lastGenerateTextUsage = readCompletionUsage(response); - - const message = response?.choices?.[0]?.message; - // Reasoning is no part of the answer, and a block the cut-off left - // open is no part of it either. - const generatedText = extractChatMessageContent( - { ...message, content: stripReasoningText(message?.content) }, - 'AzureOpenAI' - ); - - // After the text is in hand: a cut-off answer is still a failure, but - // the caller can keep the prefix instead of paying for it twice. - assertCompletionNotTruncated(response, 'AzureOpenAI', TRUNCATION_REMEDY, { - partialText: generatedText, - }); - - if (!generatedText) { - throw new Error('Invalid API response structure'); - } - - return generatedText; - } catch (error) { - // One name for a stopped request, whatever the client called it; an - // abort this service raised already carries its partial text. - const thrown = - isAbortError(error, signal) && error?.name !== 'AbortError' - ? abortedGenerationError(error, '') - : error; - console.error( - `Error generating text with AzureOpenAI: ${thrown.message}` - ); - console.debug(thrown); - throw thrown; - } - } - - async checkStatus() { - try { - this.initialize(); - - if (!this.client) { - throw new Error( - 'Azure OpenAI client not initialized - missing API key' - ); - } - - const model = process.env.AZURE_DEPLOYMENT_NAME; - - // Token-free health check against deployment metadata endpoint. - const endpoint = (config.azure.endpoint || '').replace(/\/$/, ''); - const apiVersion = config.azure.apiVersion; - if (!endpoint || !apiVersion || !model) { - throw new Error( - 'Azure endpoint, apiVersion, or deployment name missing' - ); - } - - const url = `${endpoint}/openai/deployments/${model}?api-version=${encodeURIComponent(apiVersion)}`; - const response = await axios.get(url, { - headers: { - 'api-key': config.azure.apiKey, - }, - timeout: 10000, - }); - - if (response.status < 200 || response.status >= 300) { - return { status: 'error' }; - } - - return { status: 'ok', model: model }; - } catch (error) { - console.error( - `Error generating text with Azure OpenAI: ${error.message}` - ); - console.debug(error); - return { status: 'error', error: error.message }; - } - } -} - -module.exports = new AzureOpenAIService(); +// The Azure OpenAI provider is a composition root over the shared DocumentAnalyzer +// (services/llm/). This path stays so its callers need no change. +module.exports = require('./llm/providers/azureProvider'); diff --git a/services/customService.js b/services/customService.js index 03f81f8c..1e975e81 100644 --- a/services/customService.js +++ b/services/customService.js @@ -1,704 +1,4 @@ -const { - calculateTokens, - calculateTotalPromptTokens, - truncateToTokenLimit, - extractChatMessageContent, - assertCompletionNotTruncated, - isTimeoutError, - buildTimeoutErrorMessage, - toNameList, -} = require('./serviceUtils'); -const { - abortSignal, - abortedGenerationError, - hasNumber, - hasSystemPrompt, - isAbortError, - modelOverride, - progressHandler, - readCompletionUsage, - reasoningEffortForCompatible, - reasoningEnabled, - runChatCompletionStream, - stripReasoningText, -} = require('./aiGenerateOptions'); -const OpenAI = require('openai'); -const config = require('../config/config'); -const paperlessService = require('./paperlessService'); -const fs = require('fs').promises; -const path = require('path'); -const { - THUMBNAIL_CACHE_DIR, - getThumbnailCachePath, -} = require('./thumbnailCachePaths'); -const RestrictionPromptService = require('./restrictionPromptService'); -const responseLogPath = path.join( - process.cwd(), - 'data', - 'logs', - 'response.txt' -); -const CUSTOM_PROVIDER_FALLBACK_API_KEY = 'no-auth-required'; -/** What an operator can do about an answer that hit the completion cap. */ -const TRUNCATION_REMEDY = 'Raise Response Tokens (RESPONSE_TOKENS).'; -/** - * The soft switch the Qwen family reads out of the prompt itself. - * - * `chat_template_kwargs` only reaches a server that applies the model's chat - * template; a gateway that forwards the messages and drops the rest leaves the - * model thinking anyway. Qwen understands the same wish written into the last - * line of the user message, so both are sent and whichever arrives wins. - */ -const QWEN_NO_THINK_SWITCH = '/no_think'; -/** Models that understand the line above. */ -const QWEN_MODEL_PATTERN = /qwen/i; - -class CustomOpenAIService { - constructor() { - this.client = null; - this.tokenizer = null; - } - - _extractFirstJsonValue(text) { - if (!text) { - return null; - } - - const starts = []; - const firstObject = text.indexOf('{'); - const firstArray = text.indexOf('['); - - if (firstObject !== -1) { - starts.push(firstObject); - } - if (firstArray !== -1) { - starts.push(firstArray); - } - - if (starts.length === 0) { - return null; - } - - const startIndex = Math.min(...starts); - const opening = text[startIndex]; - const closing = opening === '{' ? '}' : ']'; - - let depth = 0; - let inString = false; - let escaping = false; - - for (let i = startIndex; i < text.length; i += 1) { - const char = text[i]; - - if (inString) { - if (escaping) { - escaping = false; - } else if (char === '\\') { - escaping = true; - } else if (char === '"') { - inString = false; - } - continue; - } - - if (char === '"') { - inString = true; - continue; - } - - if (char === opening) { - depth += 1; - } else if (char === closing) { - depth -= 1; - if (depth === 0) { - return text.slice(startIndex, i + 1); - } - } - } - - return null; - } - - _parseJsonResponse(rawContent) { - const sanitizedContent = String(rawContent || '') - .replace(/[\s\S]*?<\/think>/g, '') - .replace(/```json\n?/gi, '') - .replace(/```\n?/g, '') - .trim(); - - try { - return { - parsed: JSON.parse(sanitizedContent), - normalized: sanitizedContent, - }; - } catch (directParseError) { - const extractedJson = this._extractFirstJsonValue(sanitizedContent); - if (!extractedJson) { - throw new Error('Invalid JSON response from API', { - cause: directParseError, - }); - } - - try { - return { - parsed: JSON.parse(extractedJson), - normalized: extractedJson, - }; - } catch (extractedParseError) { - throw new Error('Invalid JSON response from API', { - cause: extractedParseError, - }); - } - } - } - - initialize() { - if (!this.client && config.aiProvider === 'custom') { - this.client = new OpenAI({ - baseURL: config.custom.apiUrl, - // OpenAI-compatible SDKs require an apiKey value even when auth is disabled. - apiKey: config.custom.apiKey || CUSTOM_PROVIDER_FALLBACK_API_KEY, - }); - } - } - - async analyzeDocument( - content, - existingTags = [], - existingCorrespondentList = [], - existingDocumentTypesList = [], - id, - customPrompt = null, - options = {} - ) { - const cachePath = getThumbnailCachePath(id); - try { - this.initialize(); - const now = new Date(); - const timestamp = now.toLocaleString('de-DE', { - dateStyle: 'short', - timeStyle: 'short', - }); - - if (!this.client) { - throw new Error('Custom OpenAI client not initialized'); - } - - // Handle thumbnail caching - try { - await fs.access(cachePath); - console.log('[DEBUG] Thumbnail already cached'); - } catch { - console.log('Thumbnail not cached, fetching from Paperless'); - - const thumbnailData = await paperlessService.getThumbnailImage(id); - - if (!thumbnailData) { - console.warn('Thumbnail not found'); - return; - } - - await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); - await fs.writeFile(cachePath, thumbnailData); - } - - // Format existing data - callers may hand over entity objects or plain names - const existingTagNames = toNameList(existingTags).join(', '); - const existingCorrespondentNames = toNameList( - existingCorrespondentList - ).join(', '); - const existingDocumentTypeNames = toNameList( - existingDocumentTypesList - ).join(', '); - - // Get external API data if available and validate it - let externalApiData = options.externalApiData || null; - let validatedExternalApiData = null; - - if (externalApiData) { - try { - validatedExternalApiData = - await this._validateAndTruncateExternalApiData(externalApiData); - console.log('[DEBUG] External API data validated and included'); - } catch (error) { - console.warn( - '[WARNING] External API data validation failed:', - error.message - ); - validatedExternalApiData = null; - } - } - - let systemPrompt = ''; - let promptTags = ''; - const model = config.custom.model; - - // Parse CUSTOM_FIELDS from environment variable - let customFieldsObj; - try { - customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS); - } catch (error) { - console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); - console.debug(error); - customFieldsObj = { custom_fields: [] }; - } - - // Generate custom fields template for the prompt - const customFieldsTemplate = {}; - - customFieldsObj.custom_fields.forEach((field, index) => { - let valueHint; - if (field.data_type === 'date') { - valueHint = - 'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis'; - } else if (field.data_type === 'boolean') { - valueHint = "Fill in 'true' or 'false' based on your analysis"; - } else { - valueHint = 'Fill in the value based on your analysis'; - } - customFieldsTemplate[index] = { - field_name: field.value, - value: valueHint, - }; - }); - - // Convert template to string for replacement and wrap in custom_fields - const customFieldsStr = - '"custom_fields": ' + - JSON.stringify(customFieldsTemplate, null, 2) - .split('\n') - .map((line) => ' ' + line) // Add proper indentation - .join('\n'); - - // Get system prompt based on configuration - if ( - config.useExistingData === 'yes' && - config.restrictToExistingTags === 'no' && - config.restrictToExistingCorrespondents === 'no' - ) { - systemPrompt = - ` - Pre-existing tags: ${existingTagNames}\n\n - Pre-existing correspondents: ${existingCorrespondentNames}\n\n - Pre-existing document types: ${existingDocumentTypeNames}\n\n - ` + - process.env.SYSTEM_PROMPT + - '\n\n' + - config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr); - promptTags = ''; - } else { - const mustHavePrompt = config.mustHavePrompt.replace( - '%CUSTOMFIELDS%', - customFieldsStr - ); - systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt; - promptTags = ''; - } - - // Process placeholder replacements in system prompt - systemPrompt = RestrictionPromptService.processRestrictionsInPrompt( - systemPrompt, - existingTags, - existingCorrespondentList, - existingDocumentTypesList - ); - - // Include validated external API data if available - if (validatedExternalApiData) { - systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`; - } - - if (process.env.USE_PROMPT_TAGS === 'yes') { - promptTags = process.env.PROMPT_TAGS; - systemPrompt = - ` - Take these tags and try to match one or more to the document content.\n\n - ` + config.specialPromptPreDefinedTags; - } - - // Custom prompt override if provided - if (customPrompt) { - console.log('[DEBUG] Replace system prompt with custom prompt'); - systemPrompt = - customPrompt + - '\n\n' + - config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr); - } - - // Calculate tokens AFTER all prompt modifications are complete - const totalPromptTokens = await calculateTotalPromptTokens( - systemPrompt, - process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [], - model - ); - - const maxTokens = Number(config.tokenLimit); - const reservedTokens = totalPromptTokens + Number(config.responseTokens); - const availableTokens = maxTokens - reservedTokens; - - // Validate that we have positive available tokens. A non-finite budget - // is the same failure: it slips past a bare `<= 0` and truncates the - // document down to nothing instead of stopping. - if (!Number.isFinite(availableTokens) || availableTokens <= 0) { - console.warn( - `[WARNING] No available tokens for content. Reserved: ${reservedTokens}, Max: ${maxTokens}` - ); - throw new Error( - 'Token limit exceeded: prompt too large for available token limit' - ); - } - - console.log( - `[DEBUG] Token calculation - Prompt: ${totalPromptTokens}, Reserved: ${reservedTokens}, Available: ${availableTokens}` - ); - console.log( - `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting` - ); - console.log( - `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}` - ); - - const truncatedContent = await truncateToTokenLimit( - content, - availableTokens, - model - ); - - // console.log('######################################################################'); - // console.log(`[DEBUG] Content length: ${content.length}, Truncated content length: ${truncatedContent.length}`); - // console.log(`[DEBUG] Truncated content: ${truncatedContent}`); - // console.log(`[DEBUG] System prompt: ${systemPrompt}`); - // console.log(`[DEBUG] Prompt tags: ${promptTags}`); - // console.log(`[DEBUG] Model: ${model}`); - // console.log(`[DEBUG] Custom fields: ${customFieldsStr}`); - // console.log(`[DEBUG] Existing tags: ${existingTagNames}`); - // console.log(`[DEBUG] Existing correspondents: ${existingCorrespondentNames}`); - // console.log(`[DEBUG] Custom prompt: ${customPrompt}`); - // console.log(`[DEBUG] External API data: ${validatedExternalApiData}`); - // console.log('######################################################################'); - - const response = await this.client.chat.completions.create({ - model: model, - messages: [ - { - role: 'system', - content: systemPrompt, - }, - { - role: 'user', - content: truncatedContent, - }, - ], - temperature: config.aiTemperatureAnalysis, - }); - - // Handle response - //console.log(`MESSAGE: ${response?.choices?.[0]?.message?.content}`); - assertCompletionNotTruncated( - response, - 'Custom OpenAI', - 'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.' - ); - const message = response?.choices?.[0]?.message; - let jsonContent = extractChatMessageContent(message, 'Custom OpenAI'); - if (!jsonContent) { - throw new Error('Invalid API response structure'); - } - - // Log token usage - console.log(`[DEBUG] [${timestamp}] Custom OpenAI request sent`); - console.log( - `[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}` - ); - - const usage = response.usage; - const mappedUsage = { - promptTokens: usage.prompt_tokens, - completionTokens: usage.completion_tokens, - totalTokens: usage.total_tokens, - }; - - let parsedResponse; - try { - const parsedResult = this._parseJsonResponse(jsonContent); - parsedResponse = parsedResult.parsed; - jsonContent = parsedResult.normalized; - } catch (error) { - console.error(`Failed to parse JSON response: ${error.message}`); - console.debug(error); - throw error; - } - - try { - await fs.mkdir(path.dirname(responseLogPath), { recursive: true }); - await fs.appendFile(responseLogPath, `${jsonContent}\n`); - } catch (logError) { - console.warn('Failed to write AI response log:', logError.message); - } - - // Validate response structure - if ( - !parsedResponse || - !Array.isArray(parsedResponse.tags) || - (typeof parsedResponse.correspondent !== 'string' && - parsedResponse.correspondent !== null) - ) { - throw new Error( - 'AI could not determine assignable metadata: no tags or correspondent found' - ); - } - - return { - document: parsedResponse, - metrics: mappedUsage, - truncated: truncatedContent.length < content.length, - }; - } catch (error) { - const normalizedMessage = isTimeoutError(error) - ? buildTimeoutErrorMessage('AI') - : error.message; - - if (isTimeoutError(error)) { - console.error( - `[TIMEOUT][AI] Custom provider request timed out: ${error.message}` - ); - } - - console.error(`Failed to analyze document: ${normalizedMessage}`); - console.debug(error); - return { - document: { tags: [], correspondent: null }, - metrics: null, - error: normalizedMessage, - // Undefined for everything that is not one of ours; the scan loop - // falls back to its generic reason then. - errorCode: error.code, - }; - } - } - - /** - * Validate and truncate external API data to prevent token overflow - * @param {any} apiData - The external API data to validate - * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500) - * @returns {string} - Validated and potentially truncated data string - */ - async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) { - if (!apiData) { - return null; - } - - const dataString = - typeof apiData === 'object' - ? JSON.stringify(apiData, null, 2) - : String(apiData); - - // Calculate tokens for the data - const dataTokens = await calculateTokens(dataString, config.custom.model); - - if (dataTokens > maxTokens) { - console.warn( - `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating` - ); - return await truncateToTokenLimit( - dataString, - maxTokens, - config.custom.model - ); - } - - console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`); - return dataString; - } - - /** - * Generate text based on a prompt - * @param {string} prompt - The prompt to generate text from - * @param {object} [options] - Optional overrides for this one call - * @param {string} [options.systemPrompt] - Sent as the system message - * @param {number} [options.temperature] - Overrides AI_TEMPERATURE_GENERATION - * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS, still clamped - * @param {AbortSignal} [options.signal] - Cancels the request in flight - * @param {Function} [options.onProgress] - Streams the answer and reports it - * as it grows; see GenerateTextProgress in aiGenerateOptions - * @param {boolean} [options.reasoning] - Switches the model's thinking off - * or on for this one call; nothing new is sent when the caller says - * nothing. Off also sends reasoning_effort where the model family takes it - * @returns {Promise} - The generated text - */ - async generateText(prompt, options = {}) { - // Read before the try so the catch can tell a stopped request from a - // failed one. - const signal = abortSignal(options); - const onProgress = progressHandler(options); - - try { - this.initialize(); - - if (!this.client) { - throw new Error( - 'Custom OpenAI client not initialized - missing API key' - ); - } - - const model = modelOverride(options) || config.custom.model; - const maxContextTokens = Number(config.tokenLimit) || 128000; - const desiredCompletionTokens = - hasNumber(options.maxTokens) && options.maxTokens > 0 - ? Math.floor(options.maxTokens) - : Number(config.responseTokens) || 1000; - const promptTokens = await calculateTokens(prompt, model); - const availableCompletionTokens = Math.max( - 1, - maxContextTokens - promptTokens - 64 - ); - const maxCompletionTokens = Math.max( - 1, - Math.min(desiredCompletionTokens, availableCompletionTokens) - ); - - if (maxCompletionTokens < desiredCompletionTokens) { - console.log( - `[DEBUG] Clamped max_tokens for custom generateText from ${desiredCompletionTokens} to ${maxCompletionTokens} ` + - `(context limit: ${maxContextTokens}, prompt tokens: ${promptTokens})` - ); - } - - const reasoning = reasoningEnabled(options); - const messages = []; - if (hasSystemPrompt(options)) { - messages.push({ role: 'system', content: options.systemPrompt }); - } - messages.push({ - role: 'user', - content: - reasoning === false && QWEN_MODEL_PATTERN.test(model) - ? `${prompt}\n${QWEN_NO_THINK_SWITCH}` - : prompt, - }); - - const request = { - model: model, - messages, - temperature: hasNumber(options.temperature) - ? options.temperature - : config.aiTemperatureGeneration, - max_tokens: maxCompletionTokens, - }; - // The key every OpenAI-compatible server that applies a chat template - // understands; one that does not know it passes it through and the - // model decides for itself, as it did before. - if (reasoning !== null) { - request.chat_template_kwargs = { enable_thinking: reasoning }; - } - // A hosted reasoning model reads neither of the two above and thinks - // anyway; the families that take reasoning_effort are told in their own - // dialect. Only when the caller asked for no thinking, and only for - // those names — another model answers 400 for the field. - if (reasoning === false) { - const effort = reasoningEffortForCompatible(model); - if (effort) request.reasoning_effort = effort; - } - - this.lastGenerateTextUsage = null; - // The second argument is the SDK's request option bag; a caller that - // brought no signal gets the call it always got. - const requestOptions = signal ? { signal } : null; - - if (onProgress) { - // include_usage buys the exact completion count at the end of the - // stream; without it the report is an estimate all the way. - return await runChatCompletionStream({ - client: this.client, - request: { - ...request, - stream: true, - stream_options: { include_usage: true }, - }, - requestOptions, - signal, - onProgress, - provider: 'Custom OpenAI', - remedy: TRUNCATION_REMEDY, - onUsage: (usage) => { - this.lastGenerateTextUsage = usage; - }, - }); - } - - const response = requestOptions - ? await this.client.chat.completions.create(request, requestOptions) - : await this.client.chat.completions.create(request); - this.lastGenerateTextUsage = readCompletionUsage(response); - - const message = response?.choices?.[0]?.message; - // Reasoning is no part of the answer, and a block the cut-off left - // open is no part of it either. - const generatedText = extractChatMessageContent( - { ...message, content: stripReasoningText(message?.content) }, - 'Custom OpenAI' - ); - - // After the text is in hand: a cut-off answer is still a failure, but - // the caller can keep the prefix instead of paying for it twice. - assertCompletionNotTruncated( - response, - 'Custom OpenAI', - TRUNCATION_REMEDY, - { partialText: generatedText } - ); - - if (!generatedText) { - // Say what came back: a gateway that timed out behind a 200, a - // model that stopped after its reasoning without an answer, or an - // empty choices array all end here and need telling apart. - const finishReason = response?.choices?.[0]?.finish_reason ?? 'unknown'; - const head = JSON.stringify(response ?? null).slice(0, 200); - throw new Error( - `Invalid API response structure: the answer carried no text (finish_reason ${finishReason}; the response begins ${head})` - ); - } - - return generatedText; - } catch (error) { - // One name for a stopped request, whatever the client called it; an - // abort this service raised already carries its partial text. - const thrown = - isAbortError(error, signal) && error?.name !== 'AbortError' - ? abortedGenerationError(error, '') - : error; - console.error( - `Error generating text with Custom OpenAI: ${thrown.message}` - ); - console.debug(thrown); - throw thrown; - } - } - - async checkStatus() { - try { - this.initialize(); - - if (!this.client) { - throw new Error( - 'Custom OpenAI client not initialized - missing API key' - ); - } - - const model = config.custom.model; - - // Use token-free endpoint where supported by OpenAI-compatible providers. - await this.client.models.list(); - - return { status: 'ok', model: model }; - } catch (error) { - console.error( - `Error generating text with Custom OpenAI: ${error.message}` - ); - console.debug(error); - return { status: 'error', error: error.message }; - } - } -} - -module.exports = new CustomOpenAIService(); +// The Custom OpenAI provider is a composition root over the shared +// DocumentAnalyzer (services/llm/). This path stays so its callers need no +// change. +module.exports = require('./llm/providers/customProvider'); diff --git a/services/documentProcessingService.js b/services/documentProcessingService.js index 5a3c8964..2dbaa9d6 100644 --- a/services/documentProcessingService.js +++ b/services/documentProcessingService.js @@ -260,6 +260,31 @@ class DocumentProcessingService { const origLanguage = originalData.language ?? null; const recordedTitle = updateData.title || originalTitle; + // Predefined-scan trigger tags have done their job once the document has + // been processed; leaving them in place would make the next scan re-run + // the document forever. Removal only happens when the operator opts in. + if ( + config.predefinedMode === 'yes' && + config.removeTriggerTags === 'yes' && + Array.isArray(updateData.tags) + ) { + try { + const triggerNames = (process.env.TAGS || '') + .split(',') + .map((tag) => tag.trim()) + .filter(Boolean); + if (triggerNames.length > 0) { + updateData.removeTagIds = + await paperlessService.resolveTagIdsByName(triggerNames); + } + } catch (error) { + console.warn( + `[WARN] Could not resolve trigger tags for removal on document ${docId}:`, + error.message + ); + } + } + await documentModel.saveOriginalData( docId, originalTags, diff --git a/services/llm/analysisResult.js b/services/llm/analysisResult.js new file mode 100644 index 00000000..d4a5b345 --- /dev/null +++ b/services/llm/analysisResult.js @@ -0,0 +1,108 @@ +'use strict'; + +/** + * The one place the analyzeDocument() result shape lives. + * + * Three callers read it — server.js processDocument(), + * mistralOcrService._runAiAnalysis() and routes/setup.js processDocument() — + * and hand `document` to documentProcessingService.buildUpdateData(). A + * narrowed `document` does not fail, it silently stops setting fields. + * + * Failures are returned, never thrown: the scan loop routes a failed analysis + * to the OCR queue by matching `error` against fixed phrases + * (shouldQueueForOcrOnAiError), and a throw would skip that entirely. + */ + +/** + * @param {Object} document - What the model assigned + * @param {{promptTokens:number, completionTokens:number, totalTokens:number}|null} metrics + * @param {boolean} truncated - Whether the document was cut to fit + */ +function buildSuccess(document, metrics, truncated) { + return { document, metrics, truncated }; +} + +/** + * @param {Error} error - Why the analysis failed + * @param {string} [message] - Replaces error.message, e.g. a normalized timeout + */ +function buildFailure(error, message = error.message) { + return { + document: { tags: [], correspondent: null }, + metrics: null, + error: message, + // Undefined for everything that is not one of ours; the scan loop falls + // back to its generic reason then. + errorCode: error.code, + }; +} + +/** + * The answer of a model that declined the document as too thin to analyse. + * Returned with a placeholder document instead of thrown, so the retry loop + * does not ask again; the error phrase routes it to OCR. + * + * @param {string|number} id - Paperless document id + * @param {Object|null} metrics - The tokens the refusal cost + */ +function buildInsufficientContent(id, metrics) { + return { + document: { + tags: [], + correspondent: 'Unknown', + title: `Document ${id}`, + document_date: new Date().toISOString().split('T')[0], + document_type: 'Document', + language: 'und', + }, + metrics, + truncated: false, + error: 'Insufficient content for AI analysis', + }; +} + +/** + * Throw unless the answer assigns something: a tags array and a correspondent + * that is a string or null. + * + * @param {Object} parsed - The parsed answer + * @throws {Error} "AI could not determine assignable metadata: …" + */ +function assertAssignableMetadata(parsed) { + if ( + !parsed || + !Array.isArray(parsed.tags) || + (typeof parsed.correspondent !== 'string' && parsed.correspondent !== null) + ) { + throw new Error( + 'AI could not determine assignable metadata: no tags or correspondent found' + ); + } +} + +/** + * Every document field present: tags an array, everything else a value or + * null; fields the model invented are dropped. Shapes every provider's + * success, after assertAssignableMetadata() has seen the parsed answer. + * + * @param {Object} parsed - The parsed answer + */ +function withDocumentDefaults(parsed) { + return { + tags: Array.isArray(parsed.tags) ? parsed.tags : [], + correspondent: parsed.correspondent || null, + title: parsed.title || null, + document_date: parsed.document_date || null, + document_type: parsed.document_type || null, + language: parsed.language || null, + custom_fields: parsed.custom_fields || null, + }; +} + +module.exports = { + buildSuccess, + buildFailure, + buildInsufficientContent, + assertAssignableMetadata, + withDocumentDefaults, +}; diff --git a/services/llm/customFieldsTemplate.js b/services/llm/customFieldsTemplate.js new file mode 100644 index 00000000..7da1012a --- /dev/null +++ b/services/llm/customFieldsTemplate.js @@ -0,0 +1,71 @@ +'use strict'; + +/** + * The %CUSTOMFIELDS% block of the analysis prompt. + * + * Renders the custom fields configured in CUSTOM_FIELDS as the JSON fragment + * the model is asked to fill in, one entry per field with a hint for its data + * type. Every provider used to carry its own copy of this; Ollama had three. + */ + +const DATE_HINT = + 'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis'; +const BOOLEAN_HINT = "Fill in 'true' or 'false' based on your analysis"; +const VALUE_HINT = 'Fill in the value based on your analysis'; + +/** + * The value hint the model gets for one data type. + * + * @param {string} dataType Paperless-ngx custom field data type. + * @returns {string} + */ +function valueHintFor(dataType) { + if (dataType === 'date') return DATE_HINT; + if (dataType === 'boolean') return BOOLEAN_HINT; + return VALUE_HINT; +} + +/** + * The configured custom fields, or none when the setting cannot be read. + * + * @param {string|undefined} rawCustomFields CUSTOM_FIELDS as stored. + * @returns {{custom_fields: Array<{value: string, data_type: string}>}} + */ +function parseCustomFields(rawCustomFields) { + try { + return JSON.parse(rawCustomFields); + } catch (error) { + console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); + console.debug(error); + return { custom_fields: [] }; + } +} + +/** + * Build the `"custom_fields": {...}` fragment that replaces %CUSTOMFIELDS%. + * + * The fragment is indented so it sits inside the JSON example of + * mustHavePrompt; the model copies that layout back. + * + * @param {string|undefined} [rawCustomFields=process.env.CUSTOM_FIELDS] + * @returns {string} + */ +function buildCustomFieldsBlock(rawCustomFields = process.env.CUSTOM_FIELDS) { + const template = {}; + parseCustomFields(rawCustomFields).custom_fields.forEach((field, index) => { + template[index] = { + field_name: field.value, + value: valueHintFor(field.data_type), + }; + }); + + return ( + '"custom_fields": ' + + JSON.stringify(template, null, 2) + .split('\n') + .map((line) => ' ' + line) + .join('\n') + ); +} + +module.exports = { buildCustomFieldsBlock }; diff --git a/services/llm/documentAnalyzer.js b/services/llm/documentAnalyzer.js new file mode 100644 index 00000000..e4d174ef --- /dev/null +++ b/services/llm/documentAnalyzer.js @@ -0,0 +1,346 @@ +'use strict'; + +/** + * The single business logic behind the LLM document analysis. + * + * Each provider is a composition root (services/llm/providers/): it hands + * this class its transport, its token budget, its prompt layout and the few + * knobs that really differ, and exports the instance as its singleton. + * Nothing extends DocumentAnalyzer. What happens between the document and + * the AnalysisResult — prompt, budget, request, parsing, the metadata check, + * the field defaults, the refusal — happens here, once, for every provider. + */ + +const path = require('path'); +const fs = require('fs').promises; +const config = require('../../config/config'); +const { writePromptToFile } = require('../serviceUtils'); +const { hasNumber } = require('../aiGenerateOptions'); +const { cacheThumbnail } = require('../thumbnailCache'); +const defaultPromptBuilder = require('./promptBuilder'); +const defaultResponseParser = require('./responseParser'); +const { + buildSuccess, + buildFailure, + buildInsufficientContent, + assertAssignableMetadata, + withDocumentDefaults, +} = require('./analysisResult'); + +/** Tokens the external API data may take up in the prompt. */ +const EXTERNAL_API_DATA_MAX_TOKENS = 500; + +class DocumentAnalyzer { + /** + * @param {Object} deps + * @param {string} deps.label - Provider name for the log + * @param {Object} deps.transport - One API family: initialize(), isReady(), + * complete(request), generate(prompt, options[, sizes]), and the + * `client` / `lastGenerateTextUsage` the tests and the AI review read + * @param {Object} deps.tokenBudget - reserve(), fit(), count(), cut() + * @param {Object} deps.promptLayout - frame() and build(); see promptLayout.js + * @param {() => string} deps.resolveModel - The model of this request + * @param {(provider: DocumentAnalyzer) => Promise} deps.checkStatus + * The provider's own status probe (row g: kept as it was) + * @param {string} deps.clientNotReadyMessage - The failure without a client + * @param {(model: string) => number|undefined} [deps.analysisTemperature] + * undefined leaves the parameter out of the request + * @param {Object|null} [deps.responseSchema] - Structured-output schema + * @param {(promptText: string, responseTokens: number) => number} [deps.contextWindow] + * The context to request for a prompt; set by the providers that size it + * themselves (Ollama's num_ctx) + * @param {(content: string) => string} [deps.prepareContent] - A cut the + * provider applies before anything else (Ollama's CONTENT_MAX_LENGTH) + * @param {(error: Error) => string} [deps.describeFailure] - The failure + * message, when the provider rewords one (Custom's timeouts, rows e/m) + * @param {Object} [deps.promptBuilder] - buildAnalysisPrompt() + * @param {Object} [deps.responseParser] - parseAnalysisResponse(), + * isInsufficientContentAnswer() + * @param {(id: string|number) => Promise} [deps.cacheThumbnail] + */ + constructor({ + label, + transport, + tokenBudget, + promptLayout, + resolveModel, + checkStatus, + clientNotReadyMessage, + analysisTemperature = () => config.aiTemperatureAnalysis, + responseSchema = null, + contextWindow = null, + prepareContent = (content) => content, + describeFailure = (error) => error.message, + promptBuilder = defaultPromptBuilder, + responseParser = defaultResponseParser, + cacheThumbnail: cacheThumbnailFn = cacheThumbnail, + }) { + this.label = label; + this.transport = transport; + this.tokenBudget = tokenBudget; + this.promptLayout = promptLayout; + this.resolveModel = resolveModel; + this.clientNotReadyMessage = clientNotReadyMessage; + this.analysisTemperature = analysisTemperature; + this.responseSchema = responseSchema; + this.contextWindow = contextWindow; + this.prepareContent = prepareContent; + this.describeFailure = describeFailure; + this.promptBuilder = promptBuilder; + this.responseParser = responseParser; + this.cacheThumbnail = cacheThumbnailFn; + this._checkStatus = checkStatus; + } + + /** The SDK or axios client. It lives on the transport; tests replace it here. */ + get client() { + return this.transport.client; + } + + set client(client) { + this.transport.client = client; + } + + /** Usage of the last generateText() call; read by the AI review features. */ + get lastGenerateTextUsage() { + return this.transport.lastGenerateTextUsage; + } + + set lastGenerateTextUsage(usage) { + this.transport.lastGenerateTextUsage = usage; + } + + initialize() { + this.transport.initialize(); + } + + /** + * Analyze one document. Never throws: a failure is returned as the record + * the OCR fallback matches on (analysisResult.js). + * + * @param {string} content - The document text + * @param {Array} [existingTags] + * @param {Array} [existingCorrespondentList] + * @param {Array} [existingDocumentTypesList] + * @param {string|number} id - Paperless document id + * @param {string|null} [customPrompt] - Replaces the instructions (webhook) + * @param {Object} [options] + * @param {Object|string} [options.externalApiData] - Context to append + * @returns {Promise} The AnalysisResult + */ + async analyzeDocument( + content, + existingTags = [], + existingCorrespondentList = [], + existingDocumentTypesList = [], + id, + customPrompt = null, + options = {} + ) { + try { + this.initialize(); + + if (!this.transport.isReady()) { + throw new Error(this.clientNotReadyMessage); + } + + content = this.prepareContent(content); + + // Cache the thumbnail for the UI; never let it affect the analysis. + await this.cacheThumbnail(id); + + const externalApiData = await this._externalApiData( + options.externalApiData + ); + + const model = this.resolveModel(); + const { systemPrompt: instructions, promptTags } = + this.promptBuilder.buildAnalysisPrompt({ + existingTags, + existingCorrespondentList, + existingDocumentTypesList, + externalApiData, + customPrompt, + }); + if (customPrompt) { + console.log('[DEBUG] Replace system prompt with custom prompt'); + } + + // Calculate tokens AFTER all prompt modifications are complete + const { availableTokens } = await this.tokenBudget.reserve( + this.promptLayout.frame(instructions, { customPrompt }), + process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [], + model + ); + console.log( + `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting` + ); + console.log( + `[DEBUG] External API data: ${externalApiData ? 'included' : 'none'}` + ); + + const fittedContent = await this.tokenBudget.fit( + content, + availableTokens, + model + ); + const { systemPrompt, userContent } = this.promptLayout.build( + instructions, + fittedContent, + { customPrompt } + ); + + await writePromptToFile(systemPrompt, userContent); + + const maxResponseTokens = Number(config.responseTokens); + const { payload, metrics } = await this.transport.complete({ + systemPrompt, + userContent, + model, + temperature: this.analysisTemperature(model), + maxResponseTokens, + schema: this.responseSchema, + numCtx: this.contextWindow + ? this.contextWindow(userContent, maxResponseTokens) + : undefined, + }); + + let parsed; + let normalized; + try { + ({ parsed, normalized } = + this.responseParser.parseAnalysisResponse(payload)); + } catch (error) { + console.error(`Failed to parse JSON response: ${error.message}`); + console.debug(error); + // A model that declined the document is not a broken answer: the + // placeholder keeps the retry loop from asking again, and the + // document still reaches OCR (Finding 7c). + if (this.responseParser.isInsufficientContentAnswer(payload)) { + console.warn(`Document ${id} has insufficient content for analysis`); + return buildInsufficientContent(id, metrics); + } + throw error; + } + + await this._logResponse(normalized); + + // On the parsed answer: the defaults would turn a missing `tags` into + // [] and let every answer pass (Finding 7k). + assertAssignableMetadata(parsed); + + return buildSuccess( + withDocumentDefaults(parsed), + metrics, + String(fittedContent).length < String(content).length + ); + } catch (error) { + const message = this.describeFailure(error); + console.error( + `Failed to analyze document with ${this.label}: ${message}` + ); + console.debug(error); + return buildFailure(error, message); + } + } + + /** + * The external API data as prompt text, cut to its token share; null when + * there is none or it cannot be read. + * + * @param {Object|string|undefined} apiData + * @returns {Promise} + */ + async _externalApiData(apiData) { + if (!apiData) { + return null; + } + try { + const data = await this._validateAndTruncateExternalApiData(apiData); + console.log('[DEBUG] External API data validated and included'); + return data; + } catch (error) { + console.warn( + '[WARNING] External API data validation failed:', + error.message + ); + return null; + } + } + + /** + * Validate and truncate external API data to prevent token overflow + * @param {any} apiData - The external API data to validate + * @param {number} maxTokens - Maximum tokens allowed for external data + * @returns {Promise} - Validated and potentially truncated data string + */ + async _validateAndTruncateExternalApiData( + apiData, + maxTokens = EXTERNAL_API_DATA_MAX_TOKENS + ) { + const dataString = + typeof apiData === 'object' + ? JSON.stringify(apiData, null, 2) + : String(apiData); + + const model = this.resolveModel(); + const dataTokens = await this.tokenBudget.count(dataString, model); + + if (dataTokens > maxTokens) { + console.warn( + `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating` + ); + return await this.tokenBudget.cut(dataString, maxTokens, model); + } + + console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`); + return dataString; + } + + /** Append the answer to data/logs/response.txt; best effort. */ + async _logResponse(normalized) { + const responseLogPath = path.join( + process.cwd(), + 'data', + 'logs', + 'response.txt' + ); + try { + await fs.mkdir(path.dirname(responseLogPath), { recursive: true }); + await fs.appendFile(responseLogPath, `${normalized}\n`); + } catch (logError) { + console.warn('Failed to write AI response log:', logError.message); + } + } + + /** + * Generate text based on a prompt. The request dialect, streaming and the + * usage seam live on the transport and its provider's buildRequest(); a + * provider that sizes its own context gets the sizes handed in. + * + * @param {string} prompt + * @param {object} [options] - As read by aiGenerateOptions: systemPrompt, + * temperature, maxTokens, model, signal, onProgress, reasoning + * @returns {Promise} The generated text + */ + async generateText(prompt, options = {}) { + if (!this.contextWindow) { + return this.transport.generate(prompt, options); + } + const responseTokens = + hasNumber(options.maxTokens) && options.maxTokens > 0 + ? Math.floor(options.maxTokens) + : Number(config.responseTokens); + return this.transport.generate(prompt, options, { + responseTokens, + numCtx: this.contextWindow(prompt, responseTokens), + }); + } + + /** The provider's status probe, unchanged (row g; decided with step 8). */ + checkStatus() { + return this._checkStatus(this); + } +} + +module.exports = { DocumentAnalyzer }; diff --git a/services/llm/promptBuilder.js b/services/llm/promptBuilder.js new file mode 100644 index 00000000..5c8059dd --- /dev/null +++ b/services/llm/promptBuilder.js @@ -0,0 +1,113 @@ +'use strict'; + +/** + * The analysis instructions, assembled once for every provider. + * + * openai, azure and custom send the result as their system message; Ollama + * puts it in front of the document in its prompt and keeps its own short + * system prompt. Either way the cascade below is the same, and it used to be + * copied into each of them. + */ + +const config = require('../../config/config'); +const RestrictionPromptService = require('../restrictionPromptService'); +const { toNameList } = require('../serviceUtils'); +const { buildCustomFieldsBlock } = require('./customFieldsTemplate'); + +// The layout of the two literal blocks below is what the providers have always +// sent, down to the whitespace. It is kept as it was so that moving the code +// here does not change a single prompt. +const BLOCK_INDENT = ' '; + +function preExistingDataBlock(tags, correspondents, documentTypes) { + return ( + `\n${BLOCK_INDENT}Pre-existing tags: ${tags}\n\n\n` + + `${BLOCK_INDENT}Pre-existing correspondents: ${correspondents}\n\n\n` + + `${BLOCK_INDENT}Pre-existing document types: ${documentTypes}\n\n\n` + + BLOCK_INDENT + ); +} + +const PREDEFINED_TAGS_INTRO = + `\n${BLOCK_INDENT}Take these tags and try to match one or more to the document content.\n\n\n` + + BLOCK_INDENT; + +/** + * Build the instructions for one document analysis. + * + * In order, each step overriding or extending the previous one: + * 1. SYSTEM_PROMPT plus mustHavePrompt with the custom fields block; with + * USE_EXISTING_DATA (and no restriction) the existing tags, + * correspondents and document types in front of it. + * 2. The %RESTRICTED_*% placeholders resolved. + * 3. The external API data appended. + * 4. With USE_PROMPT_TAGS, all of the above replaced by the predefined-tags + * prompt; PROMPT_TAGS is handed out as promptTags. + * 5. A custom prompt (webhook) replaces all of it, followed by mustHavePrompt. + * + * @param {Object} input + * @param {Array} [input.existingTags] + * @param {Array} [input.existingCorrespondentList] + * @param {Array} [input.existingDocumentTypesList] + * @param {string|null} [input.externalApiData] Already validated and truncated. + * @param {string|null} [input.customPrompt] + * @param {string} [input.customFieldsBlock] Defaults to the configured fields. + * @returns {{systemPrompt: string, promptTags: string}} + */ +function buildAnalysisPrompt({ + existingTags = [], + existingCorrespondentList = [], + existingDocumentTypesList = [], + externalApiData = null, + customPrompt = null, + customFieldsBlock = buildCustomFieldsBlock(), +} = {}) { + const mustHavePrompt = config.mustHavePrompt.replace( + '%CUSTOMFIELDS%', + customFieldsBlock + ); + let systemPrompt; + let promptTags = ''; + + if ( + config.useExistingData === 'yes' && + config.restrictToExistingTags === 'no' && + config.restrictToExistingCorrespondents === 'no' + ) { + systemPrompt = + preExistingDataBlock( + toNameList(existingTags).join(', '), + toNameList(existingCorrespondentList).join(', '), + toNameList(existingDocumentTypesList).join(', ') + ) + + process.env.SYSTEM_PROMPT + + '\n\n' + + mustHavePrompt; + } else { + systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt; + } + + systemPrompt = RestrictionPromptService.processRestrictionsInPrompt( + systemPrompt, + existingTags, + existingCorrespondentList, + existingDocumentTypesList + ); + + if (externalApiData) { + systemPrompt += `\n\nAdditional context from external API:\n${externalApiData}`; + } + + if (process.env.USE_PROMPT_TAGS === 'yes') { + promptTags = process.env.PROMPT_TAGS; + systemPrompt = PREDEFINED_TAGS_INTRO + config.specialPromptPreDefinedTags; + } + + if (customPrompt) { + systemPrompt = customPrompt + '\n\n' + mustHavePrompt; + } + + return { systemPrompt, promptTags }; +} + +module.exports = { buildAnalysisPrompt }; diff --git a/services/llm/promptLayout.js b/services/llm/promptLayout.js new file mode 100644 index 00000000..9dc6f29b --- /dev/null +++ b/services/llm/promptLayout.js @@ -0,0 +1,93 @@ +'use strict'; + +/** + * How the analysis instructions and the document are laid out into one + * request, per API family. + * + * promptBuilder.js builds the instructions once for every provider. The chat + * providers (openai, azure, custom) send them as the system message and the + * document as the user message. Ollama puts them in front of the document in + * its prompt and sends a short system prompt of its own. Both layouts are + * kept exactly as the providers always sent them, down to the whitespace. + * + * A layout has two parts: + * - frame(instructions, { customPrompt }): the fixed text the token budget + * reserves room for before the document is cut; + * - build(instructions, content, { customPrompt }): the request's + * { systemPrompt, userContent }. + */ + +const { buildCustomFieldsBlock } = require('./customFieldsTemplate'); + +/** openai, azure, custom: the instructions are the system message. */ +const chatPromptLayout = { + frame(instructions) { + return instructions; + }, + build(instructions, content) { + return { systemPrompt: instructions, userContent: content }; + }, +}; + +/** Ollama's own system prompt; %CUSTOMFIELDS% is the configured block. */ +const OLLAMA_SYSTEM_PROMPT_TEMPLATE = ` + You are a document analyzer. Your task is to analyze documents and extract relevant information. You do not ask back questions. + YOU MUSTNOT: Ask for additional information or clarification, or ask questions about the document, or ask for additional context. + YOU MUSTNOT: Return a response without the desired JSON format. + YOU MUST: Return the result EXCLUSIVELY as a JSON object. The Tags, Title and Document_Type MUST be in the language that is used in the document.: + IMPORTANT: The custom_fields are optional and can be left out if not needed, only try to fill out the values if you find a matching information in the document. + Do not change the value of field_name, only fill out the values. If the field is about money only add the number without currency and always use a . for decimal places. + { + "title": "xxxxx", + "correspondent": "xxxxxxxx", + "tags": ["Tag1", "Tag2", "Tag3", "Tag4"], + "document_type": "Invoice/Contract/...", + "document_date": "YYYY-MM-DD", + "language": "en/de/es/...", + %CUSTOMFIELDS% + } + ALWAYS USE THE INFORMATION TO FILL OUT THE JSON OBJECT. DO NOT ASK BACK QUESTIONS. + `; + +/** Ollama's system prompt with the configured custom fields. */ +function buildOllamaSystemPrompt(customFieldsBlock = buildCustomFieldsBlock()) { + return OLLAMA_SYSTEM_PROMPT_TEMPLATE.replace( + '%CUSTOMFIELDS%', + customFieldsBlock + ); +} + +/** The instructions, then the document as a JSON string. */ +function ollamaPrompt(instructions, content) { + return `${instructions} + ${JSON.stringify(content)} + `; +} + +/** + * Ollama: instructions and document both go into the prompt. A custom + * (webhook) prompt is followed by a blank line instead of the indented + * block, as it always was. + */ +const ollamaPromptLayout = { + frame(instructions, { customPrompt } = {}) { + return customPrompt + ? `${instructions}\n\n` + : ollamaPrompt(instructions, ''); + }, + build(instructions, content, { customPrompt } = {}) { + return { + systemPrompt: buildOllamaSystemPrompt(), + userContent: customPrompt + ? `${instructions}\n\n${JSON.stringify(content)}` + : ollamaPrompt(instructions, content), + }; + }, +}; + +module.exports = { + chatPromptLayout, + ollamaPromptLayout, + ollamaPrompt, + buildOllamaSystemPrompt, +}; diff --git a/services/llm/providers/azureProvider.js b/services/llm/providers/azureProvider.js new file mode 100644 index 00000000..860ffcfa --- /dev/null +++ b/services/llm/providers/azureProvider.js @@ -0,0 +1,172 @@ +'use strict'; + +/** + * The Azure OpenAI composition root: the DocumentAnalyzer wired to the chat + * completions transport and the tiktoken budget. What is Azure's own lives + * here — the client, the deployment as the model, the generateText() + * request dialect, the API-version gate for streamed usage and the + * deployment probe. services/azureService.js re-exports the singleton. + */ + +const axios = require('axios'); +const AzureOpenAI = require('openai').AzureOpenAI; +const config = require('../../../config/config'); +const { + hasNumber, + hasSystemPrompt, + modelOverride, + reasoningEffortForOpenAi, + reasoningEnabled, +} = require('../../aiGenerateOptions'); +const { DocumentAnalyzer } = require('../documentAnalyzer'); +const { + ChatCompletionsTransport, +} = require('../transports/chatCompletionsTransport'); +const tiktokenBudget = require('../tokenBudget/tiktokenBudget'); +const { chatPromptLayout } = require('../promptLayout'); + +/** What an operator can do about an answer that hit the completion cap. */ +const TRUNCATION_REMEDY = 'Raise Response Tokens (RESPONSE_TOKENS).'; +/** + * The first Azure API version that answers a streamed request with a usage + * chunk. Older versions reject `stream_options` outright, which would turn a + * working deployment into a 400 the moment progress is asked for. + */ +const STREAM_OPTIONS_SINCE = '2024-06-01'; +/** Version aliases that always mean "the newest surface". */ +const ROLLING_API_VERSIONS = ['preview', 'latest', 'v1']; + +/** + * Whether the configured API version understands + * `stream_options: { include_usage: true }`. + * + * Azure versions are dates, so the comparison is a string comparison on the + * leading YYYY-MM-DD; a preview of a supported date counts as supported. + * Anything unrecognized is treated as too old and the token count for that + * deployment stays an estimate, which is the harmless half of being wrong. + * + * @param {string} apiVersion + * @returns {boolean} + */ +function supportsStreamOptions(apiVersion) { + const version = String(apiVersion || '') + .trim() + .toLowerCase(); + if (ROLLING_API_VERSIONS.includes(version)) return true; + const date = /^(\d{4}-\d{2}-\d{2})/.exec(version); + return date ? date[1] >= STREAM_OPTIONS_SINCE : false; +} + +function createClient() { + return config.aiProvider === 'azure' + ? new AzureOpenAI({ + apiKey: config.azure.apiKey, + endpoint: config.azure.endpoint, + deploymentName: config.azure.deploymentName, + apiVersion: config.azure.apiVersion, + }) + : null; +} + +/** + * The generateText() request. The transport sends it, streams it and reads + * the answer; what differs between the providers is here. + * + * @param {string} prompt + * @param {object} options - see DocumentAnalyzer.generateText() + * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS + * @param {boolean} [options.reasoning] - false sends reasoning_effort on the + * deployments that take it and nothing on the others, which have no + * thinking switch; true and null send nothing + * @returns {Object} The chat completions request + */ +function buildGenerationRequest(prompt, options) { + const model = modelOverride(options) || process.env.AZURE_DEPLOYMENT_NAME; + const messages = []; + if (hasSystemPrompt(options)) { + messages.push({ role: 'system', content: options.systemPrompt }); + } + messages.push({ role: 'user', content: prompt }); + + const request = { + model: model, + messages, + temperature: hasNumber(options.temperature) + ? options.temperature + : config.aiTemperatureGeneration, + // Was a hardcoded 1000, which quarrelled with the setting the same way + // Ollama's num_predict did — the reservation followed the operator, + // the limit did not. + max_tokens: + hasNumber(options.maxTokens) && options.maxTokens > 0 + ? Math.floor(options.maxTokens) + : Number(config.responseTokens), + }; + // As on OpenAI: the reasoning deployments take reasoning_effort, every + // other one answers 400 for it, so it is sent for those names only and + // only when the caller asked for no thinking. + if (reasoningEnabled(options) === false) { + const effort = reasoningEffortForOpenAi(model); + if (effort) request.reasoning_effort = effort; + } + + return request; +} + +async function checkStatus(provider) { + try { + provider.initialize(); + + if (!provider.client) { + throw new Error('Azure OpenAI client not initialized - missing API key'); + } + + const model = process.env.AZURE_DEPLOYMENT_NAME; + + // Token-free health check against deployment metadata endpoint. + const endpoint = (config.azure.endpoint || '').replace(/\/$/, ''); + const apiVersion = config.azure.apiVersion; + if (!endpoint || !apiVersion || !model) { + throw new Error('Azure endpoint, apiVersion, or deployment name missing'); + } + + const url = `${endpoint}/openai/deployments/${model}?api-version=${encodeURIComponent(apiVersion)}`; + const response = await axios.get(url, { + headers: { + 'api-key': config.azure.apiKey, + }, + timeout: 10000, + }); + + if (response.status < 200 || response.status >= 300) { + return { status: 'error' }; + } + + return { status: 'ok', model: model }; + } catch (error) { + console.error(`Error generating text with Azure OpenAI: ${error.message}`); + console.debug(error); + return { status: 'error', error: error.message }; + } +} + +module.exports = new DocumentAnalyzer({ + label: 'AzureOpenAI', + transport: new ChatCompletionsTransport({ + label: 'AzureOpenAI', + createClient, + generation: { + notReadyMessage: 'AzureOpenAI client not initialized - missing API key', + remedy: TRUNCATION_REMEDY, + buildRequest: buildGenerationRequest, + // Only where the configured API version takes it; an older one would + // answer 400 instead of streaming, so that deployment estimates. + streamUsage: () => supportsStreamOptions(config.azure.apiVersion), + }, + }), + tokenBudget: tiktokenBudget, + promptLayout: chatPromptLayout, + resolveModel: () => process.env.AZURE_DEPLOYMENT_NAME, + checkStatus, + clientNotReadyMessage: 'AzureOpenAI client not initialized', +}); diff --git a/services/llm/providers/customProvider.js b/services/llm/providers/customProvider.js new file mode 100644 index 00000000..81b488d0 --- /dev/null +++ b/services/llm/providers/customProvider.js @@ -0,0 +1,216 @@ +'use strict'; + +/** + * The composition root for any OpenAI-compatible server behind a custom base + * URL: the DocumentAnalyzer wired to the chat completions transport and the + * tiktoken budget. What is Custom's own lives here — the client, the + * detailed "Invalid API response structure" messages, the generateText() + * request with its context clamp and three reasoning dialects, the timeout + * wording (rows e/m, kept as they are) and the status probe. + * services/customService.js re-exports the singleton. + */ + +const OpenAI = require('openai'); +const config = require('../../../config/config'); +const { + calculateTokens, + isTimeoutError, + buildTimeoutErrorMessage, +} = require('../../serviceUtils'); +const { + hasNumber, + hasSystemPrompt, + modelOverride, + reasoningEffortForCompatible, + reasoningEnabled, +} = require('../../aiGenerateOptions'); +const { DocumentAnalyzer } = require('../documentAnalyzer'); +const { + ChatCompletionsTransport, +} = require('../transports/chatCompletionsTransport'); +const tiktokenBudget = require('../tokenBudget/tiktokenBudget'); +const { chatPromptLayout } = require('../promptLayout'); + +const CUSTOM_PROVIDER_FALLBACK_API_KEY = 'no-auth-required'; +/** What an operator can do about an answer that hit the completion cap. */ +const TRUNCATION_REMEDY = 'Raise Response Tokens (RESPONSE_TOKENS).'; +/** + * The soft switch the Qwen family reads out of the prompt itself. + * + * `chat_template_kwargs` only reaches a server that applies the model's chat + * template; a gateway that forwards the messages and drops the rest leaves the + * model thinking anyway. Qwen understands the same wish written into the last + * line of the user message, so both are sent and whichever arrives wins. + */ +const QWEN_NO_THINK_SWITCH = '/no_think'; +/** Models that understand the line above. */ +const QWEN_MODEL_PATTERN = /qwen/i; + +function createClient() { + return config.aiProvider === 'custom' + ? new OpenAI({ + baseURL: config.custom.apiUrl, + // OpenAI-compatible SDKs require an apiKey value even when auth is disabled. + apiKey: config.custom.apiKey || CUSTOM_PROVIDER_FALLBACK_API_KEY, + }) + : null; +} + +/** + * Every message keeps the "Invalid API response structure" prefix: + * shouldQueueForOcrOnAiError() matches on that phrase, and without it + * the document no longer reaches the OCR fallback. + */ +function describeMissingContent(response, message) { + const missing = !response + ? 'the response is empty' + : !response.choices + ? 'missing choices' + : !response.choices[0] + ? 'missing choices[0]' + : !response.choices[0].message + ? 'missing choices[0].message' + : !response.choices[0].message.content + ? 'missing choices[0].message.content' + : `unusable content ${message?.content}`; + return `Invalid API response structure: ${missing}.`; +} + +/** + * Say what came back: a gateway that timed out behind a 200, a + * model that stopped after its reasoning without an answer, or an + * empty choices array all end here and need telling apart. + */ +function describeEmptyText(response) { + const finishReason = response?.choices?.[0]?.finish_reason ?? 'unknown'; + const head = JSON.stringify(response ?? null).slice(0, 200); + return `Invalid API response structure: the answer carried no text (finish_reason ${finishReason}; the response begins ${head})`; +} + +/** + * The generateText() request. The transport sends it, streams it and reads + * the answer; what differs between the providers is here. + * + * @param {string} prompt + * @param {object} options - see DocumentAnalyzer.generateText() + * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS, still clamped + * @param {boolean} [options.reasoning] - Switches the model's thinking off + * or on for this one call; nothing new is sent when the caller says + * nothing. Off also sends reasoning_effort where the model family takes it + * @returns {Promise} The chat completions request + */ +async function buildGenerationRequest(prompt, options) { + const model = modelOverride(options) || config.custom.model; + const maxContextTokens = Number(config.tokenLimit) || 128000; + const desiredCompletionTokens = + hasNumber(options.maxTokens) && options.maxTokens > 0 + ? Math.floor(options.maxTokens) + : Number(config.responseTokens) || 1000; + const promptTokens = await calculateTokens(prompt, model); + const availableCompletionTokens = Math.max( + 1, + maxContextTokens - promptTokens - 64 + ); + const maxCompletionTokens = Math.max( + 1, + Math.min(desiredCompletionTokens, availableCompletionTokens) + ); + + if (maxCompletionTokens < desiredCompletionTokens) { + console.log( + `[DEBUG] Clamped max_tokens for custom generateText from ${desiredCompletionTokens} to ${maxCompletionTokens} ` + + `(context limit: ${maxContextTokens}, prompt tokens: ${promptTokens})` + ); + } + + const reasoning = reasoningEnabled(options); + const messages = []; + if (hasSystemPrompt(options)) { + messages.push({ role: 'system', content: options.systemPrompt }); + } + messages.push({ + role: 'user', + content: + reasoning === false && QWEN_MODEL_PATTERN.test(model) + ? `${prompt}\n${QWEN_NO_THINK_SWITCH}` + : prompt, + }); + + const request = { + model: model, + messages, + temperature: hasNumber(options.temperature) + ? options.temperature + : config.aiTemperatureGeneration, + max_tokens: maxCompletionTokens, + }; + // The key every OpenAI-compatible server that applies a chat template + // understands; one that does not know it passes it through and the + // model decides for itself, as it did before. + if (reasoning !== null) { + request.chat_template_kwargs = { enable_thinking: reasoning }; + } + // A hosted reasoning model reads neither of the two above and thinks + // anyway; the families that take reasoning_effort are told in their own + // dialect. Only when the caller asked for no thinking, and only for + // those names — another model answers 400 for the field. + if (reasoning === false) { + const effort = reasoningEffortForCompatible(model); + if (effort) request.reasoning_effort = effort; + } + + return request; +} + +/** Only Custom words a timeout as such (rows e/m, kept as they are). */ +function describeFailure(error) { + if (!isTimeoutError(error)) { + return error.message; + } + console.error( + `[TIMEOUT][AI] Custom provider request timed out: ${error.message}` + ); + return buildTimeoutErrorMessage('AI'); +} + +async function checkStatus(provider) { + try { + provider.initialize(); + + if (!provider.client) { + throw new Error('Custom OpenAI client not initialized - missing API key'); + } + + const model = config.custom.model; + + // Use token-free endpoint where supported by OpenAI-compatible providers. + await provider.client.models.list(); + + return { status: 'ok', model: model }; + } catch (error) { + console.error(`Error generating text with Custom OpenAI: ${error.message}`); + console.debug(error); + return { status: 'error', error: error.message }; + } +} + +module.exports = new DocumentAnalyzer({ + label: 'Custom OpenAI', + transport: new ChatCompletionsTransport({ + label: 'Custom OpenAI', + createClient, + describeMissingContent, + generation: { + notReadyMessage: 'Custom OpenAI client not initialized - missing API key', + remedy: TRUNCATION_REMEDY, + buildRequest: buildGenerationRequest, + describeEmptyText, + }, + }), + tokenBudget: tiktokenBudget, + promptLayout: chatPromptLayout, + resolveModel: () => config.custom.model, + describeFailure, + checkStatus, + clientNotReadyMessage: 'Custom OpenAI client not initialized', +}); diff --git a/services/llm/providers/ollamaProvider.js b/services/llm/providers/ollamaProvider.js new file mode 100644 index 00000000..4002c47c --- /dev/null +++ b/services/llm/providers/ollamaProvider.js @@ -0,0 +1,246 @@ +'use strict'; + +/** + * The Ollama composition root: the DocumentAnalyzer wired to the + * /api/generate transport, the character budget and Ollama's prompt layout. + * What is Ollama's own lives here — the axios client, the output schema, the + * num_ctx sizing, the CONTENT_MAX_LENGTH cut and the /api/ps status probe. + * services/ollamaService.js re-exports the singleton. + * + * The singleton keeps the seams the tests call or stub: apiUrl, model, + * documentAnalysisSchema, _buildPrompt(), _callOllamaAPI(), + * _calculatePromptTokenCount(), _calculateNumCtx(), _extractOllamaMetrics(). + * The context is sized through _calculatePromptTokenCount() and + * _calculateNumCtx(), so a stub there reaches the request. + */ + +const axios = require('axios'); +const config = require('../../../config/config'); +const { DocumentAnalyzer } = require('../documentAnalyzer'); +const { + OllamaGenerateTransport, +} = require('../transports/ollamaGenerateTransport'); +const charBudget = require('../tokenBudget/charBudget'); +const { buildAnalysisPrompt } = require('../promptBuilder'); +const { ollamaPromptLayout, ollamaPrompt } = require('../promptLayout'); + +/** The output schema Ollama is held to (`format`). */ +const DOCUMENT_ANALYSIS_SCHEMA = { + type: 'object', + properties: { + title: { type: 'string' }, + correspondent: { type: 'string' }, + tags: { + type: 'array', + items: { type: 'string' }, + }, + document_type: { type: 'string' }, + document_date: { type: 'string' }, + language: { type: 'string' }, + custom_fields: { + type: 'object', + additionalProperties: true, + }, + }, + required: [ + 'title', + 'correspondent', + 'tags', + 'document_type', + 'document_date', + 'language', + ], +}; + +/** Tokens the external API data may take up in the prompt. */ +const EXTERNAL_API_DATA_MAX_TOKENS = 500; + +/** CONTENT_MAX_LENGTH, Ollama's own cut before the budget (kept as is). */ +function truncateContent(content) { + try { + if (process.env.CONTENT_MAX_LENGTH) { + console.log( + 'Truncating content to max length:', + process.env.CONTENT_MAX_LENGTH + ); + return content.substring(0, process.env.CONTENT_MAX_LENGTH); + } + } catch (error) { + console.error(`Error truncating content: ${error.message}`); + console.debug(error); + } + return content; +} + +/** The external API data as prompt text, cut the character-budget way. */ +function externalApiDataForPrompt(apiData) { + const dataString = + typeof apiData === 'object' + ? JSON.stringify(apiData, null, 2) + : String(apiData); + return charBudget.count(dataString) > EXTERNAL_API_DATA_MAX_TOKENS + ? charBudget.cut(dataString, EXTERNAL_API_DATA_MAX_TOKENS) + : dataString; +} + +async function checkStatus(provider) { + // use ollama status endpoint + try { + const response = await provider.client.get(`${provider.apiUrl}/api/ps`, { + headers: provider._buildRequestHeaders(), + }); + if (response.status === 200) { + const data = response.data; + // Ensure data is an array and has at least one model + let modelName = null; + if (Array.isArray(data.models) && data.models.length > 0) { + modelName = data.models[0].name; + } + console.log('Ollama model name:', modelName); + return { status: 'ok', model: modelName }; + } + } catch (error) { + console.error(`Error checking Ollama service status: ${error.message}`); + console.debug(error); + } + return { status: 'error' }; +} + +const transport = new OllamaGenerateTransport({ + client: axios.create({ + timeout: 1800000, // 30 minutes timeout + }), + apiUrl: config.ollama.apiUrl, + model: config.ollama.model, +}); + +const ollamaProvider = new DocumentAnalyzer({ + label: 'Ollama', + transport, + tokenBudget: charBudget, + promptLayout: ollamaPromptLayout, + resolveModel: () => transport.model, + responseSchema: DOCUMENT_ANALYSIS_SCHEMA, + contextWindow: (promptText, responseTokens) => + ollamaProvider._calculateNumCtx( + ollamaProvider._calculatePromptTokenCount(promptText), + responseTokens + ), + prepareContent: truncateContent, + checkStatus, + clientNotReadyMessage: 'Ollama client not initialized', +}); + +Object.defineProperties(ollamaProvider, { + apiUrl: { + get() { + return this.transport.apiUrl; + }, + set(apiUrl) { + this.transport.apiUrl = apiUrl; + }, + enumerable: true, + }, + model: { + get() { + return this.transport.model; + }, + set(model) { + this.transport.model = model; + }, + enumerable: true, + }, + documentAnalysisSchema: { + get() { + return this.responseSchema; + }, + set(schema) { + this.responseSchema = schema; + }, + enumerable: true, + }, +}); + +Object.assign(ollamaProvider, { + /** + * The analysis prompt for a document: the instructions, then the document. + * @param {string} content - The document text + * @param {string[]} [existingTags] + * @param {string[]} [existingCorrespondent] + * @param {string[]} [existingDocumentTypes] + * @param {Object} [options] - May carry externalApiData + * @returns {string} + */ + _buildPrompt( + content, + existingTags = [], + existingCorrespondent = [], + existingDocumentTypes = [], + options = {} + ) { + const { systemPrompt } = buildAnalysisPrompt({ + existingTags, + existingCorrespondentList: Array.isArray(existingCorrespondent) + ? existingCorrespondent + : [], + existingDocumentTypesList: existingDocumentTypes, + externalApiData: options.externalApiData + ? externalApiDataForPrompt(options.externalApiData) + : null, + }); + return ollamaPrompt(systemPrompt, content); + }, + + /** Estimated tokens of a prompt, two characters each. */ + _calculatePromptTokenCount(prompt) { + return charBudget.estimateTokens(prompt); + }, + + /** The context window to request: prompt plus answer, at most TOKEN_LIMIT. */ + _calculateNumCtx(promptTokenCount, expectedResponseTokens) { + const numCtx = charBudget.numCtx(promptTokenCount, expectedResponseTokens); + + console.log('Prompt Token Count:', promptTokenCount); + console.log('Expected Response Tokens:', expectedResponseTokens); + console.log('Dynamic calculated num_ctx:', numCtx); + + return numCtx; + }, + + /** Request headers, with bearer auth when an API key is set. */ + _buildRequestHeaders() { + return this.transport.buildHeaders(); + }, + + /** + * Send one analysis request as it is. + * @param {string} prompt - Prompt text + * @param {string} systemPrompt - System prompt + * @param {number} numCtx - Context window size + * @param {Object} schema - Response schema + * @returns {Promise} The raw /api/generate response data + */ + async _callOllamaAPI(prompt, systemPrompt, numCtx, schema) { + const { raw } = await this.transport.complete({ + systemPrompt, + userContent: prompt, + temperature: config.aiTemperatureAnalysis, + maxResponseTokens: Number(config.responseTokens), + numCtx, + schema, + }); + return raw; + }, + + /** See OllamaGenerateTransport.assertNotTruncated(). */ + _assertNotTruncated(responseData, numPredict, numCtx, extra) { + this.transport.assertNotTruncated(responseData, numPredict, numCtx, extra); + }, + + /** Token usage of an /api/generate response. */ + _extractOllamaMetrics(responseData) { + return this.transport.extractMetrics(responseData); + }, +}); + +module.exports = ollamaProvider; diff --git a/services/llm/providers/openaiProvider.js b/services/llm/providers/openaiProvider.js new file mode 100644 index 00000000..1168423d --- /dev/null +++ b/services/llm/providers/openaiProvider.js @@ -0,0 +1,126 @@ +'use strict'; + +/** + * The OpenAI composition root: the DocumentAnalyzer wired to the chat + * completions transport and the tiktoken budget. What is OpenAI's own lives + * here — the client, the model, the generateText() request dialect and the + * status probe. services/openaiService.js re-exports the singleton. + */ + +const OpenAI = require('openai'); +const config = require('../../../config/config'); +const { + hasNumber, + hasSystemPrompt, + modelOverride, + reasoningEffortForOpenAi, + reasoningEnabled, +} = require('../../aiGenerateOptions'); +const { DocumentAnalyzer } = require('../documentAnalyzer'); +const { + ChatCompletionsTransport, +} = require('../transports/chatCompletionsTransport'); +const tiktokenBudget = require('../tokenBudget/tiktokenBudget'); +const { chatPromptLayout } = require('../promptLayout'); + +/** What an operator can do about an answer that hit the completion cap. */ +const TRUNCATION_REMEDY = + 'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.'; + +/** + * The factory hands this service out for the openai provider only (and as + * its default for an unknown one), so there is no client for any other. + */ +function createClient() { + if (config.aiProvider === 'openai' && config.openai.apiKey) { + return new OpenAI({ + apiKey: config.openai.apiKey, + }); + } + return null; +} + +/** + * The generateText() request. The transport sends it, streams it and reads + * the answer; what differs between the providers is here. + * + * @param {string} prompt + * @param {object} options - see DocumentAnalyzer.generateText() + * @param {number} [options.maxTokens] - Completion cap for this call; without + * one the model answers with whatever the deployment allows + * @param {boolean} [options.reasoning] - false sends reasoning_effort on the + * model families that take it and nothing on the others, which have no + * thinking switch; true and null send nothing + * @returns {Object} The chat completions request + */ +function buildGenerationRequest(prompt, options) { + const model = + modelOverride(options) || process.env.OPENAI_MODEL || config.openai.model; + const messages = []; + if (hasSystemPrompt(options)) { + messages.push({ role: 'system', content: options.systemPrompt }); + } + messages.push({ role: 'user', content: prompt }); + + const request = { + model: model, + messages, + temperature: hasNumber(options.temperature) + ? options.temperature + : config.aiTemperatureGeneration, + }; + // Without a cap the model answers with whatever the deployment allows, + // which is what every caller before the AI review wanted. + if (hasNumber(options.maxTokens) && options.maxTokens > 0) { + request.max_tokens = Math.floor(options.maxTokens); + } + // The one thinking switch this API has, and it exists on the reasoning + // families only: an ordinary model answers 400 for the field, so it is + // sent for those names and for no others, and only when the caller + // asked for no thinking. + if (reasoningEnabled(options) === false) { + const effort = reasoningEffortForOpenAi(model); + if (effort) request.reasoning_effort = effort; + } + + return request; +} + +async function checkStatus(provider) { + // Use a token-free endpoint for connectivity/auth checks. + try { + provider.initialize(); + + if (!provider.client) { + throw new Error('OpenAI client not initialized - missing API key'); + } + + await provider.client.models.list(); + return { status: 'ok', model: process.env.OPENAI_MODEL }; + } catch (error) { + console.error(`Error checking OpenAI status: ${error.message}`); + console.debug(error); + return { status: 'error', error: error.message }; + } +} + +module.exports = new DocumentAnalyzer({ + label: 'OpenAI', + transport: new ChatCompletionsTransport({ + label: 'OpenAI', + createClient, + generation: { + notReadyMessage: 'OpenAI client not initialized - missing API key', + remedy: TRUNCATION_REMEDY, + buildRequest: buildGenerationRequest, + }, + }), + tokenBudget: tiktokenBudget, + promptLayout: chatPromptLayout, + resolveModel: () => process.env.OPENAI_MODEL, + // o3-mini rejects the parameter. + analysisTemperature: (model) => + model !== 'o3-mini' ? config.aiTemperatureAnalysis : undefined, + checkStatus, + clientNotReadyMessage: 'OpenAI client not initialized', +}); diff --git a/services/llm/responseParser.js b/services/llm/responseParser.js new file mode 100644 index 00000000..72fcc647 --- /dev/null +++ b/services/llm/responseParser.js @@ -0,0 +1,189 @@ +'use strict'; + +/** + * Reads the JSON object out of a document analysis answer, once for every + * provider. + * + * Models wrap their answer: reasoning models put a block in front, + * many fence the JSON in ```json, some add a sentence before or after it, and + * smaller local models leave trailing commas or unquoted keys behind. Each + * provider used to cope with a different subset of that. This is the union: + * + * 1. an answer that already is an object (Ollama's structured output) is + * used as it is; + * 2. reasoning blocks and code fences are stripped; + * 3. the rest is parsed as it stands; + * 4. failing that, every top-level {...} in it is tried in turn, as written + * and then repaired, and the first one that parses wins. + * + * Every failure is an Error whose message starts with "Invalid JSON response + * from API" — shouldQueueForOcrOnAiError() matches on that phrase, and + * without it the document no longer reaches the OCR fallback. + */ + +const INVALID_JSON = 'Invalid JSON response from API'; + +/** + * What a model writes when it declines a document as too thin to analyse. + * Matched only on an answer that did not parse (Finding 7c). + */ +const INSUFFICIENT_CONTENT_MARKERS = ["i'm sorry", 'i cannot', 'insufficient']; + +/** Drop reasoning blocks and code fences around the answer. */ +function stripResponseWrapping(text) { + return String(text ?? '') + .replace(/[\s\S]*?<\/think>/g, '') + .replace(/```json\n?/gi, '') + .replace(/```\n?/g, '') + .trim(); +} + +/** + * Whether an answer that did not parse is a refusal rather than broken JSON. + * Such a document is reported as insufficient content: it still reaches OCR, + * under that reason, and the retry loop does not ask the model again. + * + * @param {string} raw - The model's answer + * @returns {boolean} + */ +function isInsufficientContentAnswer(raw) { + const text = stripResponseWrapping(raw).toLowerCase(); + return ( + text !== '' && + INSUFFICIENT_CONTENT_MARKERS.some((marker) => text.includes(marker)) + ); +} + +/** + * The end index of the balanced {...} that opens at `start`, or -1 when the + * text ends first. Braces inside string literals do not count. + */ +function findObjectEnd(text, start) { + let depth = 0; + let inString = false; + let escaping = false; + + for (let i = start; i < text.length; i += 1) { + const char = text[i]; + + if (inString) { + if (escaping) { + escaping = false; + } else if (char === '\\') { + escaping = true; + } else if (char === '"') { + inString = false; + } + continue; + } + + if (char === '"') { + inString = true; + } else if (char === '{') { + depth += 1; + } else if (char === '}') { + depth -= 1; + if (depth === 0) { + return i; + } + } + } + + return -1; +} + +/** + * Every top-level {...} in the text, left to right. An object that does not + * parse is skipped as a whole: what is nested inside it is part of a broken + * answer, not an answer of its own. + */ +function* topLevelObjects(text) { + let start = text.indexOf('{'); + while (start !== -1) { + const end = findObjectEnd(text, start); + if (end === -1) { + return; + } + yield text.slice(start, end + 1); + start = text.indexOf('{', end + 1); + } +} + +/** The repairs Ollama has always applied to an answer that would not parse. */ +function repairJson(jsonStr) { + return jsonStr + .replace(/,\s*}/g, '}') // Remove trailing commas before closing braces + .replace(/,\s*]/g, ']') // Remove trailing commas before closing brackets + .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":'); // Ensure property names are quoted +} + +function isPlainObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** The parsed object, or undefined when the text is not one. */ +function tryParseObject(text) { + try { + const parsed = JSON.parse(text); + return isPlainObject(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +/** + * Parse a document analysis answer. + * + * @param {string|object} raw - The model's answer + * @returns {{ parsed: object, normalized: string }} The answer object, and + * the JSON it was read from (what goes to the response log) + * @throws {Error} "Invalid JSON response from API: …" when no object can be + * read; the first parse error is attached as `cause` + */ +function parseAnalysisResponse(raw) { + if (isPlainObject(raw)) { + return { parsed: raw, normalized: JSON.stringify(raw) }; + } + + const text = stripResponseWrapping(raw); + if (!text) { + throw new Error(`${INVALID_JSON}: the answer is empty`); + } + + let firstError; + try { + const parsed = JSON.parse(text); + if (isPlainObject(parsed)) { + return { parsed, normalized: text }; + } + } catch (error) { + firstError = error; + } + + let sawObject = false; + for (const candidate of topLevelObjects(text)) { + sawObject = true; + for (const attempt of [candidate, repairJson(candidate)]) { + const parsed = tryParseObject(attempt); + if (parsed) { + return { parsed, normalized: attempt }; + } + } + } + + if (!sawObject) { + throw new Error(`${INVALID_JSON}: no JSON object in the answer`, { + cause: firstError, + }); + } + throw new Error( + `${INVALID_JSON}: unreadable even after repair${firstError ? `: ${firstError.message}` : ''}`, + { cause: firstError } + ); +} + +module.exports = { + parseAnalysisResponse, + stripResponseWrapping, + isInsufficientContentAnswer, +}; diff --git a/services/llm/tokenBudget/charBudget.js b/services/llm/tokenBudget/charBudget.js new file mode 100644 index 00000000..edd441da --- /dev/null +++ b/services/llm/tokenBudget/charBudget.js @@ -0,0 +1,111 @@ +'use strict'; + +/** + * Ollama's token budget, counted in characters. + * + * Two characters make one token. That is deliberately conservative: German + * and CJK text tokenizes at roughly two characters per token rather than the + * ~4 of English, and a ÷4 estimate under-sizes num_ctx for exactly those + * documents. Everything here uses the same divisor, so the prompt estimate, + * the context window and the document cut agree by construction (Finding 5 + * of docs/llm-service-refactoring.md — do not unify it with tiktoken's). + * + * Shares its reserve()/fit()/count()/cut() shape with tiktokenBudget.js. + */ + +const config = require('../../../config/config'); + +const CHARS_PER_TOKEN = 2; + +/** Estimated tokens for a text. */ +function estimateTokens(text) { + return Math.ceil(String(text).length / CHARS_PER_TOKEN); +} + +/** + * The context window to request: the prompt plus the reserved answer, but + * never more than TOKEN_LIMIT. + * + * @param {number} promptTokens - From estimateTokens() + * @param {number} responseTokens - Tokens reserved for the answer + * @returns {number} + */ +function numCtx(promptTokens, responseTokens) { + return Math.min(promptTokens + responseTokens, Number(config.tokenLimit)); +} + +/** + * Reserve room for the fixed prompt parts and the answer. + * + * A TOKEN_LIMIT or RESPONSE_TOKENS that is not a number leaves the document + * uncut (availableTokens is Infinity), as Ollama always did; it is the + * context window that is then wrong, not the document. + * + * @param {string} promptPrefix - The fully built prompt without the document + * @returns {Promise<{totalPromptTokens:number, reservedTokens:number, availableTokens:number}>} + * @throws {Error} "Token limit exceeded: …" when nothing is left for the document + */ +async function reserve(promptPrefix) { + const maxWindow = Number(config.tokenLimit); + const responseTokens = Number(config.responseTokens); + const totalPromptTokens = estimateTokens(promptPrefix); + const reservedTokens = totalPromptTokens + responseTokens; + + if (!Number.isFinite(maxWindow) || !Number.isFinite(responseTokens)) { + return { totalPromptTokens, reservedTokens, availableTokens: Infinity }; + } + + const availableTokens = maxWindow - reservedTokens; + if (availableTokens <= 0) { + throw new Error( + 'Token limit exceeded: the prompt alone does not fit the configured ' + + 'TOKEN_LIMIT. Raise TOKEN_LIMIT or shorten the system prompt.' + ); + } + return { totalPromptTokens, reservedTokens, availableTokens }; +} + +/** + * Cut the document to the tokens reserve() left for it. + * + * A document that is not a string is measured as its JSON. One that fits is + * returned as it was handed in; one that does not comes back as a string. + * + * @param {string|Object} content - Document content + * @param {number} availableTokens - From reserve() + * @returns {Promise} + */ +async function fit(content, availableTokens) { + const text = typeof content === 'string' ? content : JSON.stringify(content); + const maxChars = availableTokens * CHARS_PER_TOKEN; + if (text.length <= maxChars) { + return content; + } + + console.log( + `[DEBUG] Document content (${text.length} chars) exceeds the context ` + + `budget (${maxChars} chars for a ${config.tokenLimit}-token window minus prompt` + + ` and ${config.responseTokens} reserved answer tokens); truncating` + ); + return text.substring(0, maxChars); +} + +/** + * Cut a text to a token count, e.g. the external API data. + * + * @param {string} text + * @param {number} maxTokens + * @returns {string} + */ +function cut(text, maxTokens) { + return String(text).substring(0, maxTokens * CHARS_PER_TOKEN); +} + +module.exports = { + estimateTokens, + count: estimateTokens, + cut, + numCtx, + reserve, + fit, +}; diff --git a/services/llm/tokenBudget/tiktokenBudget.js b/services/llm/tokenBudget/tiktokenBudget.js new file mode 100644 index 00000000..2549f6bb --- /dev/null +++ b/services/llm/tokenBudget/tiktokenBudget.js @@ -0,0 +1,81 @@ +'use strict'; + +/** + * The token budget of the chat providers (openai, azure, custom). + * + * The prompt is counted with tiktoken (a ÷4 estimate for models tiktoken does + * not know), the configured answer is reserved, and the document gets what is + * left of TOKEN_LIMIT. This block used to be copied into all three services. + * + * Shares its reserve()/fit()/count()/cut() shape with charBudget.js, which is + * Ollama's. + */ + +const config = require('../../../config/config'); +const { + calculateTokens, + calculateTotalPromptTokens, + truncateToTokenLimit, +} = require('../../serviceUtils'); + +/** + * Reserve room for the prompt and the answer. + * + * @param {string} systemPrompt - The fully built system prompt + * @param {string[]} extraPrompts - Further prompt parts counted as messages + * @param {string} model - Model the prompt is counted for + * @returns {Promise<{totalPromptTokens:number, reservedTokens:number, availableTokens:number}>} + * @throws {Error} "Token limit exceeded: …" when nothing is left for the document + */ +async function reserve(systemPrompt, extraPrompts, model) { + const totalPromptTokens = await calculateTotalPromptTokens( + systemPrompt, + extraPrompts, + model + ); + + const maxTokens = Number(config.tokenLimit); + const reservedTokens = totalPromptTokens + Number(config.responseTokens); + const availableTokens = maxTokens - reservedTokens; + + // A non-finite budget is the same failure as an exhausted one: it slips + // past a bare `<= 0` and truncates the document down to nothing instead of + // stopping. + if (!Number.isFinite(availableTokens) || availableTokens <= 0) { + console.warn( + `[WARNING] No available tokens for content. Reserved: ${reservedTokens}, Max: ${maxTokens}` + ); + throw new Error( + 'Token limit exceeded: prompt too large for available token limit' + ); + } + + console.log( + `[DEBUG] Token calculation - Prompt: ${totalPromptTokens}, Reserved: ${reservedTokens}, Available: ${availableTokens}` + ); + return { totalPromptTokens, reservedTokens, availableTokens }; +} + +/** + * Cut the document to the tokens reserve() left for it. + * + * @param {string} content - Document content + * @param {number} availableTokens - From reserve() + * @param {string} model - Model the content is counted for + * @returns {Promise} + */ +function fit(content, availableTokens, model) { + return truncateToTokenLimit(content, availableTokens, model); +} + +/** Tokens of a text, e.g. the external API data. */ +function count(text, model) { + return calculateTokens(text, model); +} + +/** Cut a text to a token count, e.g. the external API data. */ +function cut(text, maxTokens, model) { + return truncateToTokenLimit(text, maxTokens, model); +} + +module.exports = { reserve, fit, count, cut }; diff --git a/services/llm/transports/chatCompletionsTransport.js b/services/llm/transports/chatCompletionsTransport.js new file mode 100644 index 00000000..64f673a6 --- /dev/null +++ b/services/llm/transports/chatCompletionsTransport.js @@ -0,0 +1,226 @@ +'use strict'; + +/** + * Talks to one chat completions API: OpenAI, Azure OpenAI, or any + * OpenAI-compatible server behind a custom base URL. + * + * A transport owns the client and the request/response mapping, and nothing + * else. It throws on a transport failure and on a provider-reported cut-off + * (error.code 'ai_response_truncated'); it never shapes an analysis result — + * that is analysisResult.js. + */ + +const { + assertCompletionNotTruncated, + extractChatMessageContent, +} = require('../../serviceUtils'); +const { + abortSignal, + abortedGenerationError, + isAbortError, + progressHandler, + readCompletionUsage, + runChatCompletionStream, + stripReasoningText, +} = require('../../aiGenerateOptions'); + +/** What an operator can do about an analysis answer that hit the cap. */ +const ANALYSIS_TRUNCATION_REMEDY = + 'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.'; + +class ChatCompletionsTransport { + /** + * @param {Object} options + * @param {string} options.label - Provider name used in logs and errors + * @param {() => Object|null} options.createClient - Builds the SDK client; + * null when the provider is not configured, which leaves it not ready + * @param {(response:Object, message:Object) => string} [options.describeMissingContent] + * The error message for an answer without usable text. It must keep the + * "Invalid API response structure" phrase the OCR fallback matches on. + * @param {Object} [options.generation] - What generate() needs from the provider + * @param {string} options.generation.notReadyMessage - Thrown without a client + * @param {(prompt:string, options:Object) => Promise} options.generation.buildRequest + * The provider's request: model, messages and its own knobs (temperature + * default, max_tokens policy, reasoning fields). The dialects differ for + * real, so they stay with the provider. + * @param {string} options.generation.remedy - Named when the answer is cut off + * @param {() => boolean} [options.generation.streamUsage] - Whether a + * streamed request may ask for its usage (stream_options.include_usage) + * @param {(response:Object) => string} [options.generation.describeEmptyText] + * The error message for an answer without text + */ + constructor({ label, createClient, describeMissingContent, generation }) { + this.label = label; + this.client = null; + /** Usage of the last generate() call; read by the AI review features. */ + this.lastGenerateTextUsage = null; + this._createClient = createClient; + this._describeMissingContent = + describeMissingContent || (() => 'Invalid API response structure'); + this._generation = { + streamUsage: () => true, + describeEmptyText: () => 'Invalid API response structure', + ...generation, + }; + } + + /** Build the client on first use; a client set from outside is kept. */ + initialize() { + if (!this.client) { + this.client = this._createClient() || null; + } + } + + isReady() { + return Boolean(this.client); + } + + /** + * Send one analysis request. + * + * @param {Object} request + * @param {string} request.systemPrompt + * @param {string} request.userContent + * @param {string} request.model + * @param {number} [request.temperature] - Not sent when undefined + * @returns {Promise<{payload:string, metrics:{promptTokens:number, completionTokens:number, totalTokens:number}}>} + */ + async complete({ systemPrompt, userContent, model, temperature }) { + const timestamp = new Date().toLocaleString('de-DE', { + dateStyle: 'short', + timeStyle: 'short', + }); + + const response = await this.client.chat.completions.create({ + model: model, + messages: [ + { + role: 'system', + content: systemPrompt, + }, + { + role: 'user', + content: userContent, + }, + ], + ...(temperature !== undefined && { temperature }), + }); + + assertCompletionNotTruncated( + response, + this.label, + ANALYSIS_TRUNCATION_REMEDY + ); + const message = response?.choices?.[0]?.message; + const payload = extractChatMessageContent(message, this.label); + if (!payload) { + throw new Error(this._describeMissingContent(response, message)); + } + + console.log(`[DEBUG] [${timestamp}] ${this.label} request sent`); + console.log( + `[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}` + ); + + const usage = response.usage; + return { + payload, + metrics: { + promptTokens: usage.prompt_tokens, + completionTokens: usage.completion_tokens, + totalTokens: usage.total_tokens, + }, + }; + } + + /** + * Send one generation request and return its text. + * + * @param {string} prompt + * @param {Object} [options] - As read by aiGenerateOptions: signal, + * onProgress, and whatever the provider's buildRequest() reads + * @returns {Promise} The answer, reasoning stripped + */ + async generate(prompt, options = {}) { + // Read before the try so the catch can tell a stopped request from a + // failed one. + const signal = abortSignal(options); + const onProgress = progressHandler(options); + const generation = this._generation; + + try { + this.initialize(); + + if (!this.client) { + throw new Error(generation.notReadyMessage); + } + + const request = await generation.buildRequest(prompt, options); + + this.lastGenerateTextUsage = null; + // The second argument is the SDK's request option bag; a caller that + // brought no signal gets the call it always got. + const requestOptions = signal ? { signal } : null; + + if (onProgress) { + const streamRequest = { ...request, stream: true }; + // include_usage buys the exact completion count at the end of the + // stream; without it the report is an estimate all the way. + if (generation.streamUsage()) { + streamRequest.stream_options = { include_usage: true }; + } + return await runChatCompletionStream({ + client: this.client, + request: streamRequest, + requestOptions, + signal, + onProgress, + provider: this.label, + remedy: generation.remedy, + onUsage: (usage) => { + this.lastGenerateTextUsage = usage; + }, + }); + } + + const response = requestOptions + ? await this.client.chat.completions.create(request, requestOptions) + : await this.client.chat.completions.create(request); + this.lastGenerateTextUsage = readCompletionUsage(response); + + const message = response?.choices?.[0]?.message; + // Reasoning is no part of the answer, and a block the cut-off left + // open is no part of it either. + const generatedText = extractChatMessageContent( + { ...message, content: stripReasoningText(message?.content) }, + this.label + ); + + // After the text is in hand: a cut-off answer is still a failure, but + // the caller can keep the prefix instead of paying for it twice. + assertCompletionNotTruncated(response, this.label, generation.remedy, { + partialText: generatedText, + }); + + if (!generatedText) { + throw new Error(generation.describeEmptyText(response)); + } + + return generatedText; + } catch (error) { + // One name for a stopped request, whatever the client called it; an + // abort this transport raised already carries its partial text. + const thrown = + isAbortError(error, signal) && error?.name !== 'AbortError' + ? abortedGenerationError(error, '') + : error; + console.error( + `Error generating text with ${this.label}: ${thrown.message}` + ); + console.debug(thrown); + throw thrown; + } + } +} + +module.exports = { ChatCompletionsTransport, ANALYSIS_TRUNCATION_REMEDY }; diff --git a/services/llm/transports/ollamaGenerateTransport.js b/services/llm/transports/ollamaGenerateTransport.js new file mode 100644 index 00000000..fdc1df66 --- /dev/null +++ b/services/llm/transports/ollamaGenerateTransport.js @@ -0,0 +1,538 @@ +'use strict'; + +/** + * Talks to Ollama's native /api/generate. + * + * Owns what makes Ollama different from the chat completions APIs: a prompt + * and a system string instead of messages, a `format` schema for structured + * output, num_ctx/num_predict, the `think` switch, one bounded retry for + * transient failures, and the done_reason cut-off check. Like every + * transport it throws on failure and never shapes an analysis result. + * + * The axios client is handed in rather than created here: the service builds + * it, and tests replace it there. + */ + +const config = require('../../../config/config'); +const { + abortSignal, + abortedGenerationError, + createProgressCollector, + hasNumber, + hasSystemPrompt, + isAbortError, + modelOverride, + progressHandler, + reasoningEnabled, + stripReasoningText, +} = require('../../aiGenerateOptions'); + +/** + * One bounded retry for transient failures (backend EOF, 5xx). The field + * report behind this: an MLX runner closed its connection once under memory + * pressure; the immediate second attempt succeeded. + */ +const MAX_ATTEMPTS = 2; + +class OllamaGenerateTransport { + /** + * @param {Object} options + * @param {Object} options.client - axios instance + * @param {string} options.apiUrl - Ollama base URL + * @param {string} options.model - Default model + * @param {number} [options.retryDelayMs=1000] - Pause before the retry + */ + constructor({ client, apiUrl, model, retryDelayMs = 1000 }) { + this.label = 'Ollama'; + this.client = client; + this.apiUrl = apiUrl; + this.model = model; + this.retryDelayMs = retryDelayMs; + /** Usage of the last generate() call; read by the AI review features. */ + this.lastGenerateTextUsage = null; + } + + initialize() {} + + isReady() { + return Boolean(this.client); + } + + /** + * Build request headers, adding bearer auth when an API key is set. + * The key is read at request time, so runtime config changes apply + * without re-instantiating the singleton. + * @returns {Object} Headers object + */ + buildHeaders() { + const headers = { 'Content-Type': 'application/json' }; + const apiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || ''; + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + return headers; + } + + /** + * Extract the cause an Ollama (or its inference backend) reported in a + * non-2xx response body. Ollama wraps backend failures — its runner closing + * the connection on an oversized prompt, for example — into + * `{ error: 'Post "http://…/v1/completions": EOF' }`; without this the + * operator only ever sees "Request failed with status code 500" and cannot + * tell an Ollama problem from a model-backend problem. + * + * @param {Object} axiosError + * @returns {string|null} + */ + extractUpstreamErrorMessage(axiosError) { + const data = axiosError?.response?.data; + if (!data) return null; + if (typeof data === 'string') { + const trimmed = data.trim(); + try { + return JSON.parse(trimmed)?.error || trimmed || null; + } catch { + return trimmed || null; + } + } + return data.error || data.message || null; + } + + /** + * True for failures worth one more attempt: the connection dropped or the + * server answered 5xx. A 4xx is a request problem and will fail again. + * + * @param {Object} axiosError + * @returns {boolean} + */ + isTransientFailure(axiosError) { + if (!axiosError.response) { + // No response at all: connection refused/reset/timeout. + return true; + } + return axiosError.response.status >= 500; + } + + /** + * Wrap an axios failure with the upstream detail attached. + * + * @param {Object} axiosError - The axios error being wrapped + * @param {number} attempt - Which attempt failed (1-based) + * @returns {Error} + */ + wrapRequestError(axiosError, attempt) { + const upstream = this.extractUpstreamErrorMessage(axiosError); + const status = axiosError?.response?.status; + const detail = upstream ? `: ${upstream}` : ''; + const wrapped = new Error( + `Ollama request failed (HTTP ${status || 'no response'}, attempt ${attempt})${detail}`, + { cause: axiosError } + ); + // Keep the axios code so the scan-loop classification (OCR fallback, + // failure reasons) keeps working unchanged. + wrapped.code = axiosError.code; + return wrapped; + } + + /** + * Send one analysis request. + * + * @param {Object} request + * @param {string} request.systemPrompt - Sent as `system` + * @param {string} request.userContent - Sent as `prompt` + * @param {string} [request.model] - Defaults to the transport's model + * @param {number} request.temperature + * @param {number} request.maxResponseTokens - Sent as num_predict + * @param {number} request.numCtx - Context window to request + * @param {Object|null} request.schema - Sent as `format` + * @returns {Promise<{payload:string|Object, metrics:Object, raw:Object}>} + */ + async complete({ + systemPrompt, + userContent, + model, + temperature, + maxResponseTokens, + numCtx, + schema, + }) { + const numPredict = maxResponseTokens; + const requestBody = { + model: model || this.model, + prompt: userContent, + system: systemPrompt, + stream: false, + format: schema, + options: { + temperature, + top_p: 0.9, + repeat_penalty: 1.1, + top_k: 7, + num_predict: numPredict, + num_ctx: numCtx, + }, + }; + + // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1) + // unless explicitly enabled via OLLAMA_THINK=true + if (!config.ollama.think) { + requestBody.think = false; + } + + let response; + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { + try { + response = await this.client.post( + `${this.apiUrl}/api/generate`, + requestBody, + { + headers: this.buildHeaders(), + } + ); + break; + } catch (error) { + const wrapped = this.wrapRequestError(error, attempt); + console.error(`[ERROR] ${wrapped.message}`); + if (attempt >= MAX_ATTEMPTS || !this.isTransientFailure(error)) { + throw wrapped; + } + await new Promise((resolve) => setTimeout(resolve, this.retryDelayMs)); + } + } + + if (!response?.data) { + throw new Error('Invalid response from Ollama API'); + } + + this.assertNotTruncated(response.data, numPredict, numCtx); + // Worded unlike the chat providers' "Invalid API response structure", so + // it matches no OCR marker (row l, kept as it is). + if (!response.data.response) { + throw new Error('No response data from Ollama API'); + } + return { + payload: response.data.response, + metrics: this.extractMetrics(response.data), + raw: response.data, + }; + } + + /** + * Send one generation request and return its text. + * + * @param {string} prompt + * @param {object} [options] - As read by aiGenerateOptions: systemPrompt, + * temperature, model, signal, onProgress, reasoning + * @param {Object} numbers - Sized by the caller + * @param {number} numbers.responseTokens - Sent as num_predict + * @param {number} numbers.numCtx - Context window to request + * @returns {Promise} - The generated text + */ + async generate(prompt, options = {}, { responseTokens, numCtx }) { + // Read before the try so the catch can tell a stopped request from a + // failed one, and so a cut-off request keeps what it measured. + const signal = abortSignal(options); + const onProgress = progressHandler(options); + let measuredUsage = null; + + try { + // Simple system prompt for text generation; a caller that knows the job + // better (the Duplicates review) brings its own. + const systemPrompt = hasSystemPrompt(options) + ? options.systemPrompt + : `You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.`; + + // Call Ollama API without enforcing a specific response format + const generateTextBody = { + model: modelOverride(options) || this.model, + prompt: prompt, + system: systemPrompt, + stream: false, + options: { + temperature: hasNumber(options.temperature) + ? options.temperature + : config.aiTemperatureGeneration, + top_p: 0.9, + // The line above reserves the same number in the context; a + // constant here would have the same quarrel with it that the + // analysis path had. + num_predict: responseTokens, + num_ctx: numCtx, + }, + }; + + // A caller that says what it wants overrules the setting; without one + // OLLAMA_THINK decides, which is what it did before. + const reasoning = reasoningEnabled(options); + if (reasoning !== null) { + generateTextBody.think = reasoning; + } else if (!config.ollama.think) { + // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1) + // unless explicitly enabled via OLLAMA_THINK=true + generateTextBody.think = false; + } + + // axios takes the signal in the request config; without one the config + // is the headers it always was. + const requestConfig = { headers: this.buildHeaders() }; + if (signal) requestConfig.signal = signal; + + if (onProgress) { + return await this._generateStreamed( + generateTextBody, + requestConfig, + { signal, onProgress, numPredict: responseTokens, numCtx }, + (usage) => { + measuredUsage = usage; + this.lastGenerateTextUsage = usage; + } + ); + } + + const response = await this.client.post( + `${this.apiUrl}/api/generate`, + generateTextBody, + requestConfig + ); + + if (!response.data || !response.data.response) { + throw new Error('Invalid response from Ollama API'); + } + + // Ollama reports its token counts on the answer itself; the callers + // that care (the Duplicates review) read them off the service after + // the call, so the return value stays a plain string. + const promptTokens = Number(response.data.prompt_eval_count); + const completionTokens = Number(response.data.eval_count); + measuredUsage = + Number.isFinite(promptTokens) || Number.isFinite(completionTokens) + ? { + promptTokens: Number.isFinite(promptTokens) ? promptTokens : null, + completionTokens: Number.isFinite(completionTokens) + ? completionTokens + : null, + totalTokens: + (Number.isFinite(promptTokens) ? promptTokens : 0) + + (Number.isFinite(completionTokens) ? completionTokens : 0), + } + : null; + this.lastGenerateTextUsage = measuredUsage; + + // A model that writes its reasoning into the answer wrote it for + // itself; only what it said afterwards is the answer. Anything that is + // not a string is left alone — it is a structured answer, not text. + const generatedText = + typeof response.data.response === 'string' + ? stripReasoningText(response.data.response).trim() + : response.data.response; + + // The cut-off check the analysis path has always run, now on this path + // too, and carrying the prefix the caller can still use. + this.assertNotTruncated(response.data, responseTokens, numCtx, { + partialText: typeof generatedText === 'string' ? generatedText : '', + }); + + return generatedText; + } catch (error) { + // Whatever was measured before the failure stays readable; a failure + // that measured nothing still leaves null behind, as it always did. + this.lastGenerateTextUsage = measuredUsage; + // One name for a stopped request, whatever axios called it; an abort + // this transport raised already carries its partial text. + const thrown = + isAbortError(error, signal) && error?.name !== 'AbortError' + ? abortedGenerationError(error, '') + : error; + console.error(`Error generating text with Ollama: ${thrown.message}`); + console.debug(thrown); + throw thrown; + } + } + + /** + * The streamed half of generate(): the same request with `stream: true`, + * read as the NDJSON lines Ollama answers with. + * + * Ollama does not speak server-sent events — every line is a complete JSON + * object with the next piece of the answer in `response`, the next piece of + * the model's thinking in `thinking`, and on the last line the counts and + * the reason it stopped. Lines arrive split across socket reads, so the tail + * of a read is kept until its newline turns up. + * + * @param {Object} body - the request body, `stream` still to be set + * @param {Object} requestConfig - headers, and the signal if there is one + * @param {Object} context + * @param {AbortSignal|null} context.signal + * @param {Function} context.onProgress + * @param {number} context.numPredict - response budget that was sent + * @param {number} context.numCtx - context window that was sent + * @param {Function} onUsage - told before a cut-off throws + * @returns {Promise} the answer, reasoning stripped + */ + async _generateStreamed(body, requestConfig, context, onUsage) { + const { signal, onProgress, numPredict, numCtx } = context; + const collector = createProgressCollector({ onProgress }); + let last = {}; + + try { + const response = await this.client.post( + `${this.apiUrl}/api/generate`, + { ...body, stream: true }, + { ...requestConfig, responseType: 'stream' } + ); + + const stream = response.data; + // An abort tears the socket down, and a socket error that arrives after + // the loop has left would be an unhandled 'error' event — which ends + // the process rather than the request. + stream.on('error', () => {}); + let pending = ''; + const takeLine = (line) => { + if (line.trim() === '') return; + let parsed; + try { + parsed = JSON.parse(line); + } catch { + // A line that is not JSON is a line this client cannot use; the + // stream goes on, and the answer is whatever the rest carries. + return; + } + collector.pushContent(parsed.response); + collector.pushReasoning(parsed.thinking); + if (parsed.done === true) last = parsed; + collector.report(); + }; + + for await (const piece of stream) { + if (signal?.aborted) { + // Nothing more is wanted, so nothing more is read: the socket goes + // now rather than after the model finished talking to itself. + stream.destroy(); + break; + } + pending += piece.toString('utf8'); + let at = pending.indexOf('\n'); + while (at !== -1) { + takeLine(pending.slice(0, at)); + pending = pending.slice(at + 1); + at = pending.indexOf('\n'); + } + } + if (!signal?.aborted) takeLine(pending); + + if (signal?.aborted) { + collector.finish(); + throw abortedGenerationError(null, collector.text()); + } + } catch (error) { + collector.finish(); + if (error?.name === 'AbortError') throw error; + if (isAbortError(error, signal)) { + throw abortedGenerationError(error, collector.text()); + } + throw error; + } + + collector.finish(); + + const promptTokens = Number(last.prompt_eval_count); + const completionTokens = Number(last.eval_count); + const measured = + Number.isFinite(promptTokens) || Number.isFinite(completionTokens) + ? { + promptTokens: Number.isFinite(promptTokens) ? promptTokens : null, + completionTokens: Number.isFinite(completionTokens) + ? completionTokens + : null, + totalTokens: + (Number.isFinite(promptTokens) ? promptTokens : 0) + + (Number.isFinite(completionTokens) ? completionTokens : 0), + } + : collector.usageSummary(); + if (typeof onUsage === 'function') onUsage(measured); + + this.assertNotTruncated(last, numPredict, numCtx, { + partialText: collector.text(), + }); + + return collector.text(); + } + + /** + * A generation Ollama had to cut short comes back with done_reason "length". + * What it returns is a prefix, and for structured output that means JSON + * without its closing braces — which no amount of sanitizing recovers, so + * there is nothing to do with it but say so. + * + * Two different limits produce the same done_reason and want opposite fixes: + * num_predict is the response budget, but the context window can leave less + * room than that, and then raising Response Tokens changes nothing. The + * message names whichever one actually bit. + * + * @param {Object} responseData - Ollama /api/generate response + * @param {number} numPredict - response token limit that was sent + * @param {number} numCtx - context window that was sent + * @param {Object} [extra] - what the caller salvaged from the cut-off answer + * @param {string} [extra.partialText] - the text that did arrive, reasoning + * stripped; put on the error as `partialText` the way the OpenAI-compatible + * providers do, so a caller can keep it instead of asking twice + */ + assertNotTruncated(responseData, numPredict, numCtx, extra) { + const evalCount = Number(responseData?.eval_count) || 0; + const promptEvalCount = Number(responseData?.prompt_eval_count) || 0; + + // done_reason is the direct signal. Reaching the limit exactly is the + // fallback for an Ollama old enough not to send one. + const stoppedOnLength = + responseData?.done_reason === 'length' || + (!responseData?.done_reason && numPredict > 0 && evalCount >= numPredict); + + if (!stoppedOnLength) { + return; + } + + /* Which limit bit is read off the count itself rather than guessed from + the context arithmetic: reaching num_predict exactly is the response + budget, stopping short of it means the context filled up first. The + arithmetic route was tried and got it backwards — the window is sized + from an estimate of the prompt, so comparing it against the real + prompt_eval_count blames the context for a run that generated its full + budget. */ + const error = + evalCount >= numPredict + ? new Error( + `Ollama stopped generating after ${evalCount} tokens, which is the configured response limit. Raise Response Tokens (RESPONSE_TOKENS) above ${numPredict}.` + ) + : new Error( + `Ollama stopped generating after ${evalCount} of ${numPredict} response tokens: the ${numCtx}-token context window was full, ${promptEvalCount} of it prompt. Raise Token Limit (TOKEN_LIMIT) or shorten the document.` + ); + + /* Carried as a code rather than matched out of the message later. The + phrase-matching in serviceUtils classifies errors that arrive from + elsewhere and have no better handle; this one is raised right here, so + rewording it should not silently reclassify the failure. */ + error.code = 'ai_response_truncated'; + if (typeof extra?.partialText === 'string') { + error.partialText = extra.partialText; + } + throw error; + } + + /** + * Token usage of an /api/generate response. + * @param {Object} responseData - Ollama API response data + * @returns {{promptTokens:number, completionTokens:number, totalTokens:number}} + */ + extractMetrics(responseData) { + const promptTokens = Number(responseData?.prompt_eval_count) || 0; + const completionTokens = Number(responseData?.eval_count) || 0; + + return { + promptTokens, + completionTokens, + totalTokens: promptTokens + completionTokens, + }; + } +} + +module.exports = { OllamaGenerateTransport }; diff --git a/services/manualService.js b/services/manualService.js deleted file mode 100644 index 4486d9d7..00000000 --- a/services/manualService.js +++ /dev/null @@ -1,266 +0,0 @@ -const { - calculateTokens, - calculateTotalPromptTokens, - truncateToTokenLimit, - writePromptToFile, - extractChatMessageContent -} = require('./serviceUtils'); -const axios = require('axios'); -const OpenAI = require('openai'); -const config = require('../config/config'); -const AzureOpenAI = require('openai').AzureOpenAI; -const emptyVar = null; - -class ManualService { - constructor() { - if(config.aiProvider === 'custom'){ - this.openai = new OpenAI({ - apiKey: config.custom.apiKey, - baseUrl: config.custom.apiUrl - }); - }else if (config.aiProvider === 'azure'){ - this.openai = new AzureOpenAI({ - apiKey: config.azure.apiKey, - endpoint: config.azure.endpoint, - deploymentName: config.azure.deploymentName, - apiVersion: config.azure.apiVersion - }); - } else { - this.openai = new OpenAI({ apiKey: config.openai.apiKey }); - this.ollama = axios.create({ - timeout: 300000 - }); - } - } - - - async analyzeDocument(content, existingTags, provider) { - try { - if (provider === 'openai') { - return this._analyzeOpenAI(content, existingTags); - } else if (provider === 'ollama') { - return this._analyzeOllama(content, existingTags); - } else if (provider === 'custom') { - return this._analyzeCustom(content, existingTags); - } else if (provider === 'azure') { - return this._analyzeAzure(content, existingTags); - } else { - throw new Error('Invalid provider'); - } - } catch (error) { - console.error('Error analyzing document:', error); - return { tags: [], correspondent: null }; - } - } - - async _analyzeOpenAI(content, existingTags) { - try { - const existingTagsList = existingTags - .map(tag => tag.name) - .join(', '); - const model = process.env.OPENAI_MODEL; - const systemPrompt = process.env.SYSTEM_PROMPT; - await writePromptToFile(systemPrompt, content); - const response = await this.openai.chat.completions.create({ - model: model, - messages: [ - { - role: "system", - content: systemPrompt - }, - { - role: "user", - content: content - } - ], - ...(model !== 'o3-mini' && { temperature: config.aiTemperatureAnalysis }), - }); - - let jsonContent = extractChatMessageContent(response?.choices?.[0]?.message, 'ManualService/OpenAI'); - if (!jsonContent) { - throw new Error('Invalid API response structure'); - } - jsonContent = jsonContent.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim(); - - let parsedResponse; - try { - parsedResponse = JSON.parse(jsonContent); - } catch (error) { - console.error('Failed to parse JSON response:', error); - throw new Error('Invalid JSON response from API'); - } - - if (!Array.isArray(parsedResponse.tags) || (typeof parsedResponse.correspondent !== 'string' && parsedResponse.correspondent !== null)) { - throw new Error('Invalid response structure'); - } - - return parsedResponse; - } catch (error) { - console.error('Failed to analyze document with OpenAI:', error); - return { tags: [], correspondent: null }; - } - } - - async _analyzeAzure(content, existingTags) { - try { - const existingTagsList = existingTags - .map(tag => tag.name) - .join(', '); - - const systemPrompt = process.env.SYSTEM_PROMPT; - await writePromptToFile(systemPrompt, content); - const response = await this.openai.chat.completions.create({ - model: process.env.AZURE_DEPLOYMENT_NAME, - messages: [ - { - role: "system", - content: systemPrompt - }, - { - role: "user", - content: content - } - ], - temperature: config.aiTemperatureAnalysis, - }); - - let jsonContent = extractChatMessageContent(response?.choices?.[0]?.message, 'ManualService/Azure'); - if (!jsonContent) { - throw new Error('Invalid API response structure'); - } - jsonContent = jsonContent.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim(); - - let parsedResponse; - try { - parsedResponse = JSON.parse(jsonContent); - } catch (error) { - console.error('Failed to parse JSON response:', error); - throw new Error('Invalid JSON response from API'); - } - - if (!Array.isArray(parsedResponse.tags) || (typeof parsedResponse.correspondent !== 'string' && parsedResponse.correspondent !== null)) { - throw new Error('Invalid response structure'); - } - - return parsedResponse; - } catch (error) { - console.error('Failed to analyze document with OpenAI:', error); - return { tags: [], correspondent: null }; - } - } - - async _analyzeCustom(content, existingTags) { - try { - const existingTagsList = existingTags - .map(tag => tag.name) - .join(', '); - - const systemPrompt = process.env.SYSTEM_PROMPT; - const model = config.custom.model; - const response = await this.openai.chat.completions.create({ - model: model, - messages: [ - { - role: "system", - content: systemPrompt - }, - { - role: "user", - content: content - } - ], - ...(model !== 'o3-mini' && { temperature: config.aiTemperatureAnalysis }), - }); - - let jsonContent = extractChatMessageContent(response?.choices?.[0]?.message, 'ManualService/Custom'); - if (!jsonContent) { - throw new Error('Invalid API response structure'); - } - jsonContent = jsonContent.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim(); - - const parsedResponse = JSON.parse(jsonContent); - - if (!Array.isArray(parsedResponse.tags) || (typeof parsedResponse.correspondent !== 'string' && parsedResponse.correspondent !== null)) { - throw new Error('Invalid response structure'); - } - - return parsedResponse; - } catch (error) { - console.error('Failed to analyze document with OpenAI:', error); - return { tags: [], correspondent: null }; - } - } - - async _analyzeOllama(content, existingTags) { - try { - const prompt = process.env.SYSTEM_PROMPT; - - const getAvailableMemory = async () => { - const totalMemory = os.totalmem(); - const freeMemory = os.freemem(); - const totalMemoryMB = (totalMemory / (1024 * 1024)).toFixed(0); - const freeMemoryMB = (freeMemory / (1024 * 1024)).toFixed(0); - return { totalMemoryMB, freeMemoryMB }; - }; - - const calculateNumCtx = (promptTokenCount, expectedResponseTokens) => { - const totalTokenUsage = promptTokenCount + expectedResponseTokens; - const maxCtxLimit = Number(config.tokenLimit); - - const numCtx = Math.min(totalTokenUsage, maxCtxLimit); - - console.log('Prompt Token Count:', promptTokenCount); - console.log('Expected Response Tokens:', expectedResponseTokens); - console.log('Dynamic calculated num_ctx:', numCtx); - - return numCtx; - }; - - const calculatePromptTokenCount = (prompt) => { - return Math.ceil(prompt.length / 4); - }; - - const { freeMemoryMB } = await getAvailableMemory(); - const expectedResponseTokens = 1024; - const promptTokenCount = calculatePromptTokenCount(prompt); - - const numCtx = calculateNumCtx(promptTokenCount, expectedResponseTokens); - - const ollamaHeaders = { 'Content-Type': 'application/json' }; - const ollamaApiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || ''; - if (ollamaApiKey) { - ollamaHeaders.Authorization = `Bearer ${ollamaApiKey}`; - } - - const response = await this.ollama.post(`${config.ollama.apiUrl}/api/generate`, { - model: config.ollama.model, - prompt: prompt, - stream: false, - options: { - temperature: config.aiTemperatureGeneration, - top_p: 0.9, - repeat_penalty: 1.1, - num_ctx: numCtx, - } - }, { headers: ollamaHeaders }); - - if (!response.data || !response.data.response) { - console.error('Unexpected Ollama response format:', response); - throw new Error('Invalid response from Ollama API'); - } - - return this._parseResponse(response.data.response); - } - - catch (error) { - if (error.code === 'ECONNABORTED') { - console.error('Timeout bei der Ollama-Anfrage:', error); - throw new Error('Die Analyse hat zu lange gedauert. Bitte versuchen Sie es erneut.'); - } - console.error('Error analyzing document with Ollama:', error); - throw error; - } - } -} - -module.exports = ManualService; diff --git a/services/ollamaService.js b/services/ollamaService.js index 18c7ed9d..f9b147e9 100644 --- a/services/ollamaService.js +++ b/services/ollamaService.js @@ -1,1110 +1,3 @@ -const { writePromptToFile, toNameList } = require('./serviceUtils'); -const { - abortSignal, - abortedGenerationError, - createProgressCollector, - hasNumber, - hasSystemPrompt, - isAbortError, - modelOverride, - progressHandler, - reasoningEnabled, - stripReasoningText, -} = require('./aiGenerateOptions'); -const axios = require('axios'); -const config = require('../config/config'); -const fs = require('fs').promises; -const paperlessService = require('./paperlessService'); -const os = require('os'); -const { - THUMBNAIL_CACHE_DIR, - getThumbnailCachePath, -} = require('./thumbnailCachePaths'); -const RestrictionPromptService = require('./restrictionPromptService'); - -/** - * Service for document analysis using Ollama - */ -class OllamaService { - /** - * Initialize the Ollama service - */ - constructor() { - this.apiUrl = config.ollama.apiUrl; - this.model = config.ollama.model; - this.client = axios.create({ - timeout: 1800000, // 30 minutes timeout - }); - - // JSON schema for document analysis output - this.documentAnalysisSchema = { - type: 'object', - properties: { - title: { type: 'string' }, - correspondent: { type: 'string' }, - tags: { - type: 'array', - items: { type: 'string' }, - }, - document_type: { type: 'string' }, - document_date: { type: 'string' }, - language: { type: 'string' }, - custom_fields: { - type: 'object', - additionalProperties: true, - }, - }, - required: [ - 'title', - 'correspondent', - 'tags', - 'document_type', - 'document_date', - 'language', - ], - }; - } - - /** - * Analyze a document and extract metadata - * @param {string} content - Document content - * @param {Array} existingTags - List of existing tags - * @param {Array} existingCorrespondentList - List of existing correspondents - * @param {string} id - Document ID - * @param {string} customPrompt - Custom prompt (optional) - * @returns {Object} Analysis results - */ - async analyzeDocument( - content, - existingTags = [], - existingCorrespondentList = [], - existingDocumentTypesList = [], - id, - customPrompt = null, - options = {} - ) { - try { - // Truncate content if needed - content = this._truncateContent(content); - - // Cache thumbnail - await this._handleThumbnailCaching(id); - - // Get external API data if available and validate it - let externalApiData = options.externalApiData || null; - let validatedExternalApiData = null; - - if (externalApiData) { - try { - validatedExternalApiData = - await this._validateAndTruncateExternalApiData(externalApiData); - console.log('[DEBUG] External API data validated and included'); - } catch (error) { - console.warn( - '[WARNING] External API data validation failed:', - error.message - ); - validatedExternalApiData = null; - } - } - - // Build prompt - let prompt; - if (!customPrompt) { - prompt = this._buildPrompt( - content, - existingTags, - existingCorrespondentList, - existingDocumentTypesList, - options - ); - } else { - // Parse CUSTOM_FIELDS for custom prompt - let customFieldsObj; - try { - customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS); - } catch (error) { - console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); - console.debug(error); - customFieldsObj = { custom_fields: [] }; - } - - const customFieldsTemplate = {}; - customFieldsObj.custom_fields.forEach((field, index) => { - let valueHint; - if (field.data_type === 'date') { - valueHint = - 'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis'; - } else if (field.data_type === 'boolean') { - valueHint = "Fill in 'true' or 'false' based on your analysis"; - } else { - valueHint = 'Fill in the value based on your analysis'; - } - customFieldsTemplate[index] = { - field_name: field.value, - value: valueHint, - }; - }); - - const customFieldsStr = - '"custom_fields": ' + - JSON.stringify(customFieldsTemplate, null, 2) - .split('\n') - .map((line) => ' ' + line) - .join('\n'); - - prompt = - customPrompt + - '\n\n' + - config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr) + - '\n\n' + - JSON.stringify(content); - console.log('[DEBUG] Ollama Service started with custom prompt'); - } - - // Generate custom fields for the prompt - const customFieldsStr = this._generateCustomFieldsTemplate(); - - // Generate system prompt - const systemPrompt = this._generateSystemPrompt(customFieldsStr); - - // Calculate context window size - const promptTokenCount = this._calculatePromptTokenCount(prompt); - const numCtx = this._calculateNumCtx( - promptTokenCount, - Number(config.responseTokens) - ); - - console.log( - `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting` - ); - console.log( - `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}` - ); - - // Call Ollama API - const response = await this._callOllamaAPI( - prompt, - systemPrompt, - numCtx, - this.documentAnalysisSchema - ); - - // Process response - const parsedResponse = this._processOllamaResponse(response); - const metrics = this._extractOllamaMetrics(response); - - // Check for missing data - if ( - parsedResponse.tags.length === 0 && - parsedResponse.correspondent === null - ) { - console.warn( - 'No tags or correspondent found in response from Ollama for Document. Please review your prompt or switch to OpenAI for better results.' - ); - } - - // Log the prompt and response - await this._logPromptAndResponse(prompt, parsedResponse); - - // Return results in consistent format - return { - document: parsedResponse, - metrics, - truncated: false, - }; - } catch (error) { - console.error(`Error analyzing document with Ollama: ${error.message}`); - console.debug(error); - return { - document: { tags: [], correspondent: null }, - metrics: null, - error: error.message, - // Undefined for everything that is not one of ours; the scan loop - // falls back to its generic reason then. - errorCode: error.code, - }; - } - } - - /** - * Truncate content to maximum length if specified - * @param {string} content - Content to truncate - * @returns {string} Truncated content - */ - _truncateContent(content) { - try { - if (process.env.CONTENT_MAX_LENGTH) { - console.log( - 'Truncating content to max length:', - process.env.CONTENT_MAX_LENGTH - ); - return content.substring(0, process.env.CONTENT_MAX_LENGTH); - } - } catch (error) { - console.error(`Error truncating content: ${error.message}`); - console.debug(error); - } - return content; - } - - /** - * Build prompt from content and existing data - * @param {string} content - Document content - * @param {Array} existingTags - List of existing tags - * @param {Array} existingCorrespondent - List of existing correspondents - * @param {Array} existingDocumentTypes - List of existing document types - * @returns {string} Formatted prompt - */ - _buildPrompt( - content, - existingTags = [], - existingCorrespondent = [], - existingDocumentTypes = [], - options = {} - ) { - let systemPrompt; - - // Validate that existingCorrespondent is an array and handle if it's not - const correspondentList = Array.isArray(existingCorrespondent) - ? existingCorrespondent - : []; - - // Parse CUSTOM_FIELDS from environment variable - let customFieldsObj; - try { - customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS); - } catch (error) { - console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); - console.debug(error); - customFieldsObj = { custom_fields: [] }; - } - - // Generate custom fields template for the prompt - const customFieldsTemplate = {}; - - customFieldsObj.custom_fields.forEach((field, index) => { - let valueHint; - if (field.data_type === 'date') { - valueHint = - 'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis'; - } else if (field.data_type === 'boolean') { - valueHint = "Fill in 'true' or 'false' based on your analysis"; - } else { - valueHint = 'Fill in the value based on your analysis'; - } - customFieldsTemplate[index] = { - field_name: field.value, - value: valueHint, - }; - }); - - // Convert template to string for replacement and wrap in custom_fields - const customFieldsStr = - '"custom_fields": ' + - JSON.stringify(customFieldsTemplate, null, 2) - .split('\n') - .map((line) => ' ' + line) // Add proper indentation - .join('\n'); - - // Get system prompt based on configuration - if ( - config.useExistingData === 'yes' && - config.restrictToExistingTags === 'no' && - config.restrictToExistingCorrespondents === 'no' - ) { - // Format existing data - callers may hand over entity objects or plain names - const existingTagsList = toNameList(existingTags).join(', '); - const existingCorrespondentList = - toNameList(correspondentList).join(', '); - const existingDocumentTypesList = toNameList(existingDocumentTypes).join( - ', ' - ); - - systemPrompt = - ` - Pre-existing tags: ${existingTagsList}\n\n - Pre-existing correspondents: ${existingCorrespondentList}\n\n - Pre-existing document types: ${existingDocumentTypesList}\n\n - ` + - process.env.SYSTEM_PROMPT + - '\n\n' + - config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr); - } else { - const mustHavePrompt = config.mustHavePrompt.replace( - '%CUSTOMFIELDS%', - customFieldsStr - ); - systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt; - } - - // Get validated external API data if available - let validatedExternalApiData = null; - if (options.externalApiData) { - try { - validatedExternalApiData = this._validateAndTruncateExternalApiData( - options.externalApiData - ); - console.log('[DEBUG] External API data validated and included'); - } catch (error) { - console.warn( - '[WARNING] External API data validation failed:', - error.message - ); - validatedExternalApiData = null; - } - } - - // Process placeholder replacements in system prompt - systemPrompt = RestrictionPromptService.processRestrictionsInPrompt( - systemPrompt, - existingTags, - correspondentList, - existingDocumentTypes - ); - - // Include validated external API data if available - if (validatedExternalApiData) { - systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`; - } - - if (process.env.USE_PROMPT_TAGS === 'yes') { - systemPrompt = - ` - Take these tags and try to match one or more to the document content.\n\n - ` + config.specialPromptPreDefinedTags; - } - - return `${systemPrompt} - ${JSON.stringify(content)} - `; - } - - /** - * Validate and truncate external API data to prevent token overflow - * @param {any} apiData - The external API data to validate - * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500) - * @returns {string} - Validated and potentially truncated data string - */ - async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) { - if (!apiData) { - return null; - } - - const dataString = - typeof apiData === 'object' - ? JSON.stringify(apiData, null, 2) - : String(apiData); - - // Calculate tokens for the data (conservative 2 chars/token for non-English) - const dataTokens = Math.ceil(dataString.length / 2); - - if (dataTokens > maxTokens) { - console.warn( - `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating` - ); - // Simple truncation based on character count - const maxChars = maxTokens * 2; - return dataString.substring(0, maxChars); - } - - console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`); - return dataString; - } - - /** - * Generate custom fields template for prompts - * @returns {string} Custom fields template as a string - */ - _generateCustomFieldsTemplate() { - let customFieldsObj; - try { - customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS); - } catch (error) { - console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); - console.debug(error); - customFieldsObj = { custom_fields: [] }; - } - - // Generate custom fields template for the prompt - const customFieldsTemplate = {}; - - customFieldsObj.custom_fields.forEach((field, index) => { - let valueHint; - if (field.data_type === 'date') { - valueHint = - 'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis'; - } else if (field.data_type === 'boolean') { - valueHint = "Fill in 'true' or 'false' based on your analysis"; - } else { - valueHint = 'Fill in the value based on your analysis'; - } - customFieldsTemplate[index] = { - field_name: field.value, - value: valueHint, - }; - }); - - // Convert template to string for replacement and wrap in custom_fields - return ( - '"custom_fields": ' + - JSON.stringify(customFieldsTemplate, null, 2) - .split('\n') - .map((line) => ' ' + line) // Add proper indentation - .join('\n') - ); - } - - /** - * Generate system prompt for document analysis - * @param {string} customFieldsStr - Custom fields as a string - * @returns {string} System prompt - */ - _generateSystemPrompt(customFieldsStr) { - let systemPromptTemplate = ` - You are a document analyzer. Your task is to analyze documents and extract relevant information. You do not ask back questions. - YOU MUSTNOT: Ask for additional information or clarification, or ask questions about the document, or ask for additional context. - YOU MUSTNOT: Return a response without the desired JSON format. - YOU MUST: Return the result EXCLUSIVELY as a JSON object. The Tags, Title and Document_Type MUST be in the language that is used in the document.: - IMPORTANT: The custom_fields are optional and can be left out if not needed, only try to fill out the values if you find a matching information in the document. - Do not change the value of field_name, only fill out the values. If the field is about money only add the number without currency and always use a . for decimal places. - { - "title": "xxxxx", - "correspondent": "xxxxxxxx", - "tags": ["Tag1", "Tag2", "Tag3", "Tag4"], - "document_type": "Invoice/Contract/...", - "document_date": "YYYY-MM-DD", - "language": "en/de/es/...", - %CUSTOMFIELDS% - } - ALWAYS USE THE INFORMATION TO FILL OUT THE JSON OBJECT. DO NOT ASK BACK QUESTIONS. - `; - - return systemPromptTemplate.replace('%CUSTOMFIELDS%', customFieldsStr); - } - - /** - * Calculate prompt token count - * @param {string} prompt - Prompt text - * @returns {number} Estimated token count - */ - _calculatePromptTokenCount(prompt) { - // Use conservative 2 chars/token estimate to avoid truncation - // with non-English models (CJK, German, etc.) where tokenization - // produces roughly 2 chars per token instead of ~4 for English - return Math.ceil(prompt.length / 2); - } - - /** - * Calculate context window size for Ollama - * @param {number} promptTokenCount - Token count for prompt - * @param {number} expectedResponseTokens - Expected response token count - * @returns {number} Context window size - */ - _calculateNumCtx(promptTokenCount, expectedResponseTokens) { - const totalTokenUsage = promptTokenCount + expectedResponseTokens; - const maxCtxLimit = Number(config.tokenLimit); - - const numCtx = Math.min(totalTokenUsage, maxCtxLimit); - - console.log('Prompt Token Count:', promptTokenCount); - console.log('Expected Response Tokens:', expectedResponseTokens); - console.log('Dynamic calculated num_ctx:', numCtx); - - return numCtx; - } - - /** - * Get available system memory - * @returns {Object} Object with totalMemoryMB and freeMemoryMB - */ - async _getAvailableMemory() { - const totalMemory = os.totalmem(); - const freeMemory = os.freemem(); - const totalMemoryMB = (totalMemory / (1024 * 1024)).toFixed(0); - const freeMemoryMB = (freeMemory / (1024 * 1024)).toFixed(0); - return { totalMemoryMB, freeMemoryMB }; - } - - /** - * Handle thumbnail caching for documents - * @param {string} id - Document ID - */ - async _handleThumbnailCaching(id) { - if (!id) return; - - const cachePath = getThumbnailCachePath(id); - try { - await fs.access(cachePath); - console.log('[DEBUG] Thumbnail already cached'); - } catch { - console.log('Thumbnail not cached, fetching from Paperless'); - const thumbnailData = await paperlessService.getThumbnailImage(id); - if (!thumbnailData) { - console.warn('Thumbnail not found'); - return; - } - await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); - await fs.writeFile(cachePath, thumbnailData); - } - } - - /** - * Build request headers, adding bearer auth when an API key is set. - * The key is read at request time so runtime config changes apply - * without re-instantiating the singleton. - * @returns {Object} Headers object - */ - _buildRequestHeaders() { - const headers = { 'Content-Type': 'application/json' }; - const apiKey = process.env.OLLAMA_API_KEY || config.ollama.apiKey || ''; - if (apiKey) { - headers.Authorization = `Bearer ${apiKey}`; - } - return headers; - } - - /** - * Call Ollama API - * @param {string} prompt - Prompt text - * @param {string} systemPrompt - System prompt - * @param {number} numCtx - Context window size - * @param {Object} schema - Response schema - * @returns {Object} Ollama API response - */ - async _callOllamaAPI(prompt, systemPrompt, numCtx, schema) { - // The same number _calculateNumCtx() already reserved room for. It used to - // be a hardcoded 256 while the reservation followed the setting, so the - // context held space for an answer the model was then forbidden to write. - const numPredict = Number(config.responseTokens); - const requestBody = { - model: this.model, - prompt: prompt, - system: systemPrompt, - stream: false, - format: schema, - options: { - temperature: config.aiTemperatureAnalysis, - top_p: 0.9, - repeat_penalty: 1.1, - top_k: 7, - num_predict: numPredict, - num_ctx: numCtx, - }, - }; - - // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1) - // unless explicitly enabled via OLLAMA_THINK=true - if (!config.ollama.think) { - requestBody.think = false; - } - - const response = await this.client.post( - `${this.apiUrl}/api/generate`, - requestBody, - { - headers: this._buildRequestHeaders(), - } - ); - - if (!response.data) { - throw new Error('Invalid response from Ollama API'); - } - - this._assertNotTruncated(response.data, numPredict, numCtx); - - return response.data; - } - - /** - * A generation Ollama had to cut short comes back with done_reason "length". - * What it returns is a prefix, and for structured output that means JSON - * without its closing braces — which no amount of sanitizing recovers, so - * there is nothing to do with it but say so. - * - * Two different limits produce the same done_reason and want opposite fixes: - * num_predict is the response budget, but the context window can leave less - * room than that, and then raising Response Tokens changes nothing. The - * message names whichever one actually bit. - * - * @param {Object} responseData - Ollama /api/generate response - * @param {number} numPredict - response token limit that was sent - * @param {number} numCtx - context window that was sent - * @param {Object} [extra] - what the caller salvaged from the cut-off answer - * @param {string} [extra.partialText] - the text that did arrive, reasoning - * stripped; put on the error as `partialText` the way the OpenAI-compatible - * providers do, so a caller can keep it instead of asking twice - */ - _assertNotTruncated(responseData, numPredict, numCtx, extra) { - const evalCount = Number(responseData?.eval_count) || 0; - const promptEvalCount = Number(responseData?.prompt_eval_count) || 0; - - // done_reason is the direct signal. Reaching the limit exactly is the - // fallback for an Ollama old enough not to send one. - const stoppedOnLength = - responseData?.done_reason === 'length' || - (!responseData?.done_reason && numPredict > 0 && evalCount >= numPredict); - - if (!stoppedOnLength) { - return; - } - - /* Which limit bit is read off the count itself rather than guessed from - the context arithmetic: reaching num_predict exactly is the response - budget, stopping short of it means the context filled up first. The - arithmetic route was tried and got it backwards — _calculateNumCtx sizes - the window from an estimate of the prompt (length / 4), so comparing it - against the real prompt_eval_count blames the context for a run that - generated its full budget. */ - const error = - evalCount >= numPredict - ? new Error( - `Ollama stopped generating after ${evalCount} tokens, which is the configured response limit. Raise Response Tokens (RESPONSE_TOKENS) above ${numPredict}.` - ) - : new Error( - `Ollama stopped generating after ${evalCount} of ${numPredict} response tokens: the ${numCtx}-token context window was full, ${promptEvalCount} of it prompt. Raise Token Limit (TOKEN_LIMIT) or shorten the document.` - ); - - /* Carried as a code rather than matched out of the message later. The - phrase-matching in serviceUtils classifies errors that arrive from - elsewhere and have no better handle; this one is raised right here, so - rewording it should not silently reclassify the failure. */ - error.code = 'ai_response_truncated'; - if (typeof extra?.partialText === 'string') { - error.partialText = extra.partialText; - } - throw error; - } - - /** - * Process Ollama API response - * @param {Object} responseData - Ollama API response data - * @returns {Object} Parsed response - */ - _processOllamaResponse(responseData) { - // Check if we got a structured response or need to parse from text - if (responseData.response && typeof responseData.response === 'object') { - // We got a structured response directly - console.log('Using structured output response'); - return { - tags: Array.isArray(responseData.response.tags) - ? responseData.response.tags - : [], - correspondent: responseData.response.correspondent || null, - title: responseData.response.title || null, - document_date: responseData.response.document_date || null, - document_type: responseData.response.document_type || null, - language: responseData.response.language || null, - custom_fields: responseData.response.custom_fields || null, - }; - } else if (responseData.response) { - // Fall back to parsing from text response - console.log('Falling back to text response parsing'); - return this._parseResponse(responseData.response); - } else { - throw new Error('No response data from Ollama API'); - } - } - - /** - * Extract token usage metrics from an Ollama /api/generate response. - * @param {Object} responseData - Ollama API response data - * @returns {{promptTokens:number, completionTokens:number, totalTokens:number}} - */ - _extractOllamaMetrics(responseData) { - const promptTokens = Number(responseData?.prompt_eval_count) || 0; - const completionTokens = Number(responseData?.eval_count) || 0; - - return { - promptTokens, - completionTokens, - totalTokens: promptTokens + completionTokens, - }; - } - - /** - * Parse text response to extract JSON - * @param {string} response - Response text - * @returns {Object} Parsed object - */ - /* Every exit from here either returns something the caller can use or - throws. It used to hand back { tags: [], correspondent: null } on each of - its three failure paths, which is indistinguishable from a model that - genuinely found nothing — so a document whose answer could not be read at - all was written back untouched and then marked processed, never to be - looked at again. openaiService has thrown at the same point all along. */ - _parseResponse(response) { - try { - // Find JSON in response using regex - const jsonMatch = response.match(/\{[\s\S]*\}/); - if (!jsonMatch) { - /* Worded to carry the marker serviceUtils already classifies, so a - document whose answer was unreadable takes the same OCR fallback it - takes under OpenAI. Truncation deliberately does not — see - _assertNotTruncated, where a second pass cannot help. */ - throw new Error( - 'Invalid JSON response from API: no JSON object in the answer' - ); - } - - let jsonStr = jsonMatch[0]; - console.log('Extracted JSON String:', jsonStr); - - try { - // Attempt to parse the JSON - const result = JSON.parse(jsonStr); - - // Validate and return the result - return { - tags: Array.isArray(result.tags) ? result.tags : [], - correspondent: result.correspondent || null, - title: result.title || null, - document_date: result.document_date || null, - document_type: result.document_type || null, - language: result.language || null, - custom_fields: result.custom_fields || null, - }; - } catch (jsonError) { - console.warn('Error parsing JSON from response:', jsonError.message); - console.warn('Attempting to sanitize the JSON...'); - - // Sanitize the JSON - jsonStr = this._sanitizeJsonString(jsonStr); - - try { - const sanitizedResult = JSON.parse(jsonStr); - return { - tags: Array.isArray(sanitizedResult.tags) - ? sanitizedResult.tags - : [], - correspondent: sanitizedResult.correspondent || null, - title: sanitizedResult.title || null, - document_date: sanitizedResult.document_date || null, - language: sanitizedResult.language || null, - }; - } catch { - console.error( - 'Final JSON parsing failed after sanitization. This happens when the JSON structure is too complex or invalid. That indicates an issue with the generated JSON string by Ollama. Switch to OpenAI for better results or fine tune your prompt.' - ); - throw new Error( - `Invalid JSON response from API, unreadable even after sanitizing: ${jsonError.message}` - ); - } - } - } catch (error) { - console.error('Error parsing Ollama response:', error.message); - throw error; - } - } - - /** - * Sanitize a JSON string - * @param {string} jsonStr - JSON string to sanitize - * @returns {string} Sanitized JSON string - */ - _sanitizeJsonString(jsonStr) { - return jsonStr - .replace(/,\s*}/g, '}') // Remove trailing commas before closing braces - .replace(/,\s*]/g, ']') // Remove trailing commas before closing brackets - .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":'); // Ensure property names are quoted - } - - /** - * Log prompt and response to file - * @param {string} prompt - Prompt text - * @param {Object} response - Response object - */ - async _logPromptAndResponse(prompt, response) { - const content = - '================================================================================' + - prompt + - '\n\n' + - JSON.stringify(response) + - '\n\n' + - '================================================================================\n\n'; - - await writePromptToFile(content); - } - - /** - * Generate text based on a prompt - * @param {string} prompt - The prompt to generate text from - * @param {object} [options] - Optional overrides for this one call - * @param {string} [options.systemPrompt] - Replaces the generic system prompt - * @param {number} [options.temperature] - Overrides AI_TEMPERATURE_GENERATION - * @param {number} [options.maxTokens] - Overrides RESPONSE_TOKENS (num_predict) - * @param {AbortSignal} [options.signal] - Cancels the request in flight - * @param {Function} [options.onProgress] - Streams the answer and reports it - * as it grows; see GenerateTextProgress in aiGenerateOptions - * @param {boolean} [options.reasoning] - Sends `think` for this one call and - * overrules OLLAMA_THINK; without it the setting decides, as before - * @returns {Promise} - The generated text - */ - async generateText(prompt, options = {}) { - // Read before the try so the catch can tell a stopped request from a - // failed one, and so a cut-off request keeps what it measured. - const signal = abortSignal(options); - const onProgress = progressHandler(options); - let measuredUsage = null; - - try { - const responseTokens = - hasNumber(options.maxTokens) && options.maxTokens > 0 - ? Math.floor(options.maxTokens) - : Number(config.responseTokens); - - // Calculate context window size based on prompt length - const promptTokenCount = this._calculatePromptTokenCount(prompt); - const numCtx = this._calculateNumCtx(promptTokenCount, responseTokens); - - // Simple system prompt for text generation; a caller that knows the job - // better (the Duplicates review) brings its own. - const systemPrompt = hasSystemPrompt(options) - ? options.systemPrompt - : `You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.`; - - // Call Ollama API without enforcing a specific response format - const generateTextBody = { - model: modelOverride(options) || this.model, - prompt: prompt, - system: systemPrompt, - stream: false, - options: { - temperature: hasNumber(options.temperature) - ? options.temperature - : config.aiTemperatureGeneration, - top_p: 0.9, - // The line above reserves the same number in the context; a - // constant here would have the same quarrel with it that the - // analysis path had. - num_predict: responseTokens, - num_ctx: numCtx, - }, - }; - - // A caller that says what it wants overrules the setting; without one - // OLLAMA_THINK decides, which is what it did before. - const reasoning = reasoningEnabled(options); - if (reasoning !== null) { - generateTextBody.think = reasoning; - } else if (!config.ollama.think) { - // Disable thinking for reasoning models (e.g. Qwen3, DeepSeek-R1) - // unless explicitly enabled via OLLAMA_THINK=true - generateTextBody.think = false; - } - - // axios takes the signal in the request config; without one the config - // is the headers it always was. - const requestConfig = { headers: this._buildRequestHeaders() }; - if (signal) requestConfig.signal = signal; - - if (onProgress) { - return await this._generateTextStreamed( - generateTextBody, - requestConfig, - { signal, onProgress, numPredict: responseTokens, numCtx }, - (usage) => { - measuredUsage = usage; - this.lastGenerateTextUsage = usage; - } - ); - } - - const response = await this.client.post( - `${this.apiUrl}/api/generate`, - generateTextBody, - requestConfig - ); - - if (!response.data || !response.data.response) { - throw new Error('Invalid response from Ollama API'); - } - - // Ollama reports its token counts on the answer itself; the callers - // that care (the Duplicates review) read them off the service after - // the call, so the return value stays a plain string. - const promptTokens = Number(response.data.prompt_eval_count); - const completionTokens = Number(response.data.eval_count); - measuredUsage = - Number.isFinite(promptTokens) || Number.isFinite(completionTokens) - ? { - promptTokens: Number.isFinite(promptTokens) ? promptTokens : null, - completionTokens: Number.isFinite(completionTokens) - ? completionTokens - : null, - totalTokens: - (Number.isFinite(promptTokens) ? promptTokens : 0) + - (Number.isFinite(completionTokens) ? completionTokens : 0), - } - : null; - this.lastGenerateTextUsage = measuredUsage; - - // A model that writes its reasoning into the answer wrote it for - // itself; only what it said afterwards is the answer. Anything that is - // not a string is left alone — it is a structured answer, not text. - const generatedText = - typeof response.data.response === 'string' - ? stripReasoningText(response.data.response).trim() - : response.data.response; - - // The cut-off check the analysis path has always run, now on this path - // too, and carrying the prefix the caller can still use. - this._assertNotTruncated(response.data, responseTokens, numCtx, { - partialText: typeof generatedText === 'string' ? generatedText : '', - }); - - return generatedText; - } catch (error) { - // Whatever was measured before the failure stays readable; a failure - // that measured nothing still leaves null behind, as it always did. - this.lastGenerateTextUsage = measuredUsage; - // One name for a stopped request, whatever axios called it; an abort - // this service raised already carries its partial text. - const thrown = - isAbortError(error, signal) && error?.name !== 'AbortError' - ? abortedGenerationError(error, '') - : error; - console.error(`Error generating text with Ollama: ${thrown.message}`); - console.debug(thrown); - throw thrown; - } - } - - /** - * The streamed half of generateText(): the same request with `stream: true`, - * read as the NDJSON lines Ollama answers with. - * - * Ollama does not speak server-sent events — every line is a complete JSON - * object with the next piece of the answer in `response`, the next piece of - * the model's thinking in `thinking`, and on the last line the counts and - * the reason it stopped. Lines arrive split across socket reads, so the tail - * of a read is kept until its newline turns up. - * - * @param {Object} body - the request body, `stream` still to be set - * @param {Object} requestConfig - headers, and the signal if there is one - * @param {Object} context - * @param {AbortSignal|null} context.signal - * @param {Function} context.onProgress - * @param {number} context.numPredict - response budget that was sent - * @param {number} context.numCtx - context window that was sent - * @param {Function} onUsage - told before a cut-off throws - * @returns {Promise} the answer, reasoning stripped - */ - async _generateTextStreamed(body, requestConfig, context, onUsage) { - const { signal, onProgress, numPredict, numCtx } = context; - const collector = createProgressCollector({ onProgress }); - let last = {}; - - try { - const response = await this.client.post( - `${this.apiUrl}/api/generate`, - { ...body, stream: true }, - { ...requestConfig, responseType: 'stream' } - ); - - const stream = response.data; - // An abort tears the socket down, and a socket error that arrives after - // the loop has left would be an unhandled 'error' event — which ends - // the process rather than the request. - stream.on('error', () => {}); - let pending = ''; - const takeLine = (line) => { - if (line.trim() === '') return; - let parsed; - try { - parsed = JSON.parse(line); - } catch { - // A line that is not JSON is a line this client cannot use; the - // stream goes on, and the answer is whatever the rest carries. - return; - } - collector.pushContent(parsed.response); - collector.pushReasoning(parsed.thinking); - if (parsed.done === true) last = parsed; - collector.report(); - }; - - for await (const piece of stream) { - if (signal?.aborted) { - // Nothing more is wanted, so nothing more is read: the socket goes - // now rather than after the model finished talking to itself. - stream.destroy(); - break; - } - pending += piece.toString('utf8'); - let at = pending.indexOf('\n'); - while (at !== -1) { - takeLine(pending.slice(0, at)); - pending = pending.slice(at + 1); - at = pending.indexOf('\n'); - } - } - if (!signal?.aborted) takeLine(pending); - - if (signal?.aborted) { - collector.finish(); - throw abortedGenerationError(null, collector.text()); - } - } catch (error) { - collector.finish(); - if (error?.name === 'AbortError') throw error; - if (isAbortError(error, signal)) { - throw abortedGenerationError(error, collector.text()); - } - throw error; - } - - collector.finish(); - - const promptTokens = Number(last.prompt_eval_count); - const completionTokens = Number(last.eval_count); - const measured = - Number.isFinite(promptTokens) || Number.isFinite(completionTokens) - ? { - promptTokens: Number.isFinite(promptTokens) ? promptTokens : null, - completionTokens: Number.isFinite(completionTokens) - ? completionTokens - : null, - totalTokens: - (Number.isFinite(promptTokens) ? promptTokens : 0) + - (Number.isFinite(completionTokens) ? completionTokens : 0), - } - : collector.usageSummary(); - if (typeof onUsage === 'function') onUsage(measured); - - this._assertNotTruncated(last, numPredict, numCtx, { - partialText: collector.text(), - }); - - return collector.text(); - } - - /** - * Check if the Ollama service is running - * @returns {Promise} - True if the service is running, false otherwise - */ - async checkStatus() { - // use ollama status endpoint - try { - const response = await this.client.get(`${this.apiUrl}/api/ps`, { - headers: this._buildRequestHeaders(), - }); - if (response.status === 200) { - const data = response.data; - // Ensure data is an array and has at least one model - let modelName = null; - if (Array.isArray(data.models) && data.models.length > 0) { - modelName = data.models[0].name; - } - console.log('Ollama model name:', modelName); - return { status: 'ok', model: modelName }; - } - } catch (error) { - console.error(`Error checking Ollama service status: ${error.message}`); - console.debug(error); - } - return { status: 'error' }; - } -} - -module.exports = new OllamaService(); +// The Ollama provider is a composition root over the shared DocumentAnalyzer +// (services/llm/). This path stays so its callers need no change. +module.exports = require('./llm/providers/ollamaProvider'); diff --git a/services/openaiService.js b/services/openaiService.js index 1816a50a..7c0ba525 100644 --- a/services/openaiService.js +++ b/services/openaiService.js @@ -1,570 +1,3 @@ -const { - calculateTokens, - calculateTotalPromptTokens, - truncateToTokenLimit, - writePromptToFile, - extractChatMessageContent, - assertCompletionNotTruncated, - toNameList, -} = require('./serviceUtils'); -const { - abortSignal, - abortedGenerationError, - hasNumber, - hasSystemPrompt, - isAbortError, - modelOverride, - progressHandler, - readCompletionUsage, - reasoningEffortForOpenAi, - reasoningEnabled, - runChatCompletionStream, - stripReasoningText, -} = require('./aiGenerateOptions'); -const OpenAI = require('openai'); -const config = require('../config/config'); -const paperlessService = require('./paperlessService'); -const fs = require('fs').promises; -const path = require('path'); -const { - THUMBNAIL_CACHE_DIR, - getThumbnailCachePath, -} = require('./thumbnailCachePaths'); -const RestrictionPromptService = require('./restrictionPromptService'); -const responseLogPath = path.join( - process.cwd(), - 'data', - 'logs', - 'response.txt' -); -/** What an operator can do about an answer that hit the completion cap. */ -const TRUNCATION_REMEDY = - 'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.'; - -class OpenAIService { - constructor() { - this.client = null; - } - - initialize() { - if (!this.client && config.aiProvider === 'ollama') { - this.client = new OpenAI({ - baseURL: config.ollama.apiUrl + '/v1', - // Ollama ignores the key unless the endpoint enforces auth (e.g. a - // reverse proxy); the OpenAI SDK requires a non-empty value. - apiKey: config.ollama.apiKey || 'ollama', - }); - } else if (!this.client && config.aiProvider === 'custom') { - this.client = new OpenAI({ - baseURL: config.custom.apiUrl, - apiKey: config.custom.apiKey, - }); - } else if (!this.client && config.aiProvider === 'openai') { - if (!this.client && config.openai.apiKey) { - this.client = new OpenAI({ - apiKey: config.openai.apiKey, - }); - } - } - } - - async analyzeDocument( - content, - existingTags = [], - existingCorrespondentList = [], - existingDocumentTypesList = [], - id, - customPrompt = null, - options = {} - ) { - const cachePath = getThumbnailCachePath(id); - try { - this.initialize(); - const now = new Date(); - const timestamp = now.toLocaleString('de-DE', { - dateStyle: 'short', - timeStyle: 'short', - }); - - if (!this.client) { - throw new Error('OpenAI client not initialized'); - } - - // Handle thumbnail caching - try { - await fs.access(cachePath); - console.log('[DEBUG] Thumbnail already cached'); - } catch { - console.log('Thumbnail not cached, fetching from Paperless'); - - const thumbnailData = await paperlessService.getThumbnailImage(id); - - if (!thumbnailData) { - console.warn('Thumbnail not found'); - return; - } - - await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); - await fs.writeFile(cachePath, thumbnailData); - } - - // Format existing data - callers may hand over entity objects or plain names - const existingTagNames = toNameList(existingTags).join(', '); - const existingCorrespondentNames = toNameList( - existingCorrespondentList - ).join(', '); - const existingDocumentTypeNames = toNameList( - existingDocumentTypesList - ).join(', '); - - // Get external API data if available and validate it - let externalApiData = options.externalApiData || null; - let validatedExternalApiData = null; - - if (externalApiData) { - try { - validatedExternalApiData = - await this._validateAndTruncateExternalApiData(externalApiData); - console.log('[DEBUG] External API data validated and included'); - } catch (error) { - console.warn( - '[WARNING] External API data validation failed:', - error.message - ); - validatedExternalApiData = null; - } - } - - let systemPrompt = ''; - let promptTags = ''; - const model = process.env.OPENAI_MODEL; - - // Parse CUSTOM_FIELDS from environment variable - let customFieldsObj; - try { - customFieldsObj = JSON.parse(process.env.CUSTOM_FIELDS); - } catch (error) { - console.error(`Failed to parse CUSTOM_FIELDS: ${error.message}`); - console.debug(error); - customFieldsObj = { custom_fields: [] }; - } - - // Generate custom fields template for the prompt - const customFieldsTemplate = {}; - - customFieldsObj.custom_fields.forEach((field, index) => { - let valueHint; - if (field.data_type === 'date') { - valueHint = - 'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis'; - } else if (field.data_type === 'boolean') { - valueHint = "Fill in 'true' or 'false' based on your analysis"; - } else { - valueHint = 'Fill in the value based on your analysis'; - } - customFieldsTemplate[index] = { - field_name: field.value, - value: valueHint, - }; - }); - - // Convert template to string for replacement and wrap in custom_fields - const customFieldsStr = - '"custom_fields": ' + - JSON.stringify(customFieldsTemplate, null, 2) - .split('\n') - .map((line) => ' ' + line) // Add proper indentation - .join('\n'); - - // Get system prompt and model - if ( - config.useExistingData === 'yes' && - config.restrictToExistingTags === 'no' && - config.restrictToExistingCorrespondents === 'no' - ) { - systemPrompt = - ` - Pre-existing tags: ${existingTagNames}\n\n - Pre-existing correspondents: ${existingCorrespondentNames}\n\n - Pre-existing document types: ${existingDocumentTypeNames}\n\n - ` + - process.env.SYSTEM_PROMPT + - '\n\n' + - config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr); - promptTags = ''; - } else { - const mustHavePrompt = config.mustHavePrompt.replace( - '%CUSTOMFIELDS%', - customFieldsStr - ); - systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + mustHavePrompt; - promptTags = ''; - } - - // Process placeholder replacements in system prompt - systemPrompt = RestrictionPromptService.processRestrictionsInPrompt( - systemPrompt, - existingTags, - existingCorrespondentList, - existingDocumentTypesList - ); - - // Include validated external API data if available - if (validatedExternalApiData) { - systemPrompt += `\n\nAdditional context from external API:\n${validatedExternalApiData}`; - } - - if (process.env.USE_PROMPT_TAGS === 'yes') { - promptTags = process.env.PROMPT_TAGS; - systemPrompt = - ` - Take these tags and try to match one or more to the document content.\n\n - ` + config.specialPromptPreDefinedTags; - } - - if (customPrompt) { - console.log( - '[DEBUG] Replace system prompt with custom prompt via WebHook' - ); - systemPrompt = - customPrompt + - '\n\n' + - config.mustHavePrompt.replace('%CUSTOMFIELDS%', customFieldsStr); - } - - // Calculate tokens AFTER all prompt modifications are complete - const totalPromptTokens = await calculateTotalPromptTokens( - systemPrompt, - process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [], - model - ); - - const maxTokens = Number(config.tokenLimit); - const reservedTokens = totalPromptTokens + Number(config.responseTokens); - const availableTokens = maxTokens - reservedTokens; - - // Validate that we have positive available tokens. A non-finite budget - // is the same failure: it slips past a bare `<= 0` and truncates the - // document down to nothing instead of stopping. - if (!Number.isFinite(availableTokens) || availableTokens <= 0) { - console.warn( - `[WARNING] No available tokens for content. Reserved: ${reservedTokens}, Max: ${maxTokens}` - ); - throw new Error( - 'Token limit exceeded: prompt too large for available token limit' - ); - } - - console.log( - `[DEBUG] Token calculation - Prompt: ${totalPromptTokens}, Reserved: ${reservedTokens}, Available: ${availableTokens}` - ); - console.log( - `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting` - ); - console.log( - `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}` - ); - - const truncatedContent = await truncateToTokenLimit( - content, - availableTokens, - model - ); - - await writePromptToFile(systemPrompt, truncatedContent); - - const response = await this.client.chat.completions.create({ - model: model, - messages: [ - { - role: 'system', - content: systemPrompt, - }, - { - role: 'user', - content: truncatedContent, - }, - ], - ...(model !== 'o3-mini' && { - temperature: config.aiTemperatureAnalysis, - }), - }); - - assertCompletionNotTruncated( - response, - 'OpenAI', - 'Reduce the number of custom fields the prompt asks for, or use a model with a larger completion limit.' - ); - const message = response?.choices?.[0]?.message; - let jsonContent = extractChatMessageContent(message, 'OpenAI'); - if (!jsonContent) { - throw new Error('Invalid API response structure'); - } - - console.log(`[DEBUG] [${timestamp}] OpenAI request sent`); - console.log( - `[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}` - ); - - const usage = response.usage; - const mappedUsage = { - promptTokens: usage.prompt_tokens, - completionTokens: usage.completion_tokens, - totalTokens: usage.total_tokens, - }; - - // Strip ... reasoning tags from models like Qwen3, DeepSeek-R1 - jsonContent = jsonContent.replace(/[\s\S]*?<\/think>/g, ''); - jsonContent = jsonContent - .replace(/```json\n?/g, '') - .replace(/```\n?/g, '') - .trim(); - - let parsedResponse; - try { - parsedResponse = JSON.parse(jsonContent); - } catch (error) { - console.error(`Failed to parse JSON response: ${error.message}`); - console.debug(error); - // Check if the response indicates the content is too minimal or API can't process it - if ( - jsonContent && - (jsonContent.toLowerCase().includes("i'm sorry") || - jsonContent.toLowerCase().includes('i cannot') || - jsonContent.toLowerCase().includes('insufficient')) - ) { - console.warn(`Document ${id} has insufficient content for analysis`); - // Return a default structure instead of throwing to prevent retry loops - return { - document: { - tags: [], - correspondent: 'Unknown', - title: `Document ${id}`, - document_date: new Date().toISOString().split('T')[0], - document_type: 'Document', - language: 'und', - }, - metrics: mappedUsage, - truncated: false, - error: 'Insufficient content for AI analysis', - }; - } - throw new Error('Invalid JSON response from API', { cause: error }); - } - - try { - await fs.mkdir(path.dirname(responseLogPath), { recursive: true }); - await fs.appendFile(responseLogPath, `${jsonContent}\n`); - } catch (logError) { - console.warn('Failed to write AI response log:', logError.message); - } - - if ( - !parsedResponse || - !Array.isArray(parsedResponse.tags) || - (typeof parsedResponse.correspondent !== 'string' && - parsedResponse.correspondent !== null) - ) { - throw new Error( - 'AI could not determine assignable metadata: no tags or correspondent found' - ); - } - - return { - document: parsedResponse, - metrics: mappedUsage, - truncated: truncatedContent.length < content.length, - }; - } catch (error) { - console.error(`Failed to analyze document: ${error.message}`); - console.debug(error); - return { - document: { tags: [], correspondent: null }, - metrics: null, - error: error.message, - // Undefined for everything that is not one of ours; the scan loop - // falls back to its generic reason then. - errorCode: error.code, - }; - } - } - - /** - * Validate and truncate external API data to prevent token overflow - * @param {any} apiData - The external API data to validate - * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500) - * @returns {string} - Validated and potentially truncated data string - */ - async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) { - if (!apiData) { - return null; - } - - const dataString = - typeof apiData === 'object' - ? JSON.stringify(apiData, null, 2) - : String(apiData); - - // Calculate tokens for the data - const dataTokens = await calculateTokens( - dataString, - process.env.OPENAI_MODEL - ); - - if (dataTokens > maxTokens) { - console.warn( - `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating` - ); - return await truncateToTokenLimit( - dataString, - maxTokens, - process.env.OPENAI_MODEL - ); - } - - console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`); - return dataString; - } - - /** - * Generate text based on a prompt - * @param {string} prompt - The prompt to generate text from - * @param {object} [options] - Optional overrides for this one call - * @param {string} [options.systemPrompt] - Sent as the system message - * @param {number} [options.temperature] - Overrides AI_TEMPERATURE_GENERATION - * @param {number} [options.maxTokens] - Completion cap for this call - * @param {AbortSignal} [options.signal] - Cancels the request in flight - * @param {Function} [options.onProgress] - Streams the answer and reports it - * as it grows; see GenerateTextProgress in aiGenerateOptions - * @param {boolean} [options.reasoning] - false sends reasoning_effort on the - * model families that take it and nothing on the others, which have no - * thinking switch; true and null send nothing - * @returns {Promise} - The generated text - */ - async generateText(prompt, options = {}) { - // Read before the try so the catch can tell a stopped request from a - // failed one. - const signal = abortSignal(options); - const onProgress = progressHandler(options); - - try { - this.initialize(); - - if (!this.client) { - throw new Error('OpenAI client not initialized - missing API key'); - } - - const model = - modelOverride(options) || - process.env.OPENAI_MODEL || - config.openai.model; - const messages = []; - if (hasSystemPrompt(options)) { - messages.push({ role: 'system', content: options.systemPrompt }); - } - messages.push({ role: 'user', content: prompt }); - - const request = { - model: model, - messages, - temperature: hasNumber(options.temperature) - ? options.temperature - : config.aiTemperatureGeneration, - }; - // Without a cap the model answers with whatever the deployment allows, - // which is what every caller before the AI review wanted. - if (hasNumber(options.maxTokens) && options.maxTokens > 0) { - request.max_tokens = Math.floor(options.maxTokens); - } - // The one thinking switch this API has, and it exists on the reasoning - // families only: an ordinary model answers 400 for the field, so it is - // sent for those names and for no others, and only when the caller - // asked for no thinking. - if (reasoningEnabled(options) === false) { - const effort = reasoningEffortForOpenAi(model); - if (effort) request.reasoning_effort = effort; - } - - this.lastGenerateTextUsage = null; - // The second argument is the SDK's request option bag; a caller that - // brought no signal gets the call it always got. - const requestOptions = signal ? { signal } : null; - - if (onProgress) { - // include_usage buys the exact completion count at the end of the - // stream; without it the report is an estimate all the way. - return await runChatCompletionStream({ - client: this.client, - request: { - ...request, - stream: true, - stream_options: { include_usage: true }, - }, - requestOptions, - signal, - onProgress, - provider: 'OpenAI', - remedy: TRUNCATION_REMEDY, - onUsage: (usage) => { - this.lastGenerateTextUsage = usage; - }, - }); - } - - const response = requestOptions - ? await this.client.chat.completions.create(request, requestOptions) - : await this.client.chat.completions.create(request); - this.lastGenerateTextUsage = readCompletionUsage(response); - - const message = response?.choices?.[0]?.message; - // Reasoning is no part of the answer, and a block the cut-off left - // open is no part of it either. - const generatedText = extractChatMessageContent( - { ...message, content: stripReasoningText(message?.content) }, - 'OpenAI' - ); - - // After the text is in hand: a cut-off answer is still a failure, but - // the caller can keep the prefix instead of paying for it twice. - assertCompletionNotTruncated(response, 'OpenAI', TRUNCATION_REMEDY, { - partialText: generatedText, - }); - - if (!generatedText) { - throw new Error('Invalid API response structure'); - } - - return generatedText; - } catch (error) { - // One name for a stopped request, whatever the client called it; an - // abort this service raised already carries its partial text. - const thrown = - isAbortError(error, signal) && error?.name !== 'AbortError' - ? abortedGenerationError(error, '') - : error; - console.error(`Error generating text with OpenAI: ${thrown.message}`); - console.debug(thrown); - throw thrown; - } - } - - async checkStatus() { - // Use a token-free endpoint for connectivity/auth checks. - try { - this.initialize(); - - if (!this.client) { - throw new Error('OpenAI client not initialized - missing API key'); - } - - await this.client.models.list(); - return { status: 'ok', model: process.env.OPENAI_MODEL }; - } catch (error) { - console.error(`Error checking OpenAI status: ${error.message}`); - console.debug(error); - return { status: 'error', error: error.message }; - } - } -} - -module.exports = new OpenAIService(); +// The OpenAI provider is a composition root over the shared DocumentAnalyzer +// (services/llm/). This path stays so its callers need no change. +module.exports = require('./llm/providers/openaiProvider'); diff --git a/services/paperlessService.js b/services/paperlessService.js index 05f0425e..4e14ce4f 100644 --- a/services/paperlessService.js +++ b/services/paperlessService.js @@ -7,6 +7,7 @@ const { stripTrailingSlashes, createRedirectGuard, } = require('./serviceUtils'); +const { CachedNameIdEnum } = require('./CachedNameIdEnum'); // Pure module, no side effects and no requires of its own: safe to load here // even though half the app loads this service. const entityNameMatcher = require('./entityNameMatcher'); @@ -177,16 +178,40 @@ function requestTimeoutMs() { return seconds * 1000; } +/** + * Wraps a failed correspondent/document-type creation once race-condition + * recovery could not resolve it. Throwing a fresh, contextualised error + * instead of rethrowing the caught one keeps the outer handler free of + * local-rethrow patterns and carries the entity context to the caller. + */ +class EntityCreationError extends Error { + /** + * @param {string} entityType - 'correspondent' or 'document type' + * @param {string} name - The name that was being created + * @param {Error} cause - The original API error + */ + constructor(entityType, name, cause) { + super(`Failed to process ${entityType} "${name}": ${cause.message}`, { + cause, + }); + this.name = 'EntityCreationError'; + this.entityType = entityType; + } +} + +/** True when Paperless-ngx rejected a create as a duplicate. */ +const isUniqueConstraintError = (error) => + error?.response?.status === 400 && + String(error?.response?.data?.error || '').includes('unique constraint'); + class PaperlessService { constructor() { this.client = null; - this.tagCache = new Map(); this.customFieldCache = new Map(); this.correspondentNameCache = new Map(); - this.lastTagRefresh = 0; + this.lastCorrespondentRefresh = 0; this.lastCorrespondentRefresh = 0; this._supportsCorrespondentIdIn = null; - this._refreshPromise = null; this._effectiveCountCache = null; this._effectiveCountCacheTtlMs = 60 * 1000; this._supportsTagsIdNone = null; @@ -199,6 +224,9 @@ class PaperlessService { // Dynamic cache lifetime from config (default: 5 minutes) // Lazy load to avoid circular dependency this._cacheTTL = null; + this._correspondents = null; + this._documentTypes = null; + this._tags = null; /** * What the creation guard compares a proposed name against, per kind: * kind -> { records: EntityRecord[], at: number }. @@ -218,6 +246,65 @@ class PaperlessService { this._documentTypeRefreshPromise = null; } + get correspondents() { + if (this._correspondents === null) { + const aiServiceFactory = require('./aiServiceFactory'); + const aiService = aiServiceFactory.getService(); + this._correspondents = new CachedNameIdEnum( + 'correspondent', + '/correspondents/', + 'Ignore case and consider typos, misspellings, and alternative spellings, misplaced Address data, seasonal ads,' + + 'abbreviations of organization form etc.. ', + () => { + return this.CACHE_LIFETIME; + }, + aiService + ); + } + return this._correspondents; + } + + get documentTypes() { + if (this._documentTypes === null) { + const aiServiceFactory = require('./aiServiceFactory'); + const aiService = aiServiceFactory.getService(); + this._documentTypes = new CachedNameIdEnum( + 'document_type', + '/document_types/', + 'Ignore case. Ignore special characters. consider typos, misspellings, alternative spellings, ' + + 'same words in a different language. Especially check if it is a synonyme for a existing document type', + () => { + return this.CACHE_LIFETIME; + }, + aiService + ); + } + return this._documentTypes; + } + + // Tags share the enum machinery with correspondents and document types. + // The larger page size matters here: tags are the one all-or-nothing read + // everything else queues behind, and a large library at the DRF default of + // 25 per page once stalled startup for a minute (54 sequential requests). + get tags() { + if (this._tags === null) { + const aiServiceFactory = require('./aiServiceFactory'); + const aiService = aiServiceFactory.getService(); + this._tags = new CachedNameIdEnum( + 'tag', + '/tags/', + 'Ignore case and consider typos, misspellings, singular vs plural forms, ' + + 'abbreviations and compound words. Only match a tag that clearly means the same thing', + () => { + return this.CACHE_LIFETIME; + }, + aiService, + TAG_PAGE_SIZE + ); + } + return this._tags; + } + get CACHE_LIFETIME() { if (this._cacheTTL === null) { // Re-require config at runtime to pick up dynamic configuration values without shadowing the top-level import @@ -455,100 +542,34 @@ class PaperlessService { } } - // Aktualisiert den Tag-Cache, wenn er älter als CACHE_LIFETIME ist + /** + * Make sure the tag cache is populated and not expired. Kept as a named + * seam because several call sites and tests speak in these terms; the + * freshness logic itself lives in CachedNameIdEnum. + */ async ensureTagCache() { - const now = Date.now(); - const cacheAge = now - this.lastTagRefresh; - if (this.tagCache.size === 0 || cacheAge > this.CACHE_LIFETIME) { - if (this._refreshPromise) { - return this._refreshPromise; - } - const ttlSeconds = Math.floor(this.CACHE_LIFETIME / 1000); - // A cache that was never filled has no age and no expiry. Dating it from - // the epoch reported "age: 1786738904s ... expired at: 1970-01-01" on - // every cold start, which reads as a clock problem rather than a first - // run. - console.log( - this.lastTagRefresh === 0 - ? `[DEBUG] Tag cache empty, building it (TTL: ${ttlSeconds}s)` - : `[DEBUG] Tag cache expired (age: ${Math.floor(cacheAge / 1000)}s, TTL: ${ttlSeconds}s, expired at: ${new Date( - this.lastTagRefresh + this.CACHE_LIFETIME - ).toISOString()})` - ); - // No race condition: synchronous code is never preempted in Node.js's - // event loop, so no other call can reach here between the check above - // and the assignment below. - this._refreshPromise = this.refreshTagCache().finally(() => { - this._refreshPromise = null; - }); - return this._refreshPromise; + if (!this.client) { + return; } + await this.tags.getCount(this.client); } /** - * Manually clear the tag cache. + * Manually clear (and rebuild on next access) the tag cache. * Useful for forcing a refresh after external tag modifications. */ clearTagCache() { + this.initialize(); console.log('[DEBUG] Manually clearing tag cache...'); - this.tagCache.clear(); - this.lastTagRefresh = 0; + this._tags?.invalidate(); + this._correspondents?.invalidate(); console.log('[DEBUG] Tag cache cleared.'); } - // Lädt alle existierenden Tags async refreshTagCache() { - try { - console.log('[DEBUG] Refreshing tag cache...'); - this.tagCache.clear(); - // The page size only has to be asked for once: Paperless-ngx builds its - // `next` link from the request URL, so every following page carries it. - let nextUrl = `/tags/?page_size=${TAG_PAGE_SIZE}`; - let reported = null; - while (nextUrl) { - const response = await this.client.get(nextUrl); - if (Number.isInteger(response?.data?.count)) { - reported = response.data.count; - } - - // Validate response structure - if (!response?.data?.results) { - console.error( - '[ERROR] Invalid response structure from API:', - response?.data - ); - break; - } - - response.data.results.forEach((tag) => { - this.tagCache.set(tag.name.toLowerCase(), tag); - }); - - // Safely extract relative path from next URL to prevent SSRF - if (response.data.next) { - nextUrl = this._safeExtractRelativePath(response.data.next); - if (nextUrl) { - console.log('[DEBUG] Next page URL:', nextUrl); - } - } else { - nextUrl = null; - } - } - this.lastTagRefresh = Date.now(); - // The cache is keyed by the lower-cased name, so two tags that differ - // only in case share one entry; the count Paperless-ngx reports says - // how many that hides. - const hidden = - reported != null && reported !== this.tagCache.size - ? ` (Paperless-ngx counts ${reported}; names that differ only in case share one entry)` - : ''; - console.log( - `[DEBUG] Tag cache refreshed. Found ${this.tagCache.size} tags${hidden}.` - ); - } catch (error) { - console.error('[ERROR] refreshing tag cache:', error.message); - throw error; - } + this.initialize(); + console.log('[DEBUG] Refreshing tag cache...'); + await this.tags.flush_cache(this.client); } async initializeWithCredentials(apiUrl, apiToken) { @@ -578,11 +599,30 @@ class PaperlessService { } async createCustomFieldSafely(fieldName, fieldType, default_currency) { + const normalizedFieldType = String(fieldType || '').trim(); + + // Check an already existing field before attempting creation. Paperless + // returns 400 for duplicate names, so without this check a configured + // type mismatch (e.g. string vs. longtext) would be silently ignored. + const existingField = await this.findExistingCustomField(fieldName); + if (existingField) { + const existingFieldType = String(existingField.data_type || '').trim(); + if (existingFieldType !== normalizedFieldType) { + const error = new Error( + `Custom field "${fieldName}" has type "${existingFieldType}" in Paperless, ` + + `but configuration requests "${normalizedFieldType}".` + ); + error.code = 'CUSTOM_FIELD_TYPE_MISMATCH'; + throw error; + } + return existingField; + } + try { // Try to create the field first const response = await this.client.post('/custom_fields/', { name: fieldName, - data_type: fieldType, + data_type: normalizedFieldType, extra_data: { default_currency: default_currency || null, }, @@ -598,6 +638,17 @@ class PaperlessService { await this.refreshCustomFieldCache(); const existingField = await this.findExistingCustomField(fieldName); if (existingField) { + const existingFieldType = String( + existingField.data_type || '' + ).trim(); + if (existingFieldType !== normalizedFieldType) { + const mismatchError = new Error( + `Custom field "${fieldName}" has type "${existingFieldType}", ` + + `but configuration requests "${normalizedFieldType}".` + ); + mismatchError.code = 'CUSTOM_FIELD_TYPE_MISMATCH'; + throw mismatchError; + } return existingField; } } @@ -657,7 +708,7 @@ class PaperlessService { ); return response.data.custom_fields || []; } catch (error) { - console.error('[ERROR] fetching document %s:', documentId, error.message); + console.error(`[ERROR] fetching document ${documentId}:`, error.message); return []; } } @@ -690,8 +741,7 @@ class PaperlessService { } } catch (error) { console.warn( - '[ERROR] searching for custom field "%s":', - fieldName, + `[ERROR] searching for custom field "${fieldName}":`, error.message ); } @@ -741,67 +791,36 @@ class PaperlessService { } async findExistingTag(tagName) { - const normalizedName = tagName.toLowerCase(); - - // 1. Zuerst im Cache suchen - const cachedTag = this.tagCache.get(normalizedName); - if (cachedTag) { - console.log( - `[DEBUG] Found tag "${tagName}" in cache with ID ${cachedTag.id}` - ); - return cachedTag; - } - - // 2. Direkte API-Suche - try { - const response = await this.client.get('/tags/', { - params: { - name__iexact: normalizedName, // Case-insensitive exact match - }, - }); - - if (response.data.results.length > 0) { - const foundTag = response.data.results[0]; - console.log( - `[DEBUG] Found existing tag "${tagName}" via API with ID ${foundTag.id}` - ); - this.tagCache.set(normalizedName, foundTag); - return foundTag; - } - } catch (error) { - console.warn('[ERROR] searching for tag "%s":', tagName, error.message); + if (!tagName || typeof tagName !== 'string') { + return null; } - - return null; + // Delegates to the enum: exact match first, then substring/term/AI + // similarity so a near-duplicate name resolves to the existing tag + // instead of creating one. + return this.tags.getByName(this.client, tagName); } async createTagSafely(tagName) { - const normalizedName = tagName.toLowerCase(); - try { - // Versuche zuerst, den Tag zu erstellen const response = await this.client.post('/tags/', { name: tagName }); const newTag = response.data; console.log( `[DEBUG] Successfully created tag "${tagName}" with ID ${newTag.id}` ); - this.tagCache.set(normalizedName, newTag); - // Invalidate cache after creating new tag to ensure consistency - this.lastTagRefresh = 0; + // Insert into the cache without a full refresh; the enum's maps stay + // consistent for the lookups that follow in the same run. + this.tags.add(newTag); return newTag; } catch (error) { if (error.response?.status === 400) { - // Bei einem 400er Fehler könnte der Tag bereits existieren - // Aktualisiere den Cache und suche erneut await this.refreshTagCache(); - // Suche nochmal nach dem Tag const existingTag = await this.findExistingTag(tagName); if (existingTag) { return existingTag; } } - throw error; // Wenn wir den Tag nicht finden konnten, werfen wir den Fehler weiter + throw error; } } @@ -914,7 +933,7 @@ class PaperlessService { processedTags.add(normalizedName); } } catch (error) { - console.error('[ERROR] processing tag "%s":', tagName, error.message); + console.error(`[ERROR] processing tag "${tagName}":`, error.message); errors.push({ tagName, error: error.message }); } } @@ -968,11 +987,11 @@ class PaperlessService { return []; } - // Use cached tags if available and not expired - await this.ensureTagCache(); - return Array.from(this.tagCache.values()); + // Cached, freshness-checked read; elements are reduced to {id, name}. + return this.tags.getAll(this.client); } + //todo /** * Fetch tags directly from API, bypassing cache. * Use only when fresh data is absolutely required. @@ -1044,10 +1063,9 @@ class PaperlessService { async getTagCount({ strict = false } = {}) { this.initialize(); try { - const response = await this.client.get('/tags/', { - params: { count: true }, - }); - return response.data.count; + // Served from the shared tag cache like the correspondent count; the + // strict flag keeps a dead backend from caching an all-zero library. + return await this.tags.getCount(this.client); } catch (error) { console.error('[ERROR] fetching tag count:', error.message); if (strict) throw error; @@ -1062,10 +1080,7 @@ class PaperlessService { async getCorrespondentCount({ strict = false } = {}) { this.initialize(); try { - const response = await this.client.get('/correspondents/', { - params: { count: true }, - }); - return response.data.count; + return await this.correspondents.getCount(this.client); } catch (error) { console.error('[ERROR] fetching correspondent count:', error.message); if (strict) throw error; @@ -1105,43 +1120,21 @@ class PaperlessService { async listCorrespondentsNames() { this.initialize(); - let allCorrespondents = []; - let page = 1; - let hasNextPage = true; - try { - while (hasNextPage) { - const response = await this.client.get('/correspondents/', { - params: { - fields: 'id,name', - count: true, - page: page, - page_size: 100, - }, - }); - - const { results, next } = response.data; - - // Füge die Ergebnisse der aktuellen Seite hinzu - allCorrespondents = allCorrespondents.concat( - results.map((correspondent) => ({ - name: correspondent.name, - id: correspondent.id, - document_count: correspondent.document_count, - })) + // Entity objects, as everywhere else: callers reduce them with + // `.map((c) => c.name)` or toNameList(). visitAllContainedElements + // collects into a Set, so flatten it into the array callers expect. + const correspondents = + await this.correspondents.visitAllContainedElements( + this.client, + (correspondent, result) => + result.add({ + id: correspondent.id, + name: correspondent.name, + document_count: correspondent.document_count, + }) ); - - // Prüfe, ob es eine nächste Seite gibt - hasNextPage = next !== null; - page++; - - // Optional: Füge eine kleine Verzögerung hinzu, um die API nicht zu überlasten - if (hasNextPage) { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - } - - return allCorrespondents; + return Array.from(correspondents); } catch (error) { console.error('[ERROR] fetching correspondent names:', error.message); return []; @@ -1150,44 +1143,20 @@ class PaperlessService { async listDocumentTypesNames() { this.initialize(); - let allDocumentTypes = []; - let page = 1; - let hasNextPage = true; - try { - while (hasNextPage) { - const response = await this.client.get('/document_types/', { - params: { - fields: 'id,name', - count: true, - page: page, - }, - }); - - const { results, next } = response.data; - - allDocumentTypes = allDocumentTypes.concat( - results.map((docType) => ({ - name: docType.name, - id: docType.id, - })) - ); - - hasNextPage = next !== null; - page++; - - if (hasNextPage) { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - } - - return allDocumentTypes; + const documentTypes = await this.documentTypes.visitAllContainedElements( + this.client, + (doc_type, result) => + result.add({ id: doc_type.id, name: doc_type.name }) + ); + return Array.from(documentTypes); } catch (error) { console.error('[ERROR] fetching document type names:', error.message); return []; } } + // todo async listTagNames() { this.initialize(); let allTags = []; @@ -1243,6 +1212,7 @@ class PaperlessService { return []; } + // todo async resolveTagIdsByName(tagNames) { const normalizedTagNames = this.parseTagList(tagNames); if (normalizedTagNames.length === 0) { @@ -1542,6 +1512,9 @@ class PaperlessService { if (error?.response?.status === 400) { this._supportsTagsIdNone = false; } else { + // Deliberate propagation: only the unsupported-filter case is + // handled here, everything else belongs to the outer handler. + // noinspection ThrowCaughtLocallyJS throw error; } } @@ -1660,15 +1633,17 @@ class PaperlessService { } console.log( - `[DEBUG] Finished fetching. Found ${documents.length} documents.` + `[INFO] Finished fetching. Found ${documents.length} documents.` ); return documents; } /** - * Normalize a mixed list of IDs into unique, positive integers. - * Guards against `Number(null) === 0`, which would otherwise turn documents - * without a correspondent into a lookup for the non-existent ID 0. + * Normalize a list of entity IDs (plain numbers or `{ id }` objects) into + * unique positive integers. + * + * @param ids Raw IDs as handed over by callers or API responses. + * @returns Unique positive integer IDs. */ _normalizeEntityIds(ids) { if (!Array.isArray(ids)) { @@ -1710,19 +1685,18 @@ class PaperlessService { return {}; } - const namesById = new Map(); - for (const tag of this.tagCache.values()) { - const id = Number(tag?.id); - if (Number.isInteger(id) && typeof tag?.name === 'string') { - namesById.set(id, tag.name); - } - } - const resolved = {}; for (const tagId of uniqueTagIds) { - const name = namesById.get(tagId); - if (name) { - resolved[tagId] = name; + try { + const tag = await this.tags.getById(this.client, tagId); + if (tag && typeof tag.name === 'string') { + resolved[tagId] = tag.name; + } + } catch (error) { + console.error( + `[ERROR] resolving tag name for #${tagId}:`, + error.message + ); } } @@ -1875,14 +1849,30 @@ class PaperlessService { */ async _primeCorrespondentNameCache() { try { - const correspondents = await this.listCorrespondentsNames(); + let page = 1; + let hasNextPage = true; let cached = 0; - for (const correspondent of correspondents) { - const id = Number(correspondent?.id); - if (Number.isInteger(id) && typeof correspondent?.name === 'string') { - this.correspondentNameCache.set(id, correspondent.name); - cached += 1; + + // Paginate directly through the axios client rather than through the + // correspondent enum: its element cache would answer from memory and a + // first lookup of an unresolvable ID would never reach Paperless-ngx. + while (hasNextPage) { + const response = await this.client.get('/correspondents/', { + params: { fields: 'id,name', count: true, page, page_size: 100 }, + }); + const results = response?.data?.results; + if (!Array.isArray(results)) { + break; + } + for (const correspondent of results) { + const id = Number(correspondent?.id); + if (Number.isInteger(id) && typeof correspondent?.name === 'string') { + this.correspondentNameCache.set(id, correspondent.name); + cached += 1; + } } + hasNextPage = Boolean(response?.data?.next); + page += 1; } return cached > 0; } catch (error) { @@ -1891,20 +1881,38 @@ class PaperlessService { } } + /** + * Get the Name of a Correspondent by its ID. + * + * @param correspondentId The id of the correspondent. + * @returns The name of the correspondent. + */ async getCorrespondentNameById(correspondentId) { - /** - * Get the Name of a Correspondent by its ID. - * - * @param id The id of the correspondent. - * @returns An object holding the correspondent name, or null. - */ - const id = Number(correspondentId); - if (!Number.isInteger(id) || id < 1) { + this.initialize(); + return this.correspondents.getById(this.client, correspondentId); + } + + async getDocumentTypeNameById(documentTypeId) { + this.initialize(); + try { + let documentType = this.documentTypes.getById( + this.client, + documentTypeId + ); + if (documentType) { + return documentType.name; + } + console.log( + `[DEBUG] Document type with ID #${documentTypeId} does not exist yet.` + ); + return null; + } catch (error) { + console.error( + `[ERROR] fetching document type #${documentTypeId}:`, + error.message + ); return null; } - - const names = await this.getCorrespondentNamesByIds([id]); - return names[id] ? { id, name: names[id] } : null; } async getTagNameById(tagId) { @@ -1914,13 +1922,17 @@ class PaperlessService { * @param id The id of the tag. * @returns The name of the tag. */ - const id = Number(tagId); - if (!Number.isInteger(id) || id < 1) { + this.initialize(); + try { + const tag = await this.tags.getById(this.client, tagId); + return tag ? tag.name : null; + } catch (error) { + console.error( + `[ERROR] fetching tag name for ID ${tagId}:`, + error.message + ); return null; } - - const names = await this.getTagNamesByIds([id]); - return names[id] || null; } async getDocumentsWithTitleTagsCorrespondentCreated() { @@ -1977,6 +1989,11 @@ class PaperlessService { } } + // Aktualisierte getDocuments Methode + async getDocuments() { + return this.getAllDocuments(); + } + async searchDocuments(query, limit = 100, mode = 'all') { this.initialize(); if (!this.client) { @@ -2044,6 +2061,9 @@ class PaperlessService { // malformed query syntax. Keep the selector usable by retrying with a // plain title filter instead of returning nothing. if (mode !== 'all') { + // Deliberate propagation: non-'all' modes have no fallback and the + // outer handler reports the failure to the caller. + // noinspection ThrowCaughtLocallyJS throw error; } @@ -2152,11 +2172,6 @@ class PaperlessService { } } - // Aktualisierte getDocuments Methode - async getDocuments() { - return this.getAllDocuments(); - } - async getDocumentContent(documentId) { this.initialize(); const response = await this.client.get(`/documents/${documentId}/`); @@ -2169,42 +2184,18 @@ class PaperlessService { const response = await this.client.get(`/documents/${documentId}/`); return response.data; } catch (error) { - console.error('[ERROR] fetching document %s:', documentId, error.message); + console.error(`[ERROR] fetching document ${documentId}:`, error.message); throw error; } } async searchForCorrespondentById(id) { + this.initialize(); try { - const response = await this.client.get('/correspondents/', { - params: { - id: id, - }, - }); - - const results = response.data.results; - - if (results.length === 0) { - console.log(`[DEBUG] No correspondent with "${id}" found`); - return null; - } - - if (results.length > 1) { - console.log(`[DEBUG] Multiple correspondents found:`); - results.forEach((c) => { - console.log(`- ID: ${c.id}, Name: ${c.name}`); - }); - return results; - } - - // Genau ein Ergebnis gefunden - return { - id: results[0].id, - name: results[0].name, - }; + return await this.correspondents.getById(this.client, id); } catch (error) { console.error( - '[ERROR] while seraching for existing correspondent:', + '[ERROR] while searching for existing correspondent:', error.message ); throw error; @@ -2213,38 +2204,8 @@ class PaperlessService { async searchForExistingCorrespondent(correspondent) { try { - const response = await this.client.get('/correspondents/', { - params: { - name__icontains: correspondent, - }, - }); - - const results = response.data.results; - - if (results.length === 0) { - console.log( - `[DEBUG] No correspondent with name "${correspondent}" found` - ); - return null; - } - - // Check for exact match in the results - thanks to @skius for the hint! - const exactMatch = results.find( - (c) => c.name.toLowerCase() === correspondent.toLowerCase() - ); - if (exactMatch) { - console.log( - `[DEBUG] Found exact match for correspondent "${correspondent}" with ID ${exactMatch.id}` - ); - return { - id: exactMatch.id, - name: exactMatch.name, - }; - } - - // No exact match found, return null - console.log(`[DEBUG] No exact match found for "${correspondent}"`); - return null; + this.initialize(); + return await this.correspondents.getByName(this.client, correspondent); } catch (error) { console.error( '[ERROR] while searching for existing correspondent:', @@ -2254,6 +2215,7 @@ class PaperlessService { } } + // todo async getOrCreateCorrespondent(name, options = {}) { this.initialize(); @@ -2268,119 +2230,68 @@ class PaperlessService { `[DEBUG] Processing correspondent with restrictToExistingCorrespondents=${restrictToExistingCorrespondents}` ); - try { - // Search for the correspondent - const existingCorrespondent = - await this.searchForExistingCorrespondent(name); + const existingCorrespondent = + await this.searchForExistingCorrespondent(name); + + if (existingCorrespondent) { console.log( - '[DEBUG] Response Correspondent Search: ', - existingCorrespondent + `[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}` ); + return existingCorrespondent; + } - if (existingCorrespondent) { - console.log( - `[DEBUG] Found existing correspondent "${name}" with ID ${existingCorrespondent.id}` - ); - return existingCorrespondent; - } + // The creation guard runs before the restriction is applied: a name + // that is only another spelling of an existing correspondent *is* an + // existing correspondent, so it is mapped rather than dropped. + const mapped = await this._mapProposedName('correspondents', name, options); + if (mapped) { + return { id: mapped.id, name: mapped.name }; + } - // The creation guard runs before the restriction is applied: a name - // that is only another spelling of an existing correspondent *is* an - // existing correspondent, so it is mapped rather than dropped. - const mapped = await this._mapProposedName( - 'correspondents', - name, - options + // If we're restricting to existing correspondents and none was found, return null + if (restrictToExistingCorrespondents) { + console.log( + `[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null` ); - if (mapped) { - return { id: mapped.id, name: mapped.name }; - } - - // If we're restricting to existing correspondents and none was found, return null - if (restrictToExistingCorrespondents) { - console.log( - `[DEBUG] Correspondent "${name}" does not exist and restrictions are enabled, returning null` - ); - return null; - } + return null; + } - // Create new correspondent only if restrictions are not enabled - try { - const createResponse = await this.client.post('/correspondents/', { - name: name, + // Create new correspondent only if restrictions are not enabled + try { + const createResponse = await this.client.post('/correspondents/', { + name: name, + }); + console.log( + `[DEBUG] Created new correspondent "${name}" with ID ${createResponse.data.id}` + ); + this._rememberGuardEntity('correspondents', createResponse.data); + return createResponse.data; + } catch (createError) { + if (isUniqueConstraintError(createError)) { + // Race condition check - another process might have created it + const retryResponse = await this.client.get('/correspondents/', { + params: { name: name }, }); - console.log( - `[DEBUG] Created new correspondent "${name}" with ID ${createResponse.data.id}` + + const justCreatedCorrespondent = retryResponse.data.results.find( + (c) => c.name.toLowerCase() === name.toLowerCase() ); - this._rememberGuardEntity('correspondents', createResponse.data); - return createResponse.data; - } catch (createError) { - if ( - createError.response?.status === 400 && - createError.response?.data?.error?.includes('unique constraint') - ) { - // Race condition check - another process might have created it - const retryResponse = await this.client.get('/correspondents/', { - params: { name: name }, - }); - const justCreatedCorrespondent = retryResponse.data.results.find( - (c) => c.name.toLowerCase() === name.toLowerCase() + if (justCreatedCorrespondent) { + console.log( + `[DEBUG] Retrieved correspondent "${name}" after constraint error with ID ${justCreatedCorrespondent.id}` ); - - if (justCreatedCorrespondent) { - console.log( - `[DEBUG] Retrieved correspondent "${name}" after constraint error with ID ${justCreatedCorrespondent.id}` - ); - return justCreatedCorrespondent; - } + return justCreatedCorrespondent; } - throw createError; } - } catch (error) { - console.error( - '[ERROR] Failed to process correspondent "%s":', - name, - error.message - ); - throw error; + throw new EntityCreationError('correspondent', name, createError); } } async searchForExistingDocumentType(documentType) { + this.initialize(); try { - const response = await this.client.get('/document_types/', { - params: { - name__icontains: documentType, - }, - }); - - const results = response.data.results; - - if (results.length === 0) { - console.log( - `[DEBUG] No document type with name "${documentType}" found` - ); - return null; - } - - // Check for exact match in the results - const exactMatch = results.find( - (dt) => dt.name.toLowerCase() === documentType.toLowerCase() - ); - if (exactMatch) { - console.log( - `[DEBUG] Found exact match for document type "${documentType}" with ID ${exactMatch.id}` - ); - return { - id: exactMatch.id, - name: exactMatch.name, - }; - } - - // No exact match found, return null - console.log(`[DEBUG] No exact match found for "${documentType}"`); - return null; + return await this.documentTypes.getByName(this.client, documentType); } catch (error) { console.error( '[ERROR] while searching for existing document type:', @@ -2403,66 +2314,55 @@ class PaperlessService { `[DEBUG] Processing document type with restrictToExistingDocumentTypes=${restrictToExistingDocumentTypes}` ); - try { - // Suche nach existierendem document_type - const existingDocType = await this.searchForExistingDocumentType(name); - console.log('[DEBUG] Response Document Type Search: ', existingDocType); + // Suche nach existierendem document_type + const existingDocType = await this.searchForExistingDocumentType(name); + console.log('[DEBUG] Response Document Type Search: ', existingDocType); - if (existingDocType) { - console.log( - `[DEBUG] Found existing document type "${name}" with ID ${existingDocType.id}` - ); - return existingDocType; - } + if (existingDocType) { + console.log( + `[DEBUG] Found existing document type "${name}" with ID ${existingDocType.id}` + ); + return existingDocType; + } - if (restrictToExistingDocumentTypes) { - console.log( - `[DEBUG] Document type "${name}" does not exist and restrictions are enabled, returning null` - ); - return null; - } + if (restrictToExistingDocumentTypes) { + console.log( + `[DEBUG] Document type "${name}" does not exist and restrictions are enabled, returning null` + ); + return null; + } - // Erstelle neuen document_type - try { - const createResponse = await this.client.post('/document_types/', { - name: name, - matching_algorithm: 1, // 1 = ANY - match: '', // Optional: Kann später angepasst werden - is_insensitive: true, + // Erstelle neuen document_type + try { + const createResponse = await this.client.post('/document_types/', { + name: name, + matching_algorithm: 1, // 1 = ANY + match: '', // Optional: Kann später angepasst werden + is_insensitive: true, + }); + console.log( + `[DEBUG] Created new document type "${name}" with ID ${createResponse.data.id}` + ); + return createResponse.data; + } catch (createError) { + if (isUniqueConstraintError(createError)) { + // Race condition check + const retryResponse = await this.client.get('/document_types/', { + params: { name: name }, }); - console.log( - `[DEBUG] Created new document type "${name}" with ID ${createResponse.data.id}` + + const justCreatedDocType = retryResponse.data.results.find( + (dt) => dt.name.toLowerCase() === name.toLowerCase() ); - return createResponse.data; - } catch (createError) { - if ( - createError.response?.status === 400 && - createError.response?.data?.error?.includes('unique constraint') - ) { - // Race condition check - const retryResponse = await this.client.get('/document_types/', { - params: { name: name }, - }); - const justCreatedDocType = retryResponse.data.results.find( - (dt) => dt.name.toLowerCase() === name.toLowerCase() + if (justCreatedDocType) { + console.log( + `[DEBUG] Retrieved document type "${name}" after constraint error with ID ${justCreatedDocType.id}` ); - - if (justCreatedDocType) { - console.log( - `[DEBUG] Retrieved document type "${name}" after constraint error with ID ${justCreatedDocType.id}` - ); - return justCreatedDocType; - } + return justCreatedDocType; } - throw createError; } - } catch (error) { - console.error( - `[ERROR] Failed to process document type "${name}":`, - error.message - ); - throw error; + throw new EntityCreationError('document type', name, createError); } } @@ -2472,8 +2372,7 @@ class PaperlessService { try { console.log( - '[DEBUG] Removing unused tags from document %s, keeping tags:', - documentId, + `[DEBUG] Removing unused tags from document ${documentId}, keeping tags:`, keepTagIds ); @@ -2504,8 +2403,7 @@ class PaperlessService { return await this.getDocument(documentId); } catch (error) { console.error( - '[ERROR] Error removing unused tags from document %s:', - documentId, + `[ERROR] Error removing unused tags from document ${documentId}:`, error.message ); throw error; @@ -2515,8 +2413,8 @@ class PaperlessService { async getTagTextFromId(tagId) { this.initialize(); try { - const response = await this.client.get(`/tags/${tagId}/`); - return response.data.name; + const tag = await this.tags.getById(this.client, tagId); + return tag ? tag.name : null; } catch (error) { console.error( `[ERROR] fetching tag text for ID ${tagId}:`, @@ -2576,7 +2474,6 @@ class PaperlessService { }; } } - /* Resolving the own user ID used to hinge entirely on PAPERLESS_USERNAME matching a name in the response, and returned null without a word when it did not — a configured display name, a case difference or a token whose @@ -2676,17 +2573,33 @@ class PaperlessService { if (updates.tags) { console.log( - '[DEBUG] Current tags for document %s:', - documentId, + `[DEBUG] Current tags for document ${documentId}:`, currentDoc.tags ); console.log(`[DEBUG] Adding new tags:`, updates.tags); console.log(`[DEBUG] Current correspondent:`, currentDoc.correspondent); console.log(`[DEBUG] New correspondent:`, updates.correspondent); - const combinedTags = [ - ...new Set([...currentDoc.tags, ...updates.tags]), - ]; + let combinedTags = [...new Set([...currentDoc.tags, ...updates.tags])]; + + // Predefined-scan trigger tags are removed here rather than by the + // caller: the merge above would otherwise re-add them from + // currentDoc.tags on every write-back. + if ( + Array.isArray(updates.removeTagIds) && + updates.removeTagIds.length > 0 + ) { + const removeSet = new Set(updates.removeTagIds.map(Number)); + combinedTags = combinedTags.filter((tagId) => { + const normalized = + typeof tagId === 'object' ? Number(tagId?.id) : Number(tagId); + return !removeSet.has(normalized); + }); + console.log( + `[DEBUG] Removed configured tag ids from document ${documentId}:`, + updates.removeTagIds + ); + } updates.tags = combinedTags; console.log(`[DEBUG] Combined tags:`, combinedTags); @@ -2767,11 +2680,7 @@ class PaperlessService { console.log('[DEBUG] Final update data:', updateData); await this.client.patch(`/documents/${documentId}/`, updateData); - console.log( - '[SUCCESS] Updated document %s with:', - documentId, - updateData - ); + console.log(`[SUCCESS] Updated document ${documentId}`); return await this.getDocument(documentId); } catch (error) { console.error( @@ -2933,10 +2842,12 @@ class PaperlessService { * instead, which is what happened before the guard existed. */ async _guardEntitiesFromTagCache() { - await this.ensureTagCache(); - return [...this.tagCache.values()] - .filter((raw) => !raw?.is_inbox_tag) - .map((raw) => this._toEntityRecord('tags', raw)); + if (!this.client) return []; + const records = []; + await this.tags.visitAllContainedElements(this.client, (raw) => { + if (!raw?.is_inbox_tag) records.push(this._toEntityRecord('tags', raw)); + }); + return records; } /** @@ -3808,6 +3719,9 @@ class PaperlessService { clearDocumentTypeCache() { this.documentTypeCache = []; this.lastDocumentTypeRefresh = 0; + // The lookup cache analysis maps document types against must not keep a + // type that was just deleted either. + this._documentTypes?.invalidate(); } /** diff --git a/services/serviceUtils.js b/services/serviceUtils.js index a25bb5d9..20950bfb 100644 --- a/services/serviceUtils.js +++ b/services/serviceUtils.js @@ -339,7 +339,7 @@ const METADATA_ENDPOINTS = [ ]; function stripIpv6Brackets(value) { - return String(value || '').replace(/^\[|\]$/g, ''); + return String(value || '').replace(/^\[|]$/g, ''); } function normalizeIpAddress(ipAddress) { @@ -641,6 +641,30 @@ function validateUrlAgainstBase(urlToValidate, expectedBaseUrl) { }; } +/** + * Safely extract the relative path from a pagination `next` URL, validating it + * against the base URL of the axios client that received it. This prevents SSRF + * by ensuring a server-controlled link cannot point anywhere but the configured + * Paperless-ngx origin. + * + * @param {string} nextUrl - The next URL from an API response + * @param {Object} client - Axios client whose defaults.baseURL is the expected origin + * @returns {string|null} The relative path if valid, null otherwise + */ +function safeExtractRelativePath(nextUrl, client) { + if (!nextUrl || !client?.defaults?.baseURL) { + return null; + } + + const validation = validateUrlAgainstBase(nextUrl, client.defaults.baseURL); + if (!validation.valid) { + console.error(`[ERROR] URL validation failed: ${validation.error}`); + return null; + } + + return validation.relativePath; +} + /** * Builds an axios `beforeRedirect` handler that keeps redirects on the host the * request was validated against. @@ -1104,6 +1128,7 @@ module.exports = { validateUrl, validateApiUrl, validateUrlAgainstBase, + safeExtractRelativePath, createRedirectGuard, validateCustomFieldValue, assertCompletionNotTruncated, diff --git a/services/thumbnailCache.js b/services/thumbnailCache.js new file mode 100644 index 00000000..61b40b3a --- /dev/null +++ b/services/thumbnailCache.js @@ -0,0 +1,62 @@ +const fs = require('fs').promises; +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('./thumbnailCachePaths'); + +/** + * Cache a document's Paperless-ngx thumbnail for the UI. + * + * Best effort by contract: the thumbnail never enters the AI prompt, so no + * outcome of this step may decide whether the document gets analysed. All four + * AI services used to let it decide, each in its own way — an id that + * getThumbnailCachePath() rejects threw straight out of analyzeDocument() + * (which the scan loop does not expect, and which answered 500 on every + * POST /manual/analyze without an id), a thumbnail Paperless did not have + * returned `undefined` from it, and an unreachable Paperless turned into an + * analysis failure. One implementation for all four keeps them from drifting + * apart again. + * + * @param {string|number} id - Paperless-ngx document ID + * @returns {Promise} true when a thumbnail is cached afterwards + */ +async function cacheThumbnail(id) { + let cachePath; + try { + cachePath = getThumbnailCachePath(id); + } catch (error) { + console.warn( + `[WARNING] Skipping thumbnail cache for document ${JSON.stringify(id)}: ${error.message}` + ); + return false; + } + + try { + await fs.access(cachePath); + console.log('[DEBUG] Thumbnail already cached'); + return true; + } catch { + console.log('Thumbnail not cached, fetching from Paperless'); + } + + try { + // Required here rather than at module load: paperlessService reaches back + // into aiServiceFactory, and the AI services are what call this. + const paperlessService = require('./paperlessService'); + const thumbnailData = await paperlessService.getThumbnailImage(id); + + if (!thumbnailData) { + console.warn('Thumbnail not found'); + return false; + } + + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile(cachePath, thumbnailData); + return true; + } catch (error) { + console.warn(`[WARNING] Thumbnail caching failed: ${error.message}`); + return false; + } +} + +module.exports = { cacheThumbnail }; diff --git a/tests/test-ai-processed-tag-without-tags.js b/tests/test-ai-processed-tag-without-tags.js index 5954008d..6ac5b218 100644 --- a/tests/test-ai-processed-tag-without-tags.js +++ b/tests/test-ai-processed-tag-without-tags.js @@ -94,17 +94,27 @@ function createTagServer({ existing = [], failCreateFor = [] } = {}) { }; } +/** + * A fresh tag cache for every case, so no case reads tags another one created. + * Its similarity search would ask the AI provider about every unknown name; a + * stub that never finds a match keeps the test offline. + */ +function resetTagCache() { + paperlessService._tags = null; + paperlessService.tags.ai_service = { + generateAnswer: async () => JSON.stringify({ match: null }), + }; +} + async function withServer(server, fn) { const originalClient = paperlessService.client; paperlessService.client = server; - paperlessService.tagCache.clear(); - paperlessService.lastTagRefresh = 0; + resetTagCache(); try { return await fn(); } finally { paperlessService.client = originalClient; - paperlessService.tagCache.clear(); - paperlessService.lastTagRefresh = 0; + paperlessService._tags = null; } } diff --git a/tests/test-cached-name-id-enum.js b/tests/test-cached-name-id-enum.js new file mode 100644 index 00000000..cb86c91c --- /dev/null +++ b/tests/test-cached-name-id-enum.js @@ -0,0 +1,263 @@ +const assert = require('assert'); +const { CachedNameIdEnum } = require('../services/CachedNameIdEnum'); + +function createEnum( + aiService = { generateAnswer: async () => JSON.stringify({ match: null }) }, + cacheTtl = () => 0 +) { + return new CachedNameIdEnum( + 'correspondent', + '/correspondents/', + 'Use the closest match.', + cacheTtl, + aiService + ); +} + +function createClient(pages) { + const calls = []; + let pageIndex = 0; + const client = { + defaults: { baseURL: 'https://paperless.example/api' }, + get: async (url, options) => { + calls.push({ url, options }); + const results = pages[pageIndex++] || []; + const next = + pageIndex < pages.length + ? `https://paperless.example/api/page-${pageIndex}` + : null; + return { data: { results, next } }; + }, + }; + client.getCalls = () => calls; + return client; +} + +async function run() { + await assert.rejects( + () => createEnum().getCount(), + /No client provided\./, + 'getCount should require a client' + ); + + const enumWithPages = createEnum(); + const pagedClient = createClient([ + [{ id: '1', name: 'Acme' }], + [{ id: '2', name: 'Tax Office' }], + ]); + const refresh = enumWithPages.flush_cache(pagedClient); + assert.strictEqual(typeof refresh.then, 'function'); + await refresh; + assert.deepStrictEqual(enumWithPages.element_list, [ + { id: '1', name: 'Acme' }, + { id: '2', name: 'Tax Office' }, + ]); + assert.deepStrictEqual( + pagedClient.getCalls().map(({ url, options }) => [url, options]), + [ + // page_size must ride in axios `params` — a flat property on the config + // is dropped by real axios and the server answers with its default. + ['/correspondents/', { params: { page_size: 100 } }], + ['/page-1', { params: { page_size: 100 } }], + ] + ); + + const exactClient = createClient([ + [ + { id: '1', name: 'Acme' }, + { id: '2', name: 'Tax Office' }, + ], + ]); + const exactEnum = createEnum(); + assert.deepStrictEqual(await exactEnum.getByName(exactClient, 'ACME'), { + id: '1', + name: 'Acme', + }); + assert.deepStrictEqual(await exactEnum.getById(exactClient, '2'), { + id: '2', + name: 'Tax Office', + }); + assert.strictEqual(await exactEnum.getByName(exactClient, 'Unknown'), null); + assert.strictEqual(await exactEnum.getById(exactClient, 'missing'), null); + + const similarityEnum = createEnum({ + generateAnswer: async () => + JSON.stringify({ + match: { id: '42', name: 'Acme Corporation' }, + }), + }); + const similarityClient = createClient([ + [{ id: '42', name: 'Acme Corporation' }], + ]); + assert.deepStrictEqual( + await similarityEnum.getByName(similarityClient, 'The Acme business'), + { id: '42', name: 'Acme Corporation' }, + 'semantic matches must unwrap the match property from the AI response' + ); + + const missEnum = createEnum(); + const missClient = createClient([ + [{ id: '1', name: 'Old' }], + [{ id: '2', name: 'New' }], + ]); + assert.deepStrictEqual(await missEnum.getByName(missClient, 'New'), { + id: '2', + name: 'New', + }); + assert.strictEqual(missClient.getCalls().length, 2); + + const visitorEnum = createEnum(); + const visitorClient = createClient([ + [ + { id: '1', name: 'Acme' }, + { id: '2', name: 'Tax Office' }, + ], + ]); + const visited = []; + const resultSet = await visitorEnum.visitAllContainedElements( + visitorClient, + (element, set) => { + visited.push(element.name); + set.add(element.id); + } + ); + assert.deepStrictEqual(visited, ['Acme', 'Tax Office']); + assert.deepStrictEqual([...resultSet], ['1', '2']); + assert.strictEqual(await visitorEnum.getCount(visitorClient), 2); + + await assert.rejects( + () => createEnum().getByName(exactClient, undefined), + /Mandatory argument name is undefined or null\./ + ); + await assert.rejects( + () => createEnum().getByName(exactClient, null), + /Mandatory argument name is undefined or null\./ + ); + await assert.rejects( + () => createEnum().getById(exactClient, undefined), + /Mandatory argument id is undefined or null\./ + ); + await assert.rejects( + () => createEnum().getById(exactClient, null), + /Mandatory argument id is undefined or null\./ + ); + + const zeroIdEnum = createEnum(); + const zeroIdClient = createClient([[{ id: 0, name: 'Invalid' }]]); + assert.strictEqual(await zeroIdEnum.getById(zeroIdClient, 0), null); + assert.strictEqual(zeroIdClient.getCalls().length, 0); + + const semanticCalls = []; + const semanticObjectEnum = createEnum({ + generateAnswer: async (...args) => { + semanticCalls.push(args); + return { match: { id: 42, name: 'Acme Corporation' } }; + }, + }); + const semanticObjectClient = createClient([ + [{ id: 42, name: 'Acme Corporation' }], + ]); + assert.deepStrictEqual( + await semanticObjectEnum.getByName( + semanticObjectClient, + 'Completely different' + ), + { id: 42, name: 'Acme Corporation' } + ); + assert.strictEqual(semanticCalls.length, 1); + assert.match(semanticCalls[0][0], /Acme Corporation \(ID: 42\)/); + assert.strictEqual(semanticCalls[0][2], 300); + assert.deepStrictEqual(semanticCalls[0][3], { + type: 'object', + properties: { + match: { + type: ['object', 'null'], + properties: { id: { type: 'number' }, name: { type: 'string' } }, + required: ['id', 'name'], + }, + }, + required: ['match'], + }); + + const failingAiEnum = createEnum({ + generateAnswer: async () => { + throw new Error('AI unavailable'); + }, + }); + const failingAiClient = createClient([[{ id: 1, name: 'Acme' }]]); + assert.strictEqual( + await failingAiEnum.getByName(failingAiClient, 'No such correspondent'), + null, + 'AI errors should be treated as a similarity miss' + ); + + const refreshEnum = createEnum(); + let resolveRefresh; + const refreshClient = { + defaults: { baseURL: 'https://paperless.example/api' }, + getCalls: () => refreshCalls, + get: async (...args) => { + refreshCalls.push(args); + await new Promise((resolve) => { + resolveRefresh = resolve; + }); + return { data: { results: [{ id: 7, name: 'Shared' }], next: null } }; + }, + }; + const refreshCalls = []; + const firstRefresh = refreshEnum.flush_cache(refreshClient); + const secondRefresh = refreshEnum.flush_cache(refreshClient); + assert.strictEqual(firstRefresh, secondRefresh); + resolveRefresh(); + await firstRefresh; + assert.strictEqual(refreshCalls.length, 1); + assert.strictEqual(await refreshEnum.getCount(refreshClient), 1); + + const expiringEnum = createEnum(undefined, () => 1); + let expiringFetchNo = 0; + const expiringCalls = []; + const expiringClient = { + defaults: { baseURL: 'https://paperless.example/api' }, + get: async (url, options) => { + expiringCalls.push({ url, options }); + expiringFetchNo += 1; + return { + data: { + results: [ + expiringFetchNo === 1 + ? { id: 1, name: 'First' } + : { id: 2, name: 'Second' }, + ], + next: null, + }, + }; + }, + getCalls: () => expiringCalls, + }; + await expiringEnum.getCount(expiringClient); + assert.strictEqual(expiringClient.getCalls().length, 1); + expiringEnum.cache_last_fetch_time = Date.now() - 10; + assert.strictEqual(await expiringEnum.getCount(expiringClient), 1); + assert.strictEqual(expiringClient.getCalls().length, 2); + + assert.throws(() => createEnum().flush_cache(), /No client provided\./); + await assert.rejects( + () => createEnum().getByName(null, 'Acme'), + /No client provided\./ + ); + await assert.rejects( + () => createEnum().getById(null, '1'), + /No client provided\./ + ); + await assert.rejects( + () => createEnum().visitAllContainedElements(null, () => {}), + /No client provided\./ + ); + + console.log('✅ test-cached-name-id-enum passed'); +} + +run().catch((error) => { + console.error('❌ test-cached-name-id-enum failed:', error); + process.exit(1); +}); diff --git a/tests/test-duplicate-merge-service.js b/tests/test-duplicate-merge-service.js index ae642bb7..b0d87a00 100644 --- a/tests/test-duplicate-merge-service.js +++ b/tests/test-duplicate-merge-service.js @@ -271,7 +271,7 @@ async function main() { await documentModel.addToHistory(1002, [48, 12], 'Doc B', 'Someone'); await documentModel.saveOriginalData(1001, [48, 9], 5, 'Doc A'); - paperlessService.tagCache.set('invoices', { id: 48 }); + paperlessService.tags.add({ id: 48, name: 'invoices' }); paperlessService.correspondentNameCache.set(5, 'Someone'); paperlessService.lastCorrespondentRefresh = Date.now(); @@ -328,7 +328,11 @@ async function main() { 'originals rewritten in place' ); - assert.strictEqual(paperlessService.tagCache.size, 0, 'tag cache empty'); + assert.strictEqual( + paperlessService.tags.element_list.length, + 0, + 'tag cache empty' + ); assert.strictEqual( paperlessService.correspondentNameCache.size, 0, @@ -848,7 +852,10 @@ async function main() { duplicateMergeService.dashboardRefreshDebounceMs = 40; try { for (const sourceId of [111, 112, 113]) { - paperlessService.tagCache.set(`source-${sourceId}`, { id: sourceId }); + paperlessService.tags.add({ + id: sourceId, + name: `source-${sourceId}`, + }); const result = await duplicateMergeService.merge({ kind: 'tags', targetId: 110, @@ -856,7 +863,7 @@ async function main() { }); assert.strictEqual(result.status, 'done'); assert.strictEqual( - paperlessService.tagCache.size, + paperlessService.tags.element_list.length, 0, 'the caches are dropped immediately, not on the timer' ); diff --git a/tests/test-entity-creation-guard.js b/tests/test-entity-creation-guard.js index 6ceb8102..51e7cff0 100644 --- a/tests/test-entity-creation-guard.js +++ b/tests/test-entity-creation-guard.js @@ -24,6 +24,8 @@ * 7. Restriction on plus no match drops the name as before * 8. Correspondents: a hard match maps, with "correspondent" in the line * 9. Correspondents: a typo-like match creates and hints + * 9a. Correspondents: a name holding every word of an existing one is + * answered by the lookup cache and never reaches the guard * 10. Correspondents: restriction on plus a hard match maps * 11. One list read per kind per minute, whatever the number of documents * 12. A name created in between is in the guard's list without a new read @@ -103,6 +105,17 @@ async function main() { paperlessService.correspondentNameCache.clear(); paperlessService.lastCorrespondentRefresh = 0; paperlessService._invalidateGuardEntities(); + // The lookup caches (CachedNameIdEnum) ask the AI provider about a name + // they cannot match by text. Offline that answer is "no match", so the + // guard is what decides. + for (const cache of [ + paperlessService.tags, + paperlessService.correspondents, + ]) { + cache.ai_service = { + generateAnswer: async () => JSON.stringify({ match: null }), + }; + } return fake; } @@ -369,40 +382,65 @@ async function main() { await clearMappings(); const { value, lines } = await withLog(() => - paperlessService.getOrCreateCorrespondent('Amazon EU S.a.r.l.', { + paperlessService.getOrCreateCorrespondent('Amazom EU', { documentId: 91, }) ); assert.notStrictEqual(value.id, 90); - assert.ok(fake.correspondentNames().includes('Amazon EU S.a.r.l.')); + assert.ok(fake.correspondentNames().includes('Amazom EU')); assert.deepStrictEqual(await documentModel.listEntityNameMappings(), []); assert.ok( duplicateLines(lines).some( (line) => line === - '[DUPLICATES] created correspondent "Amazon EU S.a.r.l." although "Amazon EU" is close (prefix, 0.85); not mapped.' + '[DUPLICATES] created correspondent "Amazom EU" although "Amazon EU" is close (fuzzy, 0.94); not mapped.' ), `no hint line:\n${duplicateLines(lines).join('\n')}` ); }); + await test('Correspondents: the lookup cache maps a name holding every word of an existing one', async () => { + // A prefix match the guard would only hint at. The lookup cache has + // settled it already, so nothing is created and the guard records nothing. + const fake = useFake({ + correspondents: [{ id: 95, name: 'Amazon EU' }], + }); + await clearMappings(); + + const { value, lines } = await withLog(() => + paperlessService.getOrCreateCorrespondent('Amazon EU S.a.r.l.', { + documentId: 96, + }) + ); + + assert.strictEqual(value.id, 95); + assert.deepStrictEqual(fake.correspondentNames(), ['Amazon EU']); + assert.deepStrictEqual(await documentModel.listEntityNameMappings(), []); + assert.deepStrictEqual(duplicateLines(lines), []); + }); + await test('Correspondents: restriction on plus a hard match maps', async () => { + // Another legal form rather than an added one: a name that only adds + // words is matched by the lookup cache before the guard runs. const fake = useFake({ - correspondents: [{ id: 100, name: 'Telekom Deutschland' }], + correspondents: [{ id: 100, name: 'Telekom Deutschland GmbH' }], }); await clearMappings(); const { value } = await withLog(() => - paperlessService.getOrCreateCorrespondent('Telekom Deutschland GmbH', { + paperlessService.getOrCreateCorrespondent('Telekom Deutschland AG', { restrictToExistingCorrespondents: true, documentId: 101, }) ); - assert.deepStrictEqual(value, { id: 100, name: 'Telekom Deutschland' }); + assert.deepStrictEqual(value, { + id: 100, + name: 'Telekom Deutschland GmbH', + }); assert.deepStrictEqual(fake.correspondentNames(), [ - 'Telekom Deutschland', + 'Telekom Deutschland GmbH', ]); const mappings = await documentModel.listEntityNameMappings(); assert.strictEqual(mappings[0].reason, 'legal-form'); @@ -443,13 +481,13 @@ async function main() { const { lines } = await withLog(async () => { // Nothing matches this, so it is created ... const created = await paperlessService.getOrCreateCorrespondent( - 'Telekom Deutschland', + 'Telekom Deutschland GmbH', { documentId: 1 } ); assert.ok(created.id > 0); // ... and the next document's proposal must find it. const mapped = await paperlessService.getOrCreateCorrespondent( - 'Telekom Deutschland GmbH', + 'Telekom Deutschland AG', { documentId: 2 } ); assert.strictEqual( @@ -471,7 +509,7 @@ async function main() { ); assert.ok( duplicateLines(lines).some((line) => - /mapped correspondent "Telekom Deutschland GmbH"/.test(line) + /mapped correspondent "Telekom Deutschland AG"/.test(line) ) ); }); diff --git a/tests/test-entity-match-ai-service.js b/tests/test-entity-match-ai-service.js index c3ef2324..7a1b8f6c 100644 --- a/tests/test-entity-match-ai-service.js +++ b/tests/test-entity-match-ai-service.js @@ -878,7 +878,10 @@ async function main() { stub.sent.openai.temperature, config.aiTemperatureGeneration ); - assert.strictEqual(stub.sent.azure.temperature, 0.7); + assert.strictEqual( + stub.sent.azure.temperature, + config.aiTemperatureGeneration + ); assert.strictEqual( stub.sent.azure.max_tokens, Number(config.responseTokens) diff --git a/tests/test-llm-analysis-result-contract.js b/tests/test-llm-analysis-result-contract.js new file mode 100644 index 00000000..405feb22 --- /dev/null +++ b/tests/test-llm-analysis-result-contract.js @@ -0,0 +1,407 @@ +/** + * The analysis result contract, pinned across all four providers. + * + * Safety net for the LLM service refactoring (docs/llm-service-refactoring.md, + * step 1). It describes what analyzeDocument() promises its three callers today — + * server.js:766-821, services/mistralOcrService.js:988-997, and + * routes/setup.js:3167-3177 — so that collapsing four implementations into one + * cannot quietly change the shape they read. + * + * Two properties carry the weight: + * + * 1. Success returns the full { document, metrics, truncated } record. Every + * field of `document` is written back by paperlessService.updateDocument(); + * a narrowed document does not fail loudly, it silently stops setting the + * title, type, date, language, and custom fields. + * + * 2. Failure returns { document: { tags: [], correspondent: null }, + * metrics: null, error, errorCode } and **never throws**. server.js relies + * on this: a throw escapes the per-document try in the scan loop instead of + * routing the document to the OCR queue. + * + * The provider client is the only collaborator stubbed. The thumbnail cache is + * pre-seeded so the analysis path reaches the client without a Paperless call. + */ + +const assert = require('assert'); +const fs = require('fs').promises; + +process.env.AI_PROVIDER = 'openai'; +process.env.OPENAI_API_KEY = 'test-key'; +process.env.OPENAI_MODEL = 'gpt-4'; +process.env.AZURE_API_KEY = 'test-key'; +process.env.AZURE_ENDPOINT = 'https://example.invalid'; +process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment'; +process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1'; +process.env.CUSTOM_API_KEY = 'test-key'; +process.env.CUSTOM_MODEL = 'test-model'; +process.env.SYSTEM_PROMPT = 'Analyse the document.'; + +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('../services/thumbnailCachePaths'); +const openaiService = require('../services/openaiService'); +const azureService = require('../services/azureService'); +const customService = require('../services/customService'); +const ollamaService = require('../services/ollamaService'); + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +/** Own id so the seeded thumbnail cannot collide with a real cached document. */ +const DOCUMENT_ID = 'testLlmResultContract'; + +/** + * Every field the write-back path reads. Deliberately includes the ones a + * narrowed result would drop without any error surfacing. + */ +const ANSWER = { + title: 'Telekom invoice July', + correspondent: 'Telekom Deutschland GmbH', + tags: ['invoice', 'telecom'], + document_type: 'Rechnung', + document_date: '2026-09-03', + language: 'de', + custom_fields: [{ field_name: 'Amount', value: '49.99' }], +}; + +function completion(content = JSON.stringify(ANSWER)) { + return { + choices: [{ message: { content }, finish_reason: 'stop' }], + usage: { prompt_tokens: 900, completion_tokens: 120, total_tokens: 1020 }, + }; +} + +/** Replaces the SDK client with one that answers whatever the case needs. */ +function stubClient(service, respond) { + service.client = { + chat: { completions: { create: respond } }, + models: { list: async () => ({ data: [] }) }, + }; +} + +/** Same for Ollama, which answers over axios instead of an SDK. */ +function stubOllamaClient(respond) { + ollamaService.client = { + post: respond, + get: async () => ({ status: 200, data: { models: [] } }), + }; +} + +function openAiCompatibleProvider(label, service) { + return { + label, + service, + arrangeSuccess: () => stubClient(service, async () => completion()), + arrangeAnswer: (content) => + stubClient(service, async () => completion(content)), + arrangeThrow: (error) => + stubClient(service, async () => { + throw error; + }), + }; +} + +const PROVIDERS = [ + openAiCompatibleProvider('OpenAI', openaiService), + openAiCompatibleProvider('Azure', azureService), + openAiCompatibleProvider('Custom', customService), + { + label: 'Ollama', + service: ollamaService, + arrangeSuccess: () => + stubOllamaClient(async () => ({ + data: { + response: JSON.stringify(ANSWER), + done_reason: 'stop', + eval_count: 120, + prompt_eval_count: 900, + }, + })), + arrangeAnswer: (content) => + stubOllamaClient(async () => ({ + data: { + response: content, + done_reason: 'stop', + eval_count: 120, + prompt_eval_count: 900, + }, + })), + arrangeThrow: (error) => + stubOllamaClient(async () => { + throw error; + }), + }, +]; + +/** Enough text that no provider takes an "insufficient content" shortcut. */ +const CONTENT = 'Invoice from Telekom Deutschland GmbH. '.repeat(20); + +const analyse = (service) => + service.analyzeDocument(CONTENT, [], [], [], DOCUMENT_ID); + +/** The failure shape server.js:774-821 branches on. */ +const assertFailureRecord = (analysis) => { + assert.ok(analysis, 'analyzeDocument returned nothing at all'); + assert.deepStrictEqual( + analysis.document, + { tags: [], correspondent: null }, + 'a failure must still carry an empty, well-formed document' + ); + assert.strictEqual(analysis.metrics, null, 'a failure records no metrics'); + assert.strictEqual( + typeof analysis.error, + 'string', + 'the failure reason must be a string — shouldQueueForOcrOnAiError() reads it' + ); + assert.ok(analysis.error.trim(), 'the failure reason must not be empty'); +}; + +(async () => { + console.log('\n=== LLM analysis result contract ==='); + + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile( + getThumbnailCachePath(DOCUMENT_ID), + Buffer.from([0x89, 0x50, 0x4e, 0x47]) + ); + + for (const { + label, + service, + arrangeSuccess, + arrangeAnswer, + arrangeThrow, + } of PROVIDERS) { + /* --- the success record ------------------------------------------- */ + + await check( + `${label}: a successful analysis returns the full record`, + async () => { + arrangeSuccess(); + const analysis = await analyse(service); + + assert.ok(analysis, 'analyzeDocument returned nothing at all'); + assert.strictEqual( + analysis.error, + undefined, + `unexpected error: ${analysis.error}` + ); + assert.strictEqual( + typeof analysis.truncated, + 'boolean', + 'truncated must always be a boolean' + ); + } + ); + + await check( + `${label}: every document field survives to the caller`, + async () => { + arrangeSuccess(); + const { document } = await analyse(service); + + // Asserted field by field: paperlessService.updateDocument() writes each + // of these, and a dropped one produces no error, just a missing value. + assert.deepStrictEqual(document.tags, ANSWER.tags); + assert.strictEqual(document.correspondent, ANSWER.correspondent); + assert.strictEqual(document.title, ANSWER.title); + assert.strictEqual(document.document_type, ANSWER.document_type); + assert.strictEqual(document.document_date, ANSWER.document_date); + assert.strictEqual(document.language, ANSWER.language); + assert.deepStrictEqual(document.custom_fields, ANSWER.custom_fields); + } + ); + + await check( + `${label}: metrics are reported as named token counts`, + async () => { + arrangeSuccess(); + const { metrics } = await analyse(service); + + // documentModel.addOpenAIMetrics() reads these three names. + assert.ok(metrics, 'a successful analysis must report metrics'); + assert.strictEqual(typeof metrics.promptTokens, 'number'); + assert.strictEqual(typeof metrics.completionTokens, 'number'); + assert.strictEqual(typeof metrics.totalTokens, 'number'); + } + ); + + /* --- the failure record, and the promise not to throw --------------- */ + + // The message text itself is not pinned here: Custom rewrites anything + // isTimeoutError() recognizes into "AI response timeout reached…", and + // Ollama's retry wrapper replaces it with "Ollama request failed (…)". + // Only the phrases that steer OCR fallback are pinned, in + // test-llm-ocr-fallback-phrases.js. + await check( + `${label}: a transport failure is returned, not thrown`, + async () => { + arrangeThrow(new Error('socket hang up')); + const analysis = await analyse(service); + + assertFailureRecord(analysis); + } + ); + + await check( + `${label}: an unparseable answer is returned, not thrown`, + async () => { + arrangeAnswer('this is not JSON at all'); + const analysis = await analyse(service); + + assertFailureRecord(analysis); + } + ); + + await check( + `${label}: an error carrying a code passes the code through`, + async () => { + // The scan loop reads errorCode to decide OCR fallback; a service that + // raises the right error but drops the code loses that decision. + const coded = new Error('provider stopped generating early'); + coded.code = 'ai_response_truncated'; + arrangeThrow(coded); + const analysis = await analyse(service); + + assertFailureRecord(analysis); + assert.strictEqual(analysis.errorCode, 'ai_response_truncated'); + } + ); + + await check( + `${label}: an error without a code leaves errorCode unset`, + async () => { + arrangeThrow(new Error('socket hang up')); + const analysis = await analyse(service); + + assert.strictEqual( + analysis.errorCode, + undefined, + 'an uncoded failure must not invent a code' + ); + } + ); + } + + /* --- an answer with no usable metadata is a failure on all four -------- */ + + /* Finding 7k, decided 2026-10-11: all four providers raise "AI could not + determine assignable metadata: no tags or correspondent found", which is + one of the phrases shouldQueueForOcrOnAiError() matches, so the document + reaches the OCR queue. The check runs on the parsed answer, before + withDocumentDefaults() would turn a missing `tags` into [] and make + every answer pass. { tags: [], correspondent: null } still passes on + all four — a model that found nothing is not a broken answer. */ + + for (const { label, service, arrangeAnswer } of PROVIDERS) { + await check( + `${label}: an answer without usable metadata is a failure`, + async () => { + arrangeAnswer(JSON.stringify({ note: 'nothing usable here' })); + const analysis = await analyse(service); + + assertFailureRecord(analysis); + assert.match( + analysis.error, + /could not determine assignable metadata/i + ); + } + ); + } + + /* --- field defaults shape every success -------------------------------- */ + + /* Field defaults for all four (step 6, decided 2026-10-11): + withDocumentDefaults() shapes every provider's success to the seven document fields — present values + survive, missing ones come back as null (tags as an empty array), + unknown fields are dropped. The answer is complete enough to pass + assertAssignableMetadata, so this pins only the shaping. The chat + providers used to return the parsed answer untouched, missing fields + and all. */ + + const PARTIAL_ANSWER = { + title: 'Telekom', + correspondent: 'Telekom Deutschland GmbH', + tags: ['invoice'], + unknown_field: 'dropped', + }; + + for (const { label, service, arrangeAnswer } of PROVIDERS) { + await check( + `${label}: a partial answer comes back default-shaped`, + async () => { + arrangeAnswer(JSON.stringify(PARTIAL_ANSWER)); + const analysis = await analyse(service); + + assert.strictEqual( + analysis.error, + undefined, + `unexpected error: ${analysis.error}` + ); + assert.deepStrictEqual(analysis.document, { + tags: PARTIAL_ANSWER.tags, + correspondent: PARTIAL_ANSWER.correspondent, + title: PARTIAL_ANSWER.title, + document_date: null, + document_type: null, + language: null, + custom_fields: null, + }); + } + ); + } + + /* Finding 7c: a refusal returns the insufficient-content placeholder on + all four providers, not the failure record, so the retry loop does not + ask again. The error phrase still routes it to OCR; the document is what + the write-back would set, so it is pinned field by field. Metrics are + the tokens the refusal cost — the stubs all report 900 prompt / 120 + completion / 1020 total. */ + for (const { label, service, arrangeAnswer } of PROVIDERS) { + await check( + `${label}: a refusal returns the insufficient-content placeholder`, + async () => { + arrangeAnswer("I'm sorry, I cannot analyse this document."); + const analysis = await analyse(service); + + assert.strictEqual( + analysis.error, + 'Insufficient content for AI analysis' + ); + assert.deepStrictEqual(analysis.document, { + tags: [], + correspondent: 'Unknown', + title: `Document ${DOCUMENT_ID}`, + document_date: new Date().toISOString().split('T')[0], + document_type: 'Document', + language: 'und', + }); + assert.deepStrictEqual(analysis.metrics, { + promptTokens: 900, + completionTokens: 120, + totalTokens: 1020, + }); + assert.strictEqual(analysis.truncated, false); + } + ); + } + + await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true }); + + if (failed) { + console.error(`\n${failed} analysis result contract case(s) failed`); + process.exitCode = 1; + return; + } + console.log('\nAll analysis result contract cases passed'); +})(); diff --git a/tests/test-llm-azure-temperature-wiring.js b/tests/test-llm-azure-temperature-wiring.js new file mode 100644 index 00000000..bf8ac446 --- /dev/null +++ b/tests/test-llm-azure-temperature-wiring.js @@ -0,0 +1,178 @@ +/** + * Azure carries the configured temperature on both its paths. + * + * Step 6 decision 7a of the LLM service refactoring + * (docs/llm-service-refactoring.md): analysis sends + * config.aiTemperatureAnalysis (was a hardcoded 0.3) and generateText() + * defaults to config.aiTemperatureGeneration (was a hardcoded 0.7). The rest + * of the request — model, messages, max_tokens, the absence of + * reasoning_effort on a plain deployment — is behaviour that OpenAI and + * Azure share, so those stay pinned exactly as before and only the + * temperature follows the config. + * + * Like the other provider wiring tests, this runs against the singleton + * with a fake client — no network, no live Azure deployment. + */ + +const assert = require('assert'); + +process.env.AI_PROVIDER = 'azure'; +process.env.OPENAI_API_KEY = 'test-key'; +process.env.OPENAI_MODEL = 'gpt-4'; +process.env.AZURE_API_KEY = 'test-key'; +process.env.AZURE_ENDPOINT = 'https://example.invalid'; +process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment'; +process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1'; +process.env.CUSTOM_API_KEY = 'test-key'; +process.env.CUSTOM_MODEL = 'test-model'; +// Deliberately different from the parseTemperature() defaults (0.3 / 0.7): +// a hardcoded value that passes here is not actually reading the config. +process.env.AI_TEMPERATURE_ANALYSIS = '0.2'; +process.env.AI_TEMPERATURE_GENERATION = '1.1'; +process.env.SYSTEM_PROMPT = 'Analyse the document.'; +process.env.USE_PROMPT_TAGS = 'no'; +process.env.RESPONSE_TOKENS = '1000'; +delete process.env.CUSTOM_FIELDS; +delete process.env.CONTENT_MAX_LENGTH; +delete process.env.OLLAMA_API_KEY; + +const config = require('../config/config'); + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +const ANSWER = JSON.stringify({ + title: 'Telekom invoice July', + correspondent: 'Telekom Deutschland GmbH', + tags: ['invoice'], + document_type: 'Rechnung', + document_date: '2026-09-03', + language: 'de', +}); + +function completion(content = ANSWER, finishReason = 'stop') { + return { + choices: [{ message: { content }, finish_reason: finishReason }], + usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 }, + }; +} + +/** Captures the request the SDK is handed; `request` is the mapping target. */ +function fakeChatClient(requests, respond = async () => completion()) { + return { + chat: { + completions: { + create: async (request) => { + requests.push(request); + return respond(request); + }, + }, + }, + models: { list: async () => ({ data: [] }) }, + }; +} + +(async () => { + console.log('\n=== LLM Azure temperature wiring (Finding 7a, step 6) ==='); + + await check('the config really carries the test values', () => { + assert.strictEqual(config.aiTemperatureAnalysis, 0.2); + assert.strictEqual(config.aiTemperatureGeneration, 1.1); + }); + + const azureService = require('../services/azureService'); + + await check('analysis sends AI_TEMPERATURE_ANALYSIS, not 0.3', async () => { + const requests = []; + azureService.client = fakeChatClient(requests); + const result = await azureService.analyzeDocument( + 'Invoice from Telekom.', + [], + [], + [], + 'doc-1' + ); + assert.strictEqual(result.error, undefined, result.error); + assert.strictEqual(requests.length, 1, JSON.stringify(requests)); + const request = requests[0]; + assert.strictEqual(request.model, 'test-deployment'); + assert.strictEqual(request.temperature, 0.2); + assert.deepStrictEqual( + request.messages.map((m) => [m.role, typeof m.content]), + [ + ['system', 'string'], + ['user', 'string'], + ], + JSON.stringify(request.messages) + ); + }); + + await check( + 'generateText() defaults to AI_TEMPERATURE_GENERATION, not 0.7', + async () => { + const requests = []; + azureService.client = fakeChatClient(requests, async () => + completion('The answer.') + ); + const text = await azureService.generateText('hello'); + assert.strictEqual(text, 'The answer.'); + assert.strictEqual(requests.length, 1, JSON.stringify(requests)); + const request = requests[0]; + assert.strictEqual(request.model, 'test-deployment'); + assert.strictEqual(request.temperature, 1.1); + assert.deepStrictEqual(request.messages, [ + { role: 'user', content: 'hello' }, + ]); + assert.strictEqual(request.max_tokens, Number(config.responseTokens)); + } + ); + + await check('the options override still wins for one call', async () => { + const requests = []; + azureService.client = fakeChatClient(requests, async () => + completion('The answer.') + ); + await azureService.generateText('hello', { + systemPrompt: 'You summarise.', + temperature: 0, + maxTokens: 42, + }); + assert.strictEqual(requests.length, 1, JSON.stringify(requests)); + const request = requests[0]; + assert.strictEqual(request.temperature, 0); + assert.strictEqual(request.max_tokens, 42); + assert.deepStrictEqual( + request.messages.map((m) => m.role), + ['system', 'user'] + ); + }); + + await check( + 'no reasoning_effort on a plain deployment even when thinking was switched off', + async () => { + const requests = []; + azureService.client = fakeChatClient(requests, async () => + completion('The answer.') + ); + await azureService.generateText('hello', { reasoning: false }); + assert.strictEqual(requests.length, 1, JSON.stringify(requests)); + const request = requests[0]; + assert.ok(!('reasoning_effort' in request), JSON.stringify(request)); + assert.strictEqual(request.temperature, 1.1); + } + ); + + if (failed > 0) { + console.error(`\n${failed} Azure temperature wiring case(s) failed`); + process.exit(1); + } + console.log('\nAll Azure temperature wiring cases passed'); +})(); diff --git a/tests/test-llm-document-analyzer.js b/tests/test-llm-document-analyzer.js new file mode 100644 index 00000000..d71fef8f --- /dev/null +++ b/tests/test-llm-document-analyzer.js @@ -0,0 +1,295 @@ +/** + * The DocumentAnalyzer on its own, every collaborator a fake. + * + * Step 6 of the LLM service refactoring (docs/llm-service-refactoring.md): + * the four providers are composition roots over this one class, so the order + * of its steps is the business logic of all four. The provider suites + * (llm-analysis-result-contract, llm-provider-parity, ...) pin the outcome + * through the real singletons; this one pins what the analyzer asks of its + * collaborators, which those suites cannot see. + */ + +const assert = require('assert'); + +process.env.SYSTEM_PROMPT = 'Analyse the document.'; +process.env.USE_PROMPT_TAGS = 'no'; +process.env.RESPONSE_TOKENS = '700'; + +const { DocumentAnalyzer } = require('../services/llm/documentAnalyzer'); +const responseParser = require('../services/llm/responseParser'); + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +const METRICS = { promptTokens: 9, completionTokens: 3, totalTokens: 12 }; +const ANSWER = JSON.stringify({ tags: ['invoice'], correspondent: 'Telekom' }); + +/** An analyzer whose collaborators record what they were asked. */ +function analyzer({ payload = ANSWER, ...overrides } = {}) { + const calls = { complete: [], reserve: [], fit: [], prompt: [], cut: [] }; + const transport = { + client: {}, + lastGenerateTextUsage: null, + initialize() {}, + isReady() { + return Boolean(this.client); + }, + async complete(request) { + calls.complete.push(request); + if (payload instanceof Error) throw payload; + return { payload, metrics: METRICS }; + }, + async generate(...args) { + calls.generate = args; + return 'generated'; + }, + }; + const subject = new DocumentAnalyzer({ + label: 'Fake', + transport, + tokenBudget: { + async reserve(frame, extras, model) { + calls.reserve.push({ frame, extras, model }); + return { availableTokens: 4 }; + }, + async fit(content, availableTokens) { + calls.fit.push({ content, availableTokens }); + return content.slice(0, availableTokens); + }, + count: (text) => text.length, + cut: (text, maxTokens) => { + calls.cut.push(maxTokens); + return text.slice(0, maxTokens); + }, + }, + promptLayout: { + frame: (instructions) => `frame:${instructions}`, + build: (instructions, content) => ({ + systemPrompt: `system:${instructions}`, + userContent: `user:${content}`, + }), + }, + promptBuilder: { + buildAnalysisPrompt(input) { + calls.prompt.push(input); + return { systemPrompt: 'instructions', promptTags: '' }; + }, + }, + responseParser, + resolveModel: () => 'fake-model', + checkStatus: async (provider) => ({ status: 'ok', provider }), + clientNotReadyMessage: 'Fake client not initialized', + cacheThumbnail: async () => false, + ...overrides, + }); + return { subject, transport, calls }; +} + +(async () => { + console.log('\n=== LLM DocumentAnalyzer ==='); + + await check('the request is laid out, budgeted and sent once', async () => { + const { subject, calls } = analyzer({ + analysisTemperature: (model) => (model === 'fake-model' ? 0.4 : 1), + responseSchema: { type: 'object' }, + contextWindow: (text, responseTokens) => text.length + responseTokens, + }); + const result = await subject.analyzeDocument('abcdefgh', [], [], [], 1); + + assert.strictEqual(result.error, undefined, result.error); + assert.deepStrictEqual(calls.reserve, [ + { frame: 'frame:instructions', extras: [], model: 'fake-model' }, + ]); + assert.deepStrictEqual(calls.complete, [ + { + systemPrompt: 'system:instructions', + userContent: 'user:abcd', + model: 'fake-model', + temperature: 0.4, + maxResponseTokens: 700, + schema: { type: 'object' }, + numCtx: 'user:abcd'.length + 700, + }, + ]); + }); + + await check('a success is default-shaped and knows it was cut', async () => { + const { subject } = analyzer(); + const result = await subject.analyzeDocument('abcdefgh', [], [], [], 1); + assert.deepStrictEqual(result, { + document: { + tags: ['invoice'], + correspondent: 'Telekom', + title: null, + document_date: null, + document_type: null, + language: null, + custom_fields: null, + }, + metrics: METRICS, + truncated: true, + }); + }); + + await check('without a context window no numCtx is sent', async () => { + const { subject, calls } = analyzer(); + await subject.analyzeDocument('abc', [], [], [], 1); + assert.strictEqual(calls.complete[0].numCtx, undefined); + assert.strictEqual(calls.complete[0].schema, null); + }); + + await check( + 'the metadata check sees the answer before the defaults', + async () => { + const { subject } = analyzer({ payload: '{"title": "only a title"}' }); + const result = await subject.analyzeDocument('abc', [], [], [], 1); + assert.match(result.error, /could not determine assignable metadata/); + assert.deepStrictEqual(result.document, { + tags: [], + correspondent: null, + }); + } + ); + + await check('a refusal returns the placeholder with its tokens', async () => { + const { subject } = analyzer({ payload: "I'm sorry, I cannot." }); + const result = await subject.analyzeDocument('abc', [], [], [], 7); + assert.strictEqual(result.error, 'Insufficient content for AI analysis'); + assert.strictEqual(result.document.title, 'Document 7'); + assert.deepStrictEqual(result.metrics, METRICS); + }); + + await check( + 'a refusal phrase inside readable JSON is no refusal', + async () => { + const { subject } = analyzer({ + payload: JSON.stringify({ + tags: ['insufficient funds'], + correspondent: null, + }), + }); + const result = await subject.analyzeDocument('abc', [], [], [], 1); + assert.strictEqual(result.error, undefined, result.error); + assert.deepStrictEqual(result.document.tags, ['insufficient funds']); + } + ); + + await check('without a client nothing is sent', async () => { + const { subject, transport, calls } = analyzer(); + transport.client = null; + const result = await subject.analyzeDocument('abc', [], [], [], 1); + assert.strictEqual(result.error, 'Fake client not initialized'); + assert.strictEqual(calls.complete.length, 0); + }); + + await check( + 'a transport failure is returned, reworded by the provider', + async () => { + const failure = new Error('socket hang up'); + failure.code = 'ECONNRESET'; + const { subject } = analyzer({ + payload: failure, + describeFailure: (error) => `reworded: ${error.message}`, + }); + const result = await subject.analyzeDocument('abc', [], [], [], 1); + assert.deepStrictEqual(result, { + document: { tags: [], correspondent: null }, + metrics: null, + error: 'reworded: socket hang up', + errorCode: 'ECONNRESET', + }); + } + ); + + await check( + 'the provider cut comes first and does not count as truncation', + async () => { + const { subject, calls } = analyzer({ + prepareContent: (content) => content.slice(0, 3), + }); + const result = await subject.analyzeDocument('abcdefgh', [], [], [], 1); + assert.strictEqual(calls.fit[0].content, 'abc'); + assert.strictEqual(result.truncated, false); + } + ); + + await check('external API data is cut to its token share', async () => { + const { subject, calls } = analyzer(); + await subject.analyzeDocument('abc', [], [], [], 1, null, { + externalApiData: 'x'.repeat(600), + }); + assert.deepStrictEqual(calls.cut, [500]); + assert.strictEqual(calls.prompt[0].externalApiData, 'x'.repeat(500)); + }); + + await check('the custom prompt reaches the prompt builder', async () => { + const { subject, calls } = analyzer(); + await subject.analyzeDocument('abc', ['t'], ['c'], ['d'], 1, 'Custom.'); + assert.deepStrictEqual(calls.prompt[0], { + existingTags: ['t'], + existingCorrespondentList: ['c'], + existingDocumentTypesList: ['d'], + externalApiData: null, + customPrompt: 'Custom.', + }); + }); + + await check('generateText() is handed on as it came', async () => { + const { subject, calls } = analyzer(); + const options = { maxTokens: 5 }; + assert.strictEqual(await subject.generateText('hi', options), 'generated'); + assert.deepStrictEqual(calls.generate, ['hi', options]); + }); + + await check( + 'generateText() gets its sizes where the provider sizes the context', + async () => { + const { subject, calls } = analyzer({ + contextWindow: (text, responseTokens) => + text.length * 100 + responseTokens, + }); + await subject.generateText('hi', { maxTokens: 50.9 }); + assert.deepStrictEqual(calls.generate[2], { + responseTokens: 50, + numCtx: 250, + }); + await subject.generateText('hi'); + assert.deepStrictEqual(calls.generate[2], { + responseTokens: 700, + numCtx: 900, + }); + } + ); + + await check( + 'checkStatus() runs the provider probe on the provider', + async () => { + const { subject } = analyzer(); + const status = await subject.checkStatus(); + assert.strictEqual(status.provider, subject); + } + ); + + await check('client and lastGenerateTextUsage are the transport’s', () => { + const { subject, transport } = analyzer(); + const client = {}; + subject.client = client; + subject.lastGenerateTextUsage = METRICS; + assert.strictEqual(transport.client, client); + assert.strictEqual(transport.lastGenerateTextUsage, METRICS); + }); + + if (failed > 0) { + console.error(`\n${failed} DocumentAnalyzer case(s) failed`); + process.exit(1); + } + console.log('\nAll DocumentAnalyzer cases passed'); +})(); diff --git a/tests/test-llm-ocr-fallback-phrases.js b/tests/test-llm-ocr-fallback-phrases.js new file mode 100644 index 00000000..fc8a1677 --- /dev/null +++ b/tests/test-llm-ocr-fallback-phrases.js @@ -0,0 +1,299 @@ +/** + * The OCR fallback phrases, pinned end to end (Finding 4). + * + * Safety net for the LLM service refactoring (docs/llm-service-refactoring.md, + * step 1). server.js:774-821 decides whether a failed analysis goes to the OCR + * queue by matching analysis.error against a fixed list of **substrings** in + * shouldQueueForOcrOnAiError(), and picks the queue reason with + * classifyOcrQueueReasonFromAiError() (services/serviceUtils.js:812-861). + * + * Nothing enforces that link today. Reword a message inside a provider service + * and the analysis still fails, the log still looks right, and the document + * silently stops reaching OCR. That is the regression this test exists to + * catch, so it asserts the full chain per provider: + * + * stub a failing client + * -> analyzeDocument() returns analysis.error + * -> shouldQueueForOcrOnAiError(analysis.error) is true + * -> classifyOcrQueueReasonFromAiError(analysis.error) is the right reason + * + * The matcher's own vocabulary is checked separately, so a phrase deleted from + * the list is caught even if no provider happens to produce it. + */ + +const assert = require('assert'); +const fs = require('fs').promises; + +process.env.AI_PROVIDER = 'openai'; +process.env.OPENAI_API_KEY = 'test-key'; +process.env.OPENAI_MODEL = 'gpt-4'; +process.env.AZURE_API_KEY = 'test-key'; +process.env.AZURE_ENDPOINT = 'https://example.invalid'; +process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment'; +process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1'; +process.env.CUSTOM_API_KEY = 'test-key'; +process.env.CUSTOM_MODEL = 'test-model'; +process.env.SYSTEM_PROMPT = 'Analyse the document.'; + +const { + shouldQueueForOcrOnAiError, + classifyOcrQueueReasonFromAiError, +} = require('../services/serviceUtils'); +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('../services/thumbnailCachePaths'); +const openaiService = require('../services/openaiService'); +const azureService = require('../services/azureService'); +const customService = require('../services/customService'); +const ollamaService = require('../services/ollamaService'); + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +/** Own id so the seeded thumbnail cannot collide with a real cached document. */ +const DOCUMENT_ID = 'testllmocrphrases'; + +/** Enough text that no provider takes an "insufficient content" shortcut. */ +const CONTENT = 'Invoice from Telekom Deutschland GmbH. '.repeat(20); + +/** + * Every marker shouldQueueForOcrOnAiError() knows, with the reason + * classifyOcrQueueReasonFromAiError() derives from it. Copied out deliberately + * rather than imported: importing the list would make the test agree with + * whatever the source says, which is the opposite of pinning it. + */ +const MARKERS = [ + ['insufficient content for ai analysis', 'ai_insufficient_content'], + ['invalid response structure', 'ai_invalid_response_structure'], + ['could not determine assignable metadata', 'ai_invalid_response_structure'], + ['invalid json response from api', 'ai_invalid_json'], + ['invalid api response structure', 'ai_invalid_api_response_structure'], +]; + +function stubClient(service, respond) { + service.client = { + chat: { completions: { create: respond } }, + models: { list: async () => ({ data: [] }) }, + }; +} + +function stubOllamaClient(respond) { + ollamaService.client = { + post: respond, + get: async () => ({ status: 200, data: { models: [] } }), + }; +} + +function openAiCompatibleProvider(label, service) { + return { + label, + service, + /** Answers with `content`, which the service then tries to parse. */ + answer: (content) => + stubClient(service, async () => ({ + choices: [{ message: { content }, finish_reason: 'stop' }], + usage: { + prompt_tokens: 900, + completion_tokens: 120, + total_tokens: 1020, + }, + })), + /** Answers with a payload that has no choices array at all. */ + answerMalformed: () => stubClient(service, async () => ({ usage: {} })), + }; +} + +const PROVIDERS = [ + openAiCompatibleProvider('OpenAI', openaiService), + openAiCompatibleProvider('Azure', azureService), + openAiCompatibleProvider('Custom', customService), + { + label: 'Ollama', + service: ollamaService, + answer: (content) => + stubOllamaClient(async () => ({ + data: { + response: content, + done_reason: 'stop', + eval_count: 120, + prompt_eval_count: 900, + }, + })), + answerMalformed: () => stubOllamaClient(async () => ({ data: {} })), + }, +]; + +const analyse = (service, content = CONTENT) => + service.analyzeDocument(content, [], [], [], DOCUMENT_ID); + +/** + * The assertion that matters: whatever the service worded, the scan loop must + * still route the document to OCR under the expected reason. + */ +const assertRoutesToOcr = (analysis, expectedReason) => { + assert.ok(analysis, 'analyzeDocument returned nothing at all'); + assert.strictEqual( + typeof analysis.error, + 'string', + 'no error message to match against' + ); + assert.ok( + shouldQueueForOcrOnAiError(analysis.error), + `"${analysis.error}" no longer matches any OCR fallback marker` + ); + assert.strictEqual( + classifyOcrQueueReasonFromAiError(analysis.error), + expectedReason, + `"${analysis.error}" classified as the wrong queue reason` + ); +}; + +(async () => { + console.log('\n=== LLM OCR fallback phrases ==='); + + /* --- the matcher's own vocabulary -------------------------------------- */ + + await check('every marker still triggers the fallback', () => { + for (const [marker, reason] of MARKERS) { + assert.ok( + shouldQueueForOcrOnAiError(marker), + `marker dropped from shouldQueueForOcrOnAiError: "${marker}"` + ); + assert.strictEqual( + classifyOcrQueueReasonFromAiError(marker), + reason, + `marker reclassified: "${marker}"` + ); + } + }); + + await check( + 'markers match case-insensitively inside longer sentences', + () => { + // Services word these as sentences, not as bare markers. + assert.ok( + shouldQueueForOcrOnAiError( + 'AI could not determine assignable metadata: no tags or correspondent found' + ) + ); + assert.strictEqual( + classifyOcrQueueReasonFromAiError( + 'AI could not determine assignable metadata: no tags or correspondent found' + ), + 'ai_invalid_response_structure' + ); + } + ); + + await check('an unrelated failure does not reach the OCR queue', () => { + // A dead provider must not be retried as an OCR job. + assert.strictEqual(shouldQueueForOcrOnAiError('socket hang up'), false); + assert.strictEqual(shouldQueueForOcrOnAiError(''), false); + assert.strictEqual(shouldQueueForOcrOnAiError(undefined), false); + assert.strictEqual( + classifyOcrQueueReasonFromAiError('socket hang up'), + 'ai_failed_unknown' + ); + }); + + /* --- the providers still produce them ---------------------------------- */ + + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile( + getThumbnailCachePath(DOCUMENT_ID), + Buffer.from([0x89, 0x50, 0x4e, 0x47]) + ); + + for (const { label, service, answer } of PROVIDERS) { + await check( + `${label}: an unparseable answer routes to OCR as invalid JSON`, + async () => { + answer('the model apologised instead of answering'); + assertRoutesToOcr(await analyse(service), 'ai_invalid_json'); + } + ); + } + + for (const { label, service, answerMalformed } of PROVIDERS.filter( + (p) => p.label !== 'Ollama' + )) { + await check( + `${label}: a payload with no choices routes to OCR`, + async () => { + answerMalformed(); + assertRoutesToOcr( + await analyse(service), + 'ai_invalid_api_response_structure' + ); + } + ); + } + + /* Pinned as it is today so the divergence is visible. Ollama words an empty + payload "No response data from Ollama API", which matches no marker, so + the document is never queued for OCR while the other three are. When the + refactor unifies the wording this case starts failing — fold it into the + loop above then. */ + await check( + 'Ollama: an empty payload does NOT route to OCR (known drift)', + async () => { + const ollama = PROVIDERS.find((p) => p.label === 'Ollama'); + ollama.answerMalformed(); + const analysis = await analyse(ollama.service); + + assert.match(analysis.error, /No response data from Ollama API/); + assert.strictEqual( + shouldQueueForOcrOnAiError(analysis.error), + false, + 'Ollama now reaches OCR here — unify the wording and update this case' + ); + } + ); + + /* Finding 7c: all four providers implement the "insufficient content" + early return — an unparseable answer containing "i'm sorry" / "i cannot" + / "insufficient" routes to OCR under that reason, not as invalid JSON. */ + for (const { label, service, answer } of PROVIDERS) { + await check( + `${label}: a refusal routes to OCR as insufficient content`, + async () => { + answer("I'm sorry, there is insufficient text to analyse."); + assertRoutesToOcr(await analyse(service), 'ai_insufficient_content'); + } + ); + } + + /* All four providers raise "could not determine assignable metadata", + so every one of them routes the document to OCR as an invalid + response structure. */ + for (const { label, service, answer } of PROVIDERS) { + await check( + `${label}: an answer without metadata routes to OCR`, + async () => { + answer(JSON.stringify({ note: 'nothing usable here' })); + assertRoutesToOcr( + await analyse(service), + 'ai_invalid_response_structure' + ); + } + ); + } + + await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true }); + + if (failed) { + console.error(`\n${failed} OCR fallback phrase case(s) failed`); + process.exitCode = 1; + return; + } + console.log('\nAll OCR fallback phrase cases passed'); +})(); diff --git a/tests/test-llm-prompt-builder.js b/tests/test-llm-prompt-builder.js new file mode 100644 index 00000000..d23f0b2c --- /dev/null +++ b/tests/test-llm-prompt-builder.js @@ -0,0 +1,468 @@ +/** + * The analysis prompt, built once for all four providers. + * + * Step 2 of the LLM service refactoring (docs/llm-service-refactoring.md). + * The %CUSTOMFIELDS% block and the prompt cascade after it — pre-existing + * data, mustHavePrompt, restriction placeholders, external API data, + * USE_PROMPT_TAGS and the webhook's custom prompt — used to be copied into + * every provider service, six times in all. They now live in + * services/llm/customFieldsTemplate.js and services/llm/promptBuilder.js. + * + * The first half pins what the builder produces. The second half checks that + * every provider really sends what the builder produced, so a provider that + * keeps a private copy, or drops part of the builder's output, fails here. + * + * One drift is resolved on the way and pinned as such: Ollama called its async + * _validateAndTruncateExternalApiData() without awaiting it and sent + * "[object Promise]" instead of the external API data. + */ + +const assert = require('assert'); +const fs = require('fs').promises; + +process.env.AI_PROVIDER = 'openai'; +process.env.OPENAI_API_KEY = 'test-key'; +process.env.OPENAI_MODEL = 'gpt-4'; +process.env.AZURE_API_KEY = 'test-key'; +process.env.AZURE_ENDPOINT = 'https://example.invalid'; +process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment'; +process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1'; +process.env.CUSTOM_API_KEY = 'test-key'; +process.env.CUSTOM_MODEL = 'test-model'; + +const config = require('../config/config'); +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('../services/thumbnailCachePaths'); +const { + buildCustomFieldsBlock, +} = require('../services/llm/customFieldsTemplate'); +const { buildAnalysisPrompt } = require('../services/llm/promptBuilder'); +const openaiService = require('../services/openaiService'); +const azureService = require('../services/azureService'); +const customService = require('../services/customService'); +const ollamaService = require('../services/ollamaService'); + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +/** Own id so the seeded thumbnail cannot collide with a real cached document. */ +const DOCUMENT_ID = 'testLlmPromptBuilder'; + +const SYSTEM_PROMPT = + 'Analyse the document. Tags: %RESTRICTED_TAGS% | Correspondents: %RESTRICTED_CORRESPONDENTS% | Types: %RESTRICTED_DOCUMENT_TYPES%'; + +const CUSTOM_FIELDS = JSON.stringify({ + custom_fields: [ + { value: 'Due date', data_type: 'date' }, + { value: 'Paid', data_type: 'boolean' }, + { value: 'Amount', data_type: 'monetary' }, + ], +}); + +// Both shapes reach the services (issue #262): entity objects and plain names. +const TAGS = [ + { id: 1, name: 'invoice' }, + { id: 2, name: 'contract' }, +]; +const CORRESPONDENTS = [{ id: 7, name: 'Acme Corp' }]; +const DOCUMENT_TYPES = ['Offer', 'Notice']; + +const ANSWER = JSON.stringify({ + title: 'Telekom invoice July', + correspondent: 'Telekom Deutschland GmbH', + tags: ['invoice'], + document_type: 'Rechnung', + document_date: '2026-09-03', + language: 'de', +}); + +const SETTINGS = { + env: ['SYSTEM_PROMPT', 'CUSTOM_FIELDS', 'USE_PROMPT_TAGS', 'PROMPT_TAGS'], + config: [ + 'useExistingData', + 'restrictToExistingTags', + 'restrictToExistingCorrespondents', + ], +}; + +/** Puts every setting the prompt reads into a known state for one case. */ +function arrange({ env = {}, settings = {} } = {}) { + const values = { + SYSTEM_PROMPT, + CUSTOM_FIELDS, + USE_PROMPT_TAGS: 'no', + PROMPT_TAGS: '', + ...env, + }; + for (const key of SETTINGS.env) { + if (values[key] === undefined) delete process.env[key]; + else process.env[key] = values[key]; + } + Object.assign( + config, + { + useExistingData: 'no', + restrictToExistingTags: 'no', + restrictToExistingCorrespondents: 'no', + }, + settings + ); +} + +function build(overrides = {}) { + return buildAnalysisPrompt({ + existingTags: TAGS, + existingCorrespondentList: CORRESPONDENTS, + existingDocumentTypesList: DOCUMENT_TYPES, + ...overrides, + }); +} + +/* --- the providers, each with a client that records what it was sent ----- */ + +function recordingChatClient(service, requests) { + service.client = { + chat: { + completions: { + create: async (request) => { + requests.push(request); + return { + choices: [{ message: { content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 }, + }; + }, + }, + }, + models: { list: async () => ({ data: [] }) }, + }; +} + +function recordingOllamaClient(requests) { + ollamaService.client = { + post: async (url, body) => { + requests.push(body); + return { + data: { + response: ANSWER, + done_reason: 'stop', + eval_count: 3, + prompt_eval_count: 9, + }, + }; + }, + get: async () => ({ status: 200, data: { models: [] } }), + }; +} + +const CHAT_PROVIDERS = [ + ['OpenAI', openaiService], + ['Azure', azureService], + ['Custom', customService], +]; + +/** The system message a chat provider sends for one analysis. */ +async function chatSystemPrompt(service, customPrompt = null, options = {}) { + const requests = []; + recordingChatClient(service, requests); + const analysis = await service.analyzeDocument( + 'Document body text.', + TAGS, + CORRESPONDENTS, + DOCUMENT_TYPES, + DOCUMENT_ID, + customPrompt, + options + ); + assert.strictEqual(analysis.error, undefined, 'the analysis succeeded'); + assert.strictEqual(requests.length, 1, 'one request was sent'); + const system = requests[0].messages.find((m) => m.role === 'system'); + assert.ok(system, 'the request carries a system message'); + return system.content; +} + +/** The prompt Ollama sends for one analysis. */ +async function ollamaPrompt(customPrompt = null, options = {}) { + const requests = []; + recordingOllamaClient(requests); + const analysis = await ollamaService.analyzeDocument( + 'Document body text.', + TAGS, + CORRESPONDENTS, + DOCUMENT_TYPES, + DOCUMENT_ID, + customPrompt, + options + ); + assert.strictEqual(analysis.error, undefined, 'the analysis succeeded'); + assert.strictEqual(requests.length, 1, 'one request was sent'); + return requests[0].prompt; +} + +(async () => { + console.log('\n=== LLM prompt builder ==='); + + /* --- the %CUSTOMFIELDS% block ------------------------------------------ */ + + await check('the custom fields block gives each data type its hint', () => { + arrange(); + const block = buildCustomFieldsBlock(); + assert.ok(block.startsWith('"custom_fields": '), block); + assert.match(block, /"field_name": "Due date"/); + assert.match( + block, + /"value": "Fill in the date in ISO 8601 format \(YYYY-MM-DD\) based on your analysis"/ + ); + assert.match(block, /"field_name": "Paid"/); + assert.match( + block, + /"value": "Fill in 'true' or 'false' based on your analysis"/ + ); + assert.match(block, /"field_name": "Amount"/); + assert.match(block, /"value": "Fill in the value based on your analysis"/); + }); + + await check( + 'the custom fields block is indented to sit inside the JSON example', + () => { + arrange(); + const lines = buildCustomFieldsBlock().split('\n'); + assert.strictEqual(lines[0], '"custom_fields": {'); + assert.strictEqual(lines[1], ' "0": {'); + assert.strictEqual(lines[lines.length - 1], ' }'); + } + ); + + await check( + 'unreadable or missing CUSTOM_FIELDS yield an empty block', + () => { + for (const value of ['{not json', undefined]) { + arrange({ env: { CUSTOM_FIELDS: value } }); + assert.strictEqual(buildCustomFieldsBlock(), '"custom_fields": {}'); + } + } + ); + + /* --- the cascade ------------------------------------------------------- */ + + await check( + 'by default: SYSTEM_PROMPT, then mustHavePrompt with the block', + () => { + arrange(); + const { systemPrompt, promptTags } = build(); + assert.ok(systemPrompt.startsWith('Analyse the document.'), systemPrompt); + assert.ok( + systemPrompt.includes( + config.mustHavePrompt + .replace('%CUSTOMFIELDS%', buildCustomFieldsBlock()) + .slice(0, 60) + ), + 'mustHavePrompt follows the system prompt' + ); + assert.ok(!systemPrompt.includes('%CUSTOMFIELDS%')); + assert.ok(!systemPrompt.includes('Pre-existing tags')); + assert.strictEqual(promptTags, ''); + } + ); + + await check('USE_EXISTING_DATA lists what Paperless-ngx already has', () => { + arrange({ settings: { useExistingData: 'yes' } }); + const { systemPrompt } = build(); + assert.ok(systemPrompt.includes('Pre-existing tags: invoice, contract')); + assert.ok(systemPrompt.includes('Pre-existing correspondents: Acme Corp')); + assert.ok( + systemPrompt.includes('Pre-existing document types: Offer, Notice') + ); + assert.ok(!systemPrompt.includes('[object Object]')); + assert.ok( + systemPrompt.indexOf('Pre-existing tags') < + systemPrompt.indexOf('Analyse the document.'), + 'the existing data comes before the system prompt' + ); + }); + + await check('a restriction switches the existing-data block off', () => { + for (const restriction of [ + 'restrictToExistingTags', + 'restrictToExistingCorrespondents', + ]) { + arrange({ settings: { useExistingData: 'yes', [restriction]: 'yes' } }); + assert.ok( + !build().systemPrompt.includes('Pre-existing tags'), + restriction + ); + } + }); + + await check('restriction placeholders resolve to the existing names', () => { + arrange(); + const { systemPrompt } = build(); + assert.ok(systemPrompt.includes('Tags: invoice, contract |'), systemPrompt); + assert.ok(systemPrompt.includes('Correspondents: Acme Corp |')); + assert.ok(systemPrompt.includes('Types: Offer, Notice')); + assert.ok(!systemPrompt.includes('%RESTRICTED_')); + }); + + await check( + 'external API data is appended after the placeholders are resolved', + () => { + arrange(); + const { systemPrompt } = build({ + externalApiData: 'IBAN DE00, see %RESTRICTED_TAGS%', + }); + assert.ok( + systemPrompt.endsWith( + '\n\nAdditional context from external API:\nIBAN DE00, see %RESTRICTED_TAGS%' + ), + systemPrompt + ); + } + ); + + await check( + 'USE_PROMPT_TAGS replaces the prompt and hands out PROMPT_TAGS', + () => { + arrange({ + env: { USE_PROMPT_TAGS: 'yes', PROMPT_TAGS: 'invoice, bank' }, + }); + const { systemPrompt, promptTags } = build({ + externalApiData: 'ignored', + }); + assert.ok( + systemPrompt.includes( + 'Take these tags and try to match one or more to the document content.' + ) + ); + assert.ok(systemPrompt.endsWith(config.specialPromptPreDefinedTags)); + assert.ok(!systemPrompt.includes('Analyse the document.')); + assert.ok(!systemPrompt.includes('ignored')); + assert.strictEqual(promptTags, 'invoice, bank'); + } + ); + + await check( + 'a custom prompt wins over everything, mustHavePrompt still follows', + () => { + arrange({ + env: { USE_PROMPT_TAGS: 'yes', PROMPT_TAGS: 'invoice' }, + settings: { useExistingData: 'yes' }, + }); + const { systemPrompt, promptTags } = build({ + customPrompt: 'Webhook prompt.', + externalApiData: 'ignored', + }); + assert.strictEqual( + systemPrompt, + 'Webhook prompt.\n\n' + + config.mustHavePrompt.replace( + '%CUSTOMFIELDS%', + buildCustomFieldsBlock() + ) + ); + assert.strictEqual( + promptTags, + 'invoice', + 'the tag list is still reported' + ); + } + ); + + /* --- every provider sends what the builder built ------------------------ */ + + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile( + getThumbnailCachePath(DOCUMENT_ID), + Buffer.from([0x89, 0x50, 0x4e, 0x47]) + ); + + const PROVIDER_CASES = [ + ['by default', {}, {}], + ['with existing data', { settings: { useExistingData: 'yes' } }, {}], + ['with a custom prompt', {}, { customPrompt: 'Webhook prompt.' }], + [ + 'with external API data', + {}, + { externalApiData: { iban: 'DE00', note: 'x' } }, + ], + ]; + + for (const [label, service] of CHAT_PROVIDERS) { + for (const [name, arrangement, input] of PROVIDER_CASES) { + await check( + `${label}: the system message is the builder's ${name}`, + async () => { + arrange(arrangement); + const sent = await chatSystemPrompt( + service, + input.customPrompt ?? null, + input.externalApiData + ? { externalApiData: input.externalApiData } + : {} + ); + const expected = build({ + customPrompt: input.customPrompt ?? null, + externalApiData: input.externalApiData + ? JSON.stringify(input.externalApiData, null, 2) + : null, + }).systemPrompt; + assert.strictEqual(sent, expected); + } + ); + } + } + + for (const [name, arrangement, input] of PROVIDER_CASES) { + await check( + `Ollama: the prompt starts with the builder's ${name}`, + async () => { + arrange(arrangement); + const sent = await ollamaPrompt( + input.customPrompt ?? null, + input.externalApiData + ? { externalApiData: input.externalApiData } + : {} + ); + const expected = build({ + customPrompt: input.customPrompt ?? null, + externalApiData: input.externalApiData + ? JSON.stringify(input.externalApiData, null, 2) + : null, + }).systemPrompt; + assert.ok(sent.startsWith(expected), `sent:\n${sent}`); + assert.ok( + sent.includes('"Document body text."'), + 'the content follows' + ); + } + ); + } + + await check( + 'Ollama: external API data is the data, not "[object Promise]"', + async () => { + arrange(); + const sent = await ollamaPrompt(null, { + externalApiData: { iban: 'DE00' }, + }); + assert.ok(!sent.includes('[object Promise]'), sent); + assert.ok(sent.includes('"iban": "DE00"'), sent); + } + ); + + await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true }); + + if (failed > 0) { + console.error(`\n${failed} prompt builder case(s) failed`); + process.exit(1); + } + console.log('\nAll prompt builder cases passed'); +})(); diff --git a/tests/test-llm-provider-parity.js b/tests/test-llm-provider-parity.js new file mode 100644 index 00000000..da6edbd8 --- /dev/null +++ b/tests/test-llm-provider-parity.js @@ -0,0 +1,212 @@ +/** + * The same answer gives the same result on all four providers, and the + * four are one DocumentAnalyzer each behind a re-export. + * + * Step 6 of the LLM service refactoring (docs/llm-service-refactoring.md, + * "Done means"): once the four providers share one DocumentAnalyzer, an + * answer must come back byte-identical from each of them — the document, the + * metrics, the truncation flag and, for a failure, the error and its code. + * Every stub reports the same token counts, so a difference anywhere in the + * result is a difference in the analysis itself. + * + * Deliberately left out, because step 6 keeps them as they are (the plan's + * "Not decided" list): transport failures, where only Custom normalizes a + * timeout (rows e/m), and an empty payload, which Ollama words "No response + * data from Ollama API" (row l). + */ + +const assert = require('assert'); +const fs = require('fs').promises; +const path = require('path'); + +process.env.AI_PROVIDER = 'openai'; +process.env.OPENAI_API_KEY = 'test-key'; +process.env.OPENAI_MODEL = 'gpt-4'; +process.env.AZURE_API_KEY = 'test-key'; +process.env.AZURE_ENDPOINT = 'https://example.invalid'; +process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment'; +process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1'; +process.env.CUSTOM_API_KEY = 'test-key'; +process.env.CUSTOM_MODEL = 'test-model'; +process.env.SYSTEM_PROMPT = 'Analyse the document.'; +process.env.USE_PROMPT_TAGS = 'no'; +delete process.env.CUSTOM_FIELDS; +delete process.env.CONTENT_MAX_LENGTH; + +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('../services/thumbnailCachePaths'); + +const { DocumentAnalyzer } = require('../services/llm/documentAnalyzer'); + +const PROVIDERS = { + OpenAI: require('../services/openaiService'), + Azure: require('../services/azureService'), + Custom: require('../services/customService'), + Ollama: require('../services/ollamaService'), +}; + +/** The module each provider's legacy path re-exports. */ +const ROOTS = { + OpenAI: ['openaiService', 'openaiProvider'], + Azure: ['azureService', 'azureProvider'], + Custom: ['customService', 'customProvider'], + Ollama: ['ollamaService', 'ollamaProvider'], +}; + +/** Own id so the seeded thumbnail cannot collide with a real cached document. */ +const DOCUMENT_ID = 'testLlmProviderParity'; + +/** Short enough that no provider cuts it. */ +const CONTENT = 'Invoice from Telekom Deutschland GmbH, July 2026.'; + +/** The answer text every provider receives, by case. */ +const ANSWERS = [ + [ + 'a complete answer', + JSON.stringify({ + title: 'Telekom invoice July', + correspondent: 'Telekom Deutschland GmbH', + tags: ['invoice', 'telecom'], + document_type: 'Invoice', + document_date: '2026-07-31', + language: 'en', + custom_fields: [{ field_name: 'Amount', value: '49.99' }], + }), + ], + [ + 'a partial answer', + JSON.stringify({ tags: ['invoice'], correspondent: 'Telekom' }), + ], + [ + 'an answer with an invented field', + JSON.stringify({ tags: ['invoice'], correspondent: null, mood: 'calm' }), + ], + [ + 'an answer that found nothing', + JSON.stringify({ tags: [], correspondent: null }), + ], + ['an answer without metadata', JSON.stringify({ title: 'Telekom' })], + [ + 'tags given as a string', + JSON.stringify({ tags: 'invoice', correspondent: 'Telekom' }), + ], + [ + 'an answer in a code fence', + '```json\n' + JSON.stringify({ tags: ['a'], correspondent: 'C' }) + '\n```', + ], + [ + 'an answer after a reasoning block', + 'Looks like an invoice.' + + JSON.stringify({ tags: ['a'], correspondent: 'C' }), + ], + ['an answer with a trailing comma', '{"tags": ["a"], "correspondent": "C",}'], + ['a refusal', "I'm sorry, I cannot analyse this document."], + ['no JSON at all', 'The document is a scanned letter.'], + ['an object cut off mid-way', '{"title": "Telekom", "tags": ["inv'], +]; + +/** Every provider answers `text` and reports 9 + 3 tokens. */ +function answerWith(label, text) { + if (label === 'Ollama') { + PROVIDERS.Ollama.client = { + post: async () => ({ + data: { + response: text, + done_reason: 'stop', + prompt_eval_count: 9, + eval_count: 3, + }, + }), + get: async () => ({ status: 200, data: { models: [] } }), + }; + return; + } + PROVIDERS[label].client = { + chat: { + completions: { + create: async () => ({ + choices: [{ message: { content: text }, finish_reason: 'stop' }], + usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 }, + }), + }, + }, + models: { list: async () => ({ data: [] }) }, + }; +} + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +(async () => { + console.log('\n=== LLM provider parity ==='); + + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile( + getThumbnailCachePath(DOCUMENT_ID), + Buffer.from([0x89, 0x50, 0x4e, 0x47]) + ); + + /* The structure behind the parity: one analyzer, four composition roots, + and the old module paths as plain re-exports. */ + for (const [label, [service, provider]] of Object.entries(ROOTS)) { + await check(`${label}: a DocumentAnalyzer behind a re-export`, async () => { + assert.ok(PROVIDERS[label] instanceof DocumentAnalyzer); + assert.strictEqual( + PROVIDERS[label], + require(`../services/llm/providers/${provider}`) + ); + const source = await fs.readFile( + path.join(__dirname, '..', 'services', `${service}.js`), + 'utf8' + ); + const code = source.replace(/^\/\/.*$/gm, '').trim(); + assert.strictEqual( + code, + `module.exports = require('./llm/providers/${provider}');` + ); + }); + } + + for (const [name, text] of ANSWERS) { + await check(`${name}: byte-identical on all four`, async () => { + const results = {}; + for (const label of Object.keys(PROVIDERS)) { + answerWith(label, text); + results[label] = JSON.stringify( + await PROVIDERS[label].analyzeDocument( + CONTENT, + [], + [], + [], + DOCUMENT_ID + ) + ); + } + for (const label of Object.keys(PROVIDERS)) { + assert.strictEqual( + results[label], + results.OpenAI, + `${label} differs from OpenAI` + ); + } + }); + } + + await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true }); + + if (failed > 0) { + console.error(`\n${failed} provider parity case(s) failed`); + process.exit(1); + } + console.log('\nAll provider parity cases passed'); +})(); diff --git a/tests/test-llm-response-parser.js b/tests/test-llm-response-parser.js new file mode 100644 index 00000000..0c4ea180 --- /dev/null +++ b/tests/test-llm-response-parser.js @@ -0,0 +1,391 @@ +/** + * The analysis answer, parsed once for all four providers. + * + * Step 3 of the LLM service refactoring (docs/llm-service-refactoring.md). + * Finding 7d: there were three parsers. Custom matched braces and stripped + * blocks, openai and azure stripped fences and called JSON.parse on + * the rest, Ollama took everything from the first "{" to the last "}" and + * repaired trailing commas and unquoted keys. The same answer could therefore + * succeed on one provider and fail on the next. services/llm/responseParser.js + * is the union of the three. + * + * The first half pins what the parser accepts and how it fails. The second + * half sends the same answers through every provider, so a provider that + * keeps a private parser, or narrows the result again, fails here. + * + * Every failure must keep the "Invalid JSON response from API" phrase: the + * OCR fallback matches on it (Finding 4). + */ + +const assert = require('assert'); +const fs = require('fs').promises; + +process.env.AI_PROVIDER = 'openai'; +process.env.OPENAI_API_KEY = 'test-key'; +process.env.OPENAI_MODEL = 'gpt-4'; +process.env.AZURE_API_KEY = 'test-key'; +process.env.AZURE_ENDPOINT = 'https://example.invalid'; +process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment'; +process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1'; +process.env.CUSTOM_API_KEY = 'test-key'; +process.env.CUSTOM_MODEL = 'test-model'; +process.env.SYSTEM_PROMPT = 'Analyse the document.'; +delete process.env.CUSTOM_FIELDS; + +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('../services/thumbnailCachePaths'); +const { + parseAnalysisResponse, + stripResponseWrapping, + isInsufficientContentAnswer, +} = require('../services/llm/responseParser'); +const { shouldQueueForOcrOnAiError } = require('../services/serviceUtils'); +const openaiService = require('../services/openaiService'); +const azureService = require('../services/azureService'); +const customService = require('../services/customService'); +const ollamaService = require('../services/ollamaService'); + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +/** Own id so the seeded thumbnail cannot collide with a real cached document. */ +const DOCUMENT_ID = 'testLlmResponseParser'; + +const RESULT = { + title: 'Telekom invoice July', + correspondent: 'Telekom Deutschland GmbH', + tags: ['invoice'], + document_type: 'Rechnung', + document_date: '2026-09-03', + language: 'de', + custom_fields: { 0: { field_name: 'Amount', value: '12.50' } }, +}; +const JSON_ANSWER = JSON.stringify(RESULT, null, 2); + +/** Answers every provider has to read, and what each one is a case of. */ +const READABLE_ANSWERS = [ + ['plain JSON', JSON_ANSWER], + ['a fenced block', '```json\n' + JSON_ANSWER + '\n```'], + ['an upper-case fence', '```JSON\n' + JSON_ANSWER + '\n```'], + [ + 'a block in front', + 'The sender is "Telekom" {maybe}.\n' + JSON_ANSWER, + ], + [ + 'prose around the object', + 'Here is the analysis:\n' + JSON_ANSWER + '\nLet me know {if} it helps.', + ], + ['a trailing comma', JSON_ANSWER.replace(/\n}$/, ',\n}')], + [ + 'braces inside a string value', + 'Result: ' + + JSON_ANSWER.replace('Telekom invoice July', 'Invoice {July} }') + + ' (done)', + ], +]; + +/** The object a READABLE_ANSWERS entry stands for. */ +function expectedFor(name) { + if (name === 'partial answer') { + /* PARTIAL_ANSWER passed through withDocumentDefaults(): the three + assigned fields survive, the missing ones come back as null, and + the invented field is dropped. */ + return { + tags: PARTIAL_ANSWER.tags, + correspondent: PARTIAL_ANSWER.correspondent, + title: PARTIAL_ANSWER.title, + document_date: null, + document_type: null, + language: null, + custom_fields: null, + }; + } + /* The title ends in "}"; the object's real closing brace is the one + after the string. */ + return name === 'braces inside a string value' + ? { ...RESULT, title: 'Invoice {July} }' } + : RESULT; +} + +/** + * A parseable answer the model leaves incomplete: it assigns tags and a + * correspondent, so it passes assertAssignableMetadata, but omits every + * other field. Field defaults for all four (step 6) must bring the missing + * ones back as null and drop whatever the model invented. + */ +const PARTIAL_ANSWER = { + tags: ['invoice'], + correspondent: 'Telekom Deutschland GmbH', + title: 'Telekom', + unknown_field: 'invented by the model', +}; + +/** Answers no provider can read; none of them may count as a success. */ +const UNREADABLE_ANSWERS = [ + ['no JSON at all', 'The document is a scanned letter.'], + ['an object cut off mid-way', '{"title": "Telekom", "tags": ["inv'], + ['an object beyond repair', '{"title": "Telekom" "tags" ["invoice"]}'], +]; + +/* --- the providers, each with a client that answers a fixed text ---------- */ + +function chatClient(service, content) { + service.client = { + chat: { + completions: { + create: async () => ({ + choices: [{ message: { content }, finish_reason: 'stop' }], + usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 }, + }), + }, + }, + models: { list: async () => ({ data: [] }) }, + }; +} + +function ollamaClient(response) { + ollamaService.client = { + post: async () => ({ + data: { + response, + done_reason: 'stop', + eval_count: 3, + prompt_eval_count: 9, + }, + }), + get: async () => ({ status: 200, data: { models: [] } }), + }; +} + +const PROVIDERS = [ + ['OpenAI', openaiService, (answer) => chatClient(openaiService, answer)], + ['Azure', azureService, (answer) => chatClient(azureService, answer)], + ['Custom', customService, (answer) => chatClient(customService, answer)], + ['Ollama', ollamaService, ollamaClient], +]; + +async function analyze(service, answerWith, answer) { + answerWith(answer); + return service.analyzeDocument( + 'Document body text.', + [], + [], + [], + DOCUMENT_ID + ); +} + +/** Every provider shapes its success with withDocumentDefaults(). */ +function sameDocument(label, actual, expected) { + label; // the signature keeps the call sites; the check is one for all four + assert.deepStrictEqual( + actual, + { + tags: expected.tags, + correspondent: expected.correspondent, + title: expected.title, + document_date: expected.document_date, + document_type: expected.document_type, + language: expected.language, + custom_fields: expected.custom_fields, + }, + 'every provider returns the seven default-shaped document fields' + ); +} + +(async () => { + console.log('\n=== LLM response parser ==='); + + /* --- the parser on its own --------------------------------------------- */ + + for (const [name, answer] of READABLE_ANSWERS) { + await check(`reads ${name}`, () => { + const { parsed } = parseAnalysisResponse(answer); + assert.deepStrictEqual(parsed, expectedFor(name)); + }); + } + + await check('normalized is the JSON that was parsed', () => { + const { normalized } = parseAnalysisResponse( + 'xSure:\n' + JSON_ANSWER + '\nDone.' + ); + assert.deepStrictEqual(JSON.parse(normalized), RESULT); + assert.ok(normalized.startsWith('{'), normalized); + assert.ok(normalized.endsWith('}'), normalized); + }); + + await check('an answer that is already an object passes through', () => { + const { parsed, normalized } = parseAnalysisResponse(RESULT); + assert.strictEqual(parsed, RESULT); + assert.deepStrictEqual(JSON.parse(normalized), RESULT); + }); + + await check('the first top-level object wins, not one nested in it', () => { + // The outer object is beyond repair; its custom_fields entry would parse + // on its own, but it is not the answer and must not be taken for one. + const answer = '{"title": "A" "custom_fields": {"0": {"value": "1"}}}'; + assert.throws(() => parseAnalysisResponse(answer), /Invalid JSON/); + }); + + await check( + 'an unreadable object does not hide a readable one after it', + () => { + const { parsed } = parseAnalysisResponse( + 'Example: {broken "x"}\nAnswer: ' + JSON_ANSWER + ); + assert.deepStrictEqual(parsed, RESULT); + } + ); + + /* Finding 7c: an unreadable answer that reads like a refusal is reported + as insufficient content instead. The answers above must stay plain + failures, or the cases further down would test the refusal path. */ + for (const [name, answer] of UNREADABLE_ANSWERS) { + await check(`${name} is not a refusal`, () => { + assert.strictEqual(isInsufficientContentAnswer(answer), false); + }); + } + + await check('a refusal is recognised, case and wrapping aside', () => { + for (const answer of [ + "I'm sorry, I cannot help with this document.", + 'I CANNOT analyse this.', + 'too shortThe content is insufficient.', + '```\nInsufficient text.\n```', + ]) { + assert.strictEqual(isInsufficientContentAnswer(answer), true, answer); + } + for (const answer of ['', null, undefined, 'A scanned letter.']) { + assert.strictEqual(isInsufficientContentAnswer(answer), false, answer); + } + }); + + for (const [name, answer] of UNREADABLE_ANSWERS) { + await check(`fails on ${name} with the OCR fallback phrase`, () => { + assert.throws( + () => parseAnalysisResponse(answer), + (error) => + /^Invalid JSON response from API/.test(error.message) && + shouldQueueForOcrOnAiError(error.message) + ); + }); + } + + await check('fails on an empty answer with the OCR fallback phrase', () => { + for (const answer of ['', ' ', null, undefined]) { + assert.throws( + () => parseAnalysisResponse(answer), + (error) => /^Invalid JSON response from API/.test(error.message) + ); + } + }); + + await check('stripResponseWrapping drops reasoning and fences only', () => { + assert.strictEqual( + stripResponseWrapping('a\nb\n```json\n{"a":1}\n```\n'), + '{"a":1}' + ); + assert.strictEqual(stripResponseWrapping("I'm sorry."), "I'm sorry."); + }); + + /* --- every provider reads the same answers ----------------------------- */ + + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile(getThumbnailCachePath(DOCUMENT_ID), 'thumbnail'); + + for (const [label, service, answerWith] of PROVIDERS) { + for (const [name, answer] of READABLE_ANSWERS) { + await check(`${label}: reads ${name}`, async () => { + const analysis = await analyze(service, answerWith, answer); + assert.strictEqual(analysis.error, undefined, analysis.error); + sameDocument(label, analysis.document, expectedFor(name)); + }); + } + + for (const [name, answer] of UNREADABLE_ANSWERS) { + await check(`${label}: ${name} reaches the OCR fallback`, async () => { + const analysis = await analyze(service, answerWith, answer); + assert.match(analysis.error || '', /Invalid JSON response from API/); + assert.ok(shouldQueueForOcrOnAiError(analysis.error), analysis.error); + }); + } + } + + /* --- Finding 7k: an answer with no usable metadata -------------------- */ + + /* Decided 2026-10-11: all four providers fail the answer the same way. + Ollama used to log and return the defaults as a *success* (no tags, no + correspondent — the document landed in Paperless all null). The check + raises on the parsed answer, before withDocumentDefaults() would fill + in the missing fields and mask it. */ + for (const [label, service, answerWith] of PROVIDERS) { + await check( + `${label}: an answer without tags or correspondent fails`, + async () => { + const analysis = await analyze( + service, + answerWith, + JSON.stringify({ note: 'nothing assignable here' }) + ); + assert.match( + analysis.error || '', + /could not determine assignable metadata/i + ); + assert.ok( + shouldQueueForOcrOnAiError(analysis.error), + `the failure routes to OCR: ${analysis.error}` + ); + assert.deepStrictEqual(analysis.document, { + tags: [], + correspondent: null, + }); + assert.strictEqual(analysis.metrics, null); + } + ); + } + + /* --- field defaults: every success comes back default-shaped ---------- */ + + /* Decided 2026-10-11: withDocumentDefaults() shapes every provider's + success to the seven document fields, present values surviving and + the invented ones dropped. Before, only Ollama did this; the three + chat providers passed the parsed answer on untouched. */ + for (const [label, service, answerWith] of PROVIDERS) { + await check( + `${label}: a partial answer comes back default-shaped`, + async () => { + const analysis = await analyze( + service, + answerWith, + JSON.stringify(PARTIAL_ANSWER) + ); + assert.strictEqual(analysis.error, undefined, analysis.error); + sameDocument(label, analysis.document, expectedFor('partial answer')); + } + ); + } + + await check('Ollama: an answer object is used as it is', async () => { + const analysis = await analyze(ollamaService, ollamaClient, RESULT); + assert.strictEqual(analysis.error, undefined, analysis.error); + sameDocument('Ollama', analysis.document, RESULT); + }); + + await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true }); + + if (failed > 0) { + console.error(`\n${failed} response parser case(s) failed`); + process.exit(1); + } + console.log('\nAll response parser cases passed'); +})(); diff --git a/tests/test-llm-token-budget.js b/tests/test-llm-token-budget.js new file mode 100644 index 00000000..ab9e5117 --- /dev/null +++ b/tests/test-llm-token-budget.js @@ -0,0 +1,407 @@ +/** + * The two token budgets, each written once. + * + * Step 4 of the LLM service refactoring (docs/llm-service-refactoring.md). + * openai, azure and custom carried the same reserve-then-truncate block: + * tiktoken counts for the prompt, the configured answer reserved, the + * document cut to what is left. It lives in + * services/llm/tokenBudget/tiktokenBudget.js now. Ollama's arithmetic is a + * different model on purpose (Finding 5) — 2 chars per token, num_ctx sized + * as min(prompt + answer, TOKEN_LIMIT), the document fitted by characters — + * and lives in services/llm/tokenBudget/charBudget.js. Both expose the same + * reserve()/fit() pair, so the analyzer of step 6 can take either. + * + * The first half pins each budget on its own, the second half checks that + * every provider spends the budget it was given. + */ + +const assert = require('assert'); +const fs = require('fs').promises; + +process.env.AI_PROVIDER = 'openai'; +process.env.OPENAI_API_KEY = 'test-key'; +process.env.OPENAI_MODEL = 'gpt-4'; +process.env.AZURE_API_KEY = 'test-key'; +process.env.AZURE_ENDPOINT = 'https://example.invalid'; +process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment'; +process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1'; +process.env.CUSTOM_API_KEY = 'test-key'; +process.env.CUSTOM_MODEL = 'test-model'; +process.env.SYSTEM_PROMPT = 'Analyse the document.'; +process.env.USE_PROMPT_TAGS = 'no'; +delete process.env.CUSTOM_FIELDS; +delete process.env.CONTENT_MAX_LENGTH; + +const config = require('../config/config'); +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('../services/thumbnailCachePaths'); +const { + calculateTotalPromptTokens, + truncateToTokenLimit, +} = require('../services/serviceUtils'); +const tiktokenBudget = require('../services/llm/tokenBudget/tiktokenBudget'); +const charBudget = require('../services/llm/tokenBudget/charBudget'); +const openaiService = require('../services/openaiService'); +const azureService = require('../services/azureService'); +const customService = require('../services/customService'); +const ollamaService = require('../services/ollamaService'); + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +/** Own id so the seeded thumbnail cannot collide with a real cached document. */ +const DOCUMENT_ID = 'testLlmTokenBudget'; + +const CHAT_LIMIT_MESSAGE = + 'Token limit exceeded: prompt too large for available token limit'; + +const ANSWER = JSON.stringify({ + title: 'Telekom invoice July', + correspondent: 'Telekom Deutschland GmbH', + tags: ['invoice'], + document_type: 'Rechnung', + document_date: '2026-09-03', + language: 'de', +}); + +/** The success document every provider shapes to the seven fields. */ +const DEFAULT_DOCUMENT = { + tags: ['invoice'], + correspondent: 'Telekom Deutschland GmbH', + title: 'Telekom invoice July', + document_date: '2026-09-03', + document_type: 'Rechnung', + language: 'de', + custom_fields: null, +}; + +/** Field defaults for all four (step 6): the success comes back default-shaped. */ +function assertDefaultDocument(analysis) { + assert.deepStrictEqual( + analysis.document, + DEFAULT_DOCUMENT, + 'the success document carries the seven default-shaped fields' + ); +} + +/** Long enough that every limit below has to cut it. */ +const LONG_CONTENT = 'Invoice from Telekom Deutschland GmbH, July. '.repeat( + 400 +); + +function limits(tokenLimit, responseTokens) { + Object.assign(config, { tokenLimit, responseTokens }); +} + +/* --- the providers, each with a client that records what it was sent ----- */ + +function recordingChatClient(service, requests) { + service.client = { + chat: { + completions: { + create: async (request) => { + requests.push(request); + return { + choices: [{ message: { content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 }, + }; + }, + }, + }, + models: { list: async () => ({ data: [] }) }, + }; +} + +function recordingOllamaClient(requests) { + ollamaService.client = { + post: async (url, body) => { + requests.push(body); + return { + data: { + response: ANSWER, + done_reason: 'stop', + eval_count: 3, + prompt_eval_count: 9, + }, + }; + }, + get: async () => ({ status: 200, data: { models: [] } }), + }; +} + +const CHAT_PROVIDERS = [ + ['OpenAI', openaiService, () => process.env.OPENAI_MODEL], + ['Azure', azureService, () => process.env.AZURE_DEPLOYMENT_NAME], + ['Custom', customService, () => config.custom.model], +]; + +async function analyzeRecorded(service, content, record) { + const requests = []; + record(requests); + const analysis = await service.analyzeDocument( + content, + [], + [], + [], + DOCUMENT_ID + ); + return { analysis, requests }; +} + +(async () => { + console.log('\n=== LLM token budget ==='); + + /* --- tiktokenBudget ----------------------------------------------------- */ + + await check( + 'tiktoken: reserve counts the prompt and the answer', + async () => { + limits(4000, 500); + const prompt = 'Analyse the document and answer in JSON.'; + const budget = await tiktokenBudget.reserve( + prompt, + ['extra tags'], + 'gpt-4' + ); + const counted = await calculateTotalPromptTokens( + prompt, + ['extra tags'], + 'gpt-4' + ); + assert.deepStrictEqual(budget, { + totalPromptTokens: counted, + reservedTokens: counted + 500, + availableTokens: 4000 - counted - 500, + }); + } + ); + + await check('tiktoken: no room left is "Token limit exceeded"', async () => { + limits(100, 100); + await assert.rejects( + () => tiktokenBudget.reserve('Analyse.', [], 'gpt-4'), + (error) => error.message === CHAT_LIMIT_MESSAGE + ); + }); + + await check('tiktoken: a broken limit is refused, not spent', async () => { + for (const [tokenLimit, responseTokens] of [ + [Number.NaN, 100], + [4000, Number.NaN], + [undefined, 100], + ]) { + limits(tokenLimit, responseTokens); + await assert.rejects( + () => tiktokenBudget.reserve('Analyse.', [], 'gpt-4'), + (error) => error.message === CHAT_LIMIT_MESSAGE + ); + } + }); + + await check('tiktoken: fit is truncateToTokenLimit', async () => { + assert.strictEqual( + await tiktokenBudget.fit(LONG_CONTENT, 50, 'gpt-4'), + await truncateToTokenLimit(LONG_CONTENT, 50, 'gpt-4') + ); + assert.strictEqual(await tiktokenBudget.fit('short', 50, 'gpt-4'), 'short'); + }); + + /* --- charBudget (Finding 5) --------------------------------------------- */ + + await check('char: two characters make one token, rounded up', () => { + assert.strictEqual(charBudget.estimateTokens(''), 0); + assert.strictEqual(charBudget.estimateTokens('a'), 1); + assert.strictEqual(charBudget.estimateTokens('abcd'), 2); + assert.strictEqual(charBudget.estimateTokens('abcde'), 3); + }); + + await check('char: num_ctx is min(prompt + answer, TOKEN_LIMIT)', () => { + limits(8192, 1000); + assert.strictEqual(charBudget.numCtx(2000, 1000), 3000); + assert.strictEqual(charBudget.numCtx(7500, 1000), 8192); + }); + + await check( + 'char: reserve leaves TOKEN_LIMIT - answer - prompt/2', + async () => { + limits(8192, 1000); + const budget = await charBudget.reserve('x'.repeat(301)); + assert.deepStrictEqual(budget, { + totalPromptTokens: 151, + reservedTokens: 1151, + availableTokens: 8192 - 1151, + }); + } + ); + + await check('char: no room left is "Token limit exceeded"', async () => { + limits(1000, 900); + await assert.rejects( + () => charBudget.reserve('x'.repeat(200)), + /^Error: Token limit exceeded: the prompt alone does not fit the configured TOKEN_LIMIT/ + ); + }); + + await check('char: a broken limit leaves the document whole', async () => { + limits(Number.NaN, 1000); + const budget = await charBudget.reserve('x'.repeat(200)); + assert.strictEqual(budget.availableTokens, Infinity); + assert.strictEqual( + await charBudget.fit(LONG_CONTENT, budget.availableTokens), + LONG_CONTENT + ); + }); + + await check( + 'char: fit cuts to two characters per available token', + async () => { + assert.strictEqual( + await charBudget.fit(LONG_CONTENT, 100), + LONG_CONTENT.slice(0, 200) + ); + assert.strictEqual(await charBudget.fit('short', 100), 'short'); + } + ); + + await check('char: fit measures a non-string document as JSON', async () => { + const doc = { text: 'x'.repeat(500) }; + const cut = await charBudget.fit(doc, 50); + assert.strictEqual(cut, JSON.stringify(doc).slice(0, 100)); + // One that fits comes back as it was handed in, not as its JSON. + const small = { a: 1 }; + assert.strictEqual(await charBudget.fit(small, 50), small); + }); + + /* --- every provider spends the budget it was given ---------------------- */ + + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile(getThumbnailCachePath(DOCUMENT_ID), 'thumbnail'); + + for (const [label, service, model] of CHAT_PROVIDERS) { + const record = (requests) => recordingChatClient(service, requests); + + await check( + `${label}: the document is cut to the tiktoken budget`, + async () => { + limits(600, 200); + const { analysis, requests } = await analyzeRecorded( + service, + LONG_CONTENT, + record + ); + assert.strictEqual(analysis.error, undefined, analysis.error); + assert.strictEqual(requests.length, 1); + const system = requests[0].messages.find((m) => m.role === 'system'); + const user = requests[0].messages.find((m) => m.role === 'user'); + const budget = await tiktokenBudget.reserve( + system.content, + [], + model() + ); + assert.strictEqual( + user.content, + await tiktokenBudget.fit( + LONG_CONTENT, + budget.availableTokens, + model() + ) + ); + assert.strictEqual(analysis.truncated, true); + assertDefaultDocument(analysis); + } + ); + + await check(`${label}: a document that fits is sent whole`, async () => { + limits(8000, 200); + const { analysis, requests } = await analyzeRecorded( + service, + 'Short invoice.', + record + ); + const user = requests[0].messages.find((m) => m.role === 'user'); + assert.strictEqual(user.content, 'Short invoice.'); + assert.strictEqual(analysis.truncated, false); + assertDefaultDocument(analysis); + }); + + await check(`${label}: no room left fails before any request`, async () => { + limits(100, 100); + const { analysis, requests } = await analyzeRecorded( + service, + LONG_CONTENT, + record + ); + assert.strictEqual(analysis.error, CHAT_LIMIT_MESSAGE); + assert.strictEqual(requests.length, 0); + }); + } + + await check('Ollama: the document is cut to the char budget', async () => { + limits(2000, 500); + const { analysis, requests } = await analyzeRecorded( + ollamaService, + LONG_CONTENT, + recordingOllamaClient + ); + assert.strictEqual(analysis.error, undefined, analysis.error); + const prompt = requests[0].prompt; + // The prompt ends with the fitted document as a JSON string literal. + const start = prompt.indexOf('"Invoice from'); + const sentContent = JSON.parse(prompt.slice(start).trim()); + const prefix = prompt.slice(0, start - '\n '.length); + const budget = await charBudget.reserve(prefix + '\n '); + assert.ok(sentContent.length < LONG_CONTENT.length, 'the document was cut'); + assert.ok( + sentContent.length <= budget.availableTokens * 2, + `${sentContent.length} chars sent for ${budget.availableTokens} tokens` + ); + assert.strictEqual(analysis.truncated, true); + }); + + await check( + 'Ollama: num_ctx is min(prompt/2 + answer, TOKEN_LIMIT)', + async () => { + limits(2000, 500); + const { requests } = await analyzeRecorded( + ollamaService, + 'Short invoice.', + recordingOllamaClient + ); + const { prompt, options } = requests[0]; + assert.strictEqual( + options.num_ctx, + Math.min(Math.ceil(prompt.length / 2) + 500, 2000) + ); + assert.strictEqual(options.num_predict, 500); + } + ); + + await check('Ollama: no room left fails before any request', async () => { + limits(600, 500); + const { analysis, requests } = await analyzeRecorded( + ollamaService, + LONG_CONTENT, + recordingOllamaClient + ); + assert.match(analysis.error || '', /^Token limit exceeded/); + assert.strictEqual(requests.length, 0); + }); + + await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true }); + + if (failed > 0) { + console.error(`\n${failed} token budget case(s) failed`); + process.exit(1); + } + console.log('\nAll token budget cases passed'); +})(); diff --git a/tests/test-llm-transport-request-mapping.js b/tests/test-llm-transport-request-mapping.js new file mode 100644 index 00000000..f5d9671c --- /dev/null +++ b/tests/test-llm-transport-request-mapping.js @@ -0,0 +1,908 @@ +/** + * The transports: what each API family is sent, and what comes back. + * + * Step 5 of the LLM service refactoring (docs/llm-service-refactoring.md). + * A transport owns exactly one thing — how to talk to one API family — and + * never shapes an analysis result: + * + * services/llm/transports/chatCompletionsTransport.js openai, azure, custom + * services/llm/transports/ollamaGenerateTransport.js Ollama /api/generate + * + * The first half pins each transport on its own with a fake client. The + * second half checks that every provider's analysis request goes through its + * transport and is mapped the way it was before: OpenAI omits the + * temperature for o3-mini, Azure sends the configured analysis + * temperature (Finding 7a, step 6), Ollama sends format, num_ctx, + * num_predict and think:false unless OLLAMA_THINK is set. + */ + +const assert = require('assert'); +const fs = require('fs').promises; + +process.env.AI_PROVIDER = 'openai'; +process.env.OPENAI_API_KEY = 'test-key'; +process.env.OPENAI_MODEL = 'gpt-4'; +process.env.AZURE_API_KEY = 'test-key'; +process.env.AZURE_ENDPOINT = 'https://example.invalid'; +process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment'; +process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1'; +process.env.CUSTOM_API_KEY = 'test-key'; +process.env.CUSTOM_MODEL = 'test-model'; +process.env.SYSTEM_PROMPT = 'Analyse the document.'; +process.env.USE_PROMPT_TAGS = 'no'; +delete process.env.CUSTOM_FIELDS; +delete process.env.CONTENT_MAX_LENGTH; +delete process.env.OLLAMA_API_KEY; + +const config = require('../config/config'); +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('../services/thumbnailCachePaths'); +const { + ChatCompletionsTransport, +} = require('../services/llm/transports/chatCompletionsTransport'); +const { + OllamaGenerateTransport, +} = require('../services/llm/transports/ollamaGenerateTransport'); +const openaiService = require('../services/openaiService'); +const azureService = require('../services/azureService'); +const customService = require('../services/customService'); +const ollamaService = require('../services/ollamaService'); + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +/** Own id so the seeded thumbnail cannot collide with a real cached document. */ +const DOCUMENT_ID = 'testLlmTransportMapping'; + +const ANSWER = JSON.stringify({ + title: 'Telekom invoice July', + correspondent: 'Telekom Deutschland GmbH', + tags: ['invoice'], + document_type: 'Rechnung', + document_date: '2026-09-03', + language: 'de', +}); + +function completion(content = ANSWER, finishReason = 'stop') { + return { + choices: [{ message: { content }, finish_reason: finishReason }], + usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 }, + }; +} + +function fakeChatClient(requests, respond = async () => completion()) { + return { + chat: { + completions: { + create: async (request) => { + requests.push(request); + return respond(request); + }, + }, + }, + models: { list: async () => ({ data: [] }) }, + }; +} + +function ollamaData(overrides = {}) { + return { + response: ANSWER, + done_reason: 'stop', + eval_count: 3, + prompt_eval_count: 9, + ...overrides, + }; +} + +/** An axios-shaped client; `answers` are returned or thrown in turn. */ +function fakeOllamaClient(answers, posts) { + let next = 0; + return { + post: async (url, body, requestConfig) => { + posts.push({ url, body, requestConfig }); + const answer = answers[Math.min(next, answers.length - 1)]; + next += 1; + if (answer instanceof Error) throw answer; + return answer; + }, + get: async () => ({ status: 200, data: { models: [] } }), + }; +} + +function axiosFailure(status, data) { + const error = new Error( + status ? `Request failed with status code ${status}` : 'socket hang up' + ); + error.code = status ? 'ERR_BAD_RESPONSE' : 'ECONNRESET'; + if (status) error.response = { status, data }; + return error; +} + +const ANALYSIS_REQUEST = { + systemPrompt: 'You analyse documents.', + userContent: 'Invoice from Telekom.', + model: 'gpt-4', + temperature: 0.2, +}; + +(async () => { + console.log('\n=== LLM transport request mapping ==='); + + /* --- ChatCompletionsTransport -------------------------------------------- */ + + await check('chat: the client is created once, on first use', () => { + let created = 0; + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => { + created += 1; + return fakeChatClient([]); + }, + }); + assert.strictEqual(transport.isReady(), false); + transport.initialize(); + transport.initialize(); + assert.strictEqual(created, 1); + assert.strictEqual(transport.isReady(), true); + }); + + await check('chat: a factory without credentials leaves it not ready', () => { + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => null, + }); + transport.initialize(); + assert.strictEqual(transport.isReady(), false); + assert.strictEqual(transport.client, null); + }); + + await check('chat: sends a system and a user message', async () => { + const requests = []; + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => fakeChatClient(requests), + }); + transport.initialize(); + await transport.complete(ANALYSIS_REQUEST); + assert.deepStrictEqual(requests, [ + { + model: 'gpt-4', + messages: [ + { role: 'system', content: 'You analyse documents.' }, + { role: 'user', content: 'Invoice from Telekom.' }, + ], + temperature: 0.2, + }, + ]); + }); + + await check('chat: an undefined temperature is not sent at all', async () => { + const requests = []; + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => fakeChatClient(requests), + }); + transport.initialize(); + await transport.complete({ ...ANALYSIS_REQUEST, temperature: undefined }); + assert.ok(!('temperature' in requests[0]), JSON.stringify(requests[0])); + }); + + await check('chat: returns the text and the usage as metrics', async () => { + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => fakeChatClient([]), + }); + transport.initialize(); + const result = await transport.complete(ANALYSIS_REQUEST); + assert.deepStrictEqual(result, { + payload: ANSWER, + metrics: { promptTokens: 9, completionTokens: 3, totalTokens: 12 }, + }); + }); + + await check('chat: a cut-off answer is ai_response_truncated', async () => { + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => + fakeChatClient([], async () => completion('{"tags": [', 'length')), + }); + transport.initialize(); + await assert.rejects( + () => transport.complete(ANALYSIS_REQUEST), + (error) => error.code === 'ai_response_truncated' + ); + }); + + await check( + 'chat: no content is "Invalid API response structure"', + async () => { + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => fakeChatClient([], async () => ({ choices: [] })), + }); + transport.initialize(); + await assert.rejects( + () => transport.complete(ANALYSIS_REQUEST), + (error) => error.message === 'Invalid API response structure' + ); + } + ); + + await check( + 'chat: a provider may describe the missing content itself', + async () => { + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => fakeChatClient([], async () => ({ choices: [] })), + describeMissingContent: (response) => + `Invalid API response structure: ${response.choices.length} choices.`, + }); + transport.initialize(); + await assert.rejects( + () => transport.complete(ANALYSIS_REQUEST), + (error) => + error.message === 'Invalid API response structure: 0 choices.' + ); + } + ); + + /* --- OllamaGenerateTransport --------------------------------------------- */ + + const ollamaTransport = (answers, posts, options = {}) => + new OllamaGenerateTransport({ + client: fakeOllamaClient(answers, posts), + apiUrl: 'http://ollama.invalid:11434', + model: 'llama3', + retryDelayMs: 1, + ...options, + }); + + const OLLAMA_REQUEST = { + systemPrompt: 'system', + userContent: 'prompt', + temperature: 0.2, + maxResponseTokens: 700, + numCtx: 4096, + schema: { type: 'object' }, + }; + + await check('ollama: maps the request onto /api/generate', async () => { + const posts = []; + const previousThink = config.ollama.think; + config.ollama.think = false; + try { + await ollamaTransport([{ data: ollamaData() }], posts).complete( + OLLAMA_REQUEST + ); + } finally { + config.ollama.think = previousThink; + } + assert.strictEqual(posts.length, 1); + assert.strictEqual( + posts[0].url, + 'http://ollama.invalid:11434/api/generate' + ); + assert.deepStrictEqual(posts[0].body, { + model: 'llama3', + prompt: 'prompt', + system: 'system', + stream: false, + format: { type: 'object' }, + options: { + temperature: 0.2, + top_p: 0.9, + repeat_penalty: 1.1, + top_k: 7, + num_predict: 700, + num_ctx: 4096, + }, + think: false, + }); + assert.deepStrictEqual(posts[0].requestConfig, { + headers: { 'Content-Type': 'application/json' }, + }); + }); + + await check('ollama: OLLAMA_THINK=true sends no think field', async () => { + const posts = []; + const previousThink = config.ollama.think; + config.ollama.think = true; + try { + await ollamaTransport([{ data: ollamaData() }], posts).complete( + OLLAMA_REQUEST + ); + } finally { + config.ollama.think = previousThink; + } + assert.ok(!('think' in posts[0].body), JSON.stringify(posts[0].body)); + }); + + await check('ollama: an API key becomes a bearer header', async () => { + const posts = []; + process.env.OLLAMA_API_KEY = 'secret'; + try { + await ollamaTransport([{ data: ollamaData() }], posts).complete( + OLLAMA_REQUEST + ); + } finally { + delete process.env.OLLAMA_API_KEY; + } + assert.strictEqual( + posts[0].requestConfig.headers.Authorization, + 'Bearer secret' + ); + }); + + await check( + 'ollama: returns the answer, its metrics and the raw data', + async () => { + const data = ollamaData(); + const result = await ollamaTransport([{ data }], []).complete( + OLLAMA_REQUEST + ); + assert.strictEqual(result.payload, ANSWER); + assert.deepStrictEqual(result.metrics, { + promptTokens: 9, + completionTokens: 3, + totalTokens: 12, + }); + assert.strictEqual(result.raw, data); + } + ); + + await check( + 'ollama: a 5xx is tried once more, and the second answer wins', + async () => { + const posts = []; + const result = await ollamaTransport( + [axiosFailure(500, { error: 'EOF' }), { data: ollamaData() }], + posts + ).complete(OLLAMA_REQUEST); + assert.strictEqual(posts.length, 2); + assert.strictEqual(result.payload, ANSWER); + } + ); + + await check( + 'ollama: a 4xx is not retried and names the upstream cause', + async () => { + const posts = []; + await assert.rejects( + () => + ollamaTransport( + [axiosFailure(404, { error: 'model "llama3" not found' })], + posts + ).complete(OLLAMA_REQUEST), + (error) => + error.message === + 'Ollama request failed (HTTP 404, attempt 1): model "llama3" not found' && + error.code === 'ERR_BAD_RESPONSE' + ); + assert.strictEqual(posts.length, 1); + } + ); + + await check( + 'ollama: no response at all is retried, then reported', + async () => { + const posts = []; + await assert.rejects( + () => + ollamaTransport([axiosFailure(null)], posts).complete(OLLAMA_REQUEST), + (error) => + error.message === + 'Ollama request failed (HTTP no response, attempt 2)' && + error.code === 'ECONNRESET' + ); + assert.strictEqual(posts.length, 2); + } + ); + + await check( + 'ollama: done_reason "length" is ai_response_truncated', + async () => { + await assert.rejects( + () => + ollamaTransport( + [{ data: ollamaData({ done_reason: 'length', eval_count: 700 }) }], + [] + ).complete(OLLAMA_REQUEST), + (error) => + error.code === 'ai_response_truncated' && + /Raise Response Tokens \(RESPONSE_TOKENS\) above 700/.test( + error.message + ) + ); + } + ); + + await check( + 'ollama: an empty HTTP body is "Invalid response from Ollama API"', + async () => { + await assert.rejects( + () => ollamaTransport([{ data: null }], []).complete(OLLAMA_REQUEST), + (error) => error.message === 'Invalid response from Ollama API' + ); + } + ); + + /* --- every provider's analysis goes through its transport ---------------- */ + + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile(getThumbnailCachePath(DOCUMENT_ID), 'thumbnail'); + Object.assign(config, { tokenLimit: 128000, responseTokens: 1000 }); + + const analyse = (service) => + service.analyzeDocument('Invoice from Telekom.', [], [], [], DOCUMENT_ID); + + for (const [label, service, transportLabel] of [ + ['OpenAI', openaiService, 'OpenAI'], + ['Azure', azureService, 'AzureOpenAI'], + ['Custom', customService, 'Custom OpenAI'], + ]) { + await check(`${label}: owns a chat completions transport`, () => { + assert.ok(service.transport instanceof ChatCompletionsTransport); + assert.strictEqual(service.transport.label, transportLabel); + }); + + await check(`${label}: the client it is given is the transport's`, () => { + const client = fakeChatClient([]); + service.client = client; + assert.strictEqual(service.transport.client, client); + assert.strictEqual(service.client, client); + }); + } + + await check('OpenAI: analysis sends AI_TEMPERATURE_ANALYSIS', async () => { + const requests = []; + openaiService.client = fakeChatClient(requests); + const analysis = await analyse(openaiService); + assert.strictEqual(analysis.error, undefined, analysis.error); + assert.strictEqual(requests[0].model, 'gpt-4'); + assert.strictEqual(requests[0].temperature, config.aiTemperatureAnalysis); + assert.deepStrictEqual( + requests[0].messages.map((m) => m.role), + ['system', 'user'] + ); + }); + + await check( + 'OpenAI: analysis with o3-mini sends no temperature', + async () => { + const requests = []; + openaiService.client = fakeChatClient(requests); + process.env.OPENAI_MODEL = 'o3-mini'; + try { + await analyse(openaiService); + } finally { + process.env.OPENAI_MODEL = 'gpt-4'; + } + assert.strictEqual(requests[0].model, 'o3-mini'); + assert.ok(!('temperature' in requests[0]), JSON.stringify(requests[0])); + } + ); + + // Finding 7h: the factory hands out openaiService for the openai provider + // (and as its default) only, so OpenAI builds no client for Ollama or + // Custom — those have composition roots of their own. + await check( + 'OpenAI: builds an OpenAI client for the openai provider only (7h)', + () => { + const previousProvider = config.aiProvider; + const build = (provider) => { + config.aiProvider = provider; + openaiService.client = null; + openaiService.transport.initialize(); + return openaiService.client; + }; + try { + assert.match(build('openai').baseURL, /api\.openai\.com/); + for (const provider of ['ollama', 'custom', 'azure', 'unknown']) { + assert.strictEqual(build(provider), null, provider); + } + } finally { + config.aiProvider = previousProvider; + openaiService.client = null; + } + } + ); + + await check( + 'Azure: analysis sends AI_TEMPERATURE_ANALYSIS (7a)', + async () => { + const requests = []; + azureService.client = fakeChatClient(requests); + await analyse(azureService); + assert.strictEqual(requests[0].model, 'test-deployment'); + assert.strictEqual(requests[0].temperature, config.aiTemperatureAnalysis); + } + ); + + await check('Custom: analysis sends AI_TEMPERATURE_ANALYSIS', async () => { + const requests = []; + customService.client = fakeChatClient(requests); + await analyse(customService); + assert.strictEqual(requests[0].model, config.custom.model); + assert.strictEqual(requests[0].temperature, config.aiTemperatureAnalysis); + }); + + await check( + 'Custom: keeps its detailed missing-content message', + async () => { + customService.client = fakeChatClient([], async () => ({ choices: [] })); + const analysis = await analyse(customService); + assert.strictEqual( + analysis.error, + 'Invalid API response structure: missing choices[0].' + ); + } + ); + + await check('Ollama: owns a generate transport', () => { + assert.ok(ollamaService.transport instanceof OllamaGenerateTransport); + }); + + await check("Ollama: client, apiUrl and model are the transport's", () => { + const client = fakeOllamaClient([{ data: ollamaData() }], []); + const previous = { + apiUrl: ollamaService.apiUrl, + model: ollamaService.model, + }; + ollamaService.client = client; + ollamaService.apiUrl = 'http://elsewhere.invalid'; + ollamaService.model = 'other-model'; + try { + assert.strictEqual(ollamaService.transport.client, client); + assert.strictEqual( + ollamaService.transport.apiUrl, + 'http://elsewhere.invalid' + ); + assert.strictEqual(ollamaService.transport.model, 'other-model'); + } finally { + Object.assign(ollamaService, previous); + } + }); + + await check( + 'Ollama: analysis sends format, num_ctx and num_predict', + async () => { + const posts = []; + ollamaService.client = fakeOllamaClient([{ data: ollamaData() }], posts); + const analysis = await analyse(ollamaService); + assert.strictEqual(analysis.error, undefined, analysis.error); + const { body } = posts[0]; + assert.deepStrictEqual(body.format, ollamaService.documentAnalysisSchema); + assert.strictEqual(body.model, ollamaService.model); + assert.strictEqual(body.options.num_predict, 1000); + assert.strictEqual( + body.options.num_ctx, + Math.min(Math.ceil(body.prompt.length / 2) + 1000, 128000) + ); + assert.strictEqual( + body.options.temperature, + config.aiTemperatureAnalysis + ); + } + ); + + /* --- generateText(): the shared half in the transport (step 5b) ---------- */ + + const generationTransport = (respond, generation = {}) => { + const requests = []; + const calls = []; + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => ({ + chat: { + completions: { + create: async (request, requestOptions) => { + requests.push(request); + calls.push(requestOptions); + return respond(request); + }, + }, + }, + }), + generation: { + notReadyMessage: 'Fake client not initialized - missing API key', + remedy: 'Raise Response Tokens (RESPONSE_TOKENS).', + buildRequest: async (prompt) => ({ + model: 'fake-model', + messages: [{ role: 'user', content: prompt }], + temperature: 0.7, + }), + ...generation, + }, + }); + return { transport, requests, calls }; + }; + + await check('chat generate: sends what the provider built', async () => { + const { transport, requests, calls } = generationTransport(async () => + completion('hmThe answer.') + ); + const text = await transport.generate('hello'); + assert.strictEqual(text, 'The answer.'); + assert.deepStrictEqual(requests, [ + { + model: 'fake-model', + messages: [{ role: 'user', content: 'hello' }], + temperature: 0.7, + }, + ]); + assert.deepStrictEqual(calls, [undefined], 'no request options'); + assert.deepStrictEqual(transport.lastGenerateTextUsage, { + promptTokens: 9, + completionTokens: 3, + totalTokens: 12, + reasoningTokens: null, + }); + }); + + await check( + 'chat generate: a signal rides in the request options', + async () => { + const { transport, calls } = generationTransport(async () => + completion('ok') + ); + const controller = new AbortController(); + await transport.generate('hello', { signal: controller.signal }); + assert.deepStrictEqual(calls, [{ signal: controller.signal }]); + } + ); + + await check('chat generate: not ready names the provider', async () => { + const transport = new ChatCompletionsTransport({ + label: 'Fake', + createClient: () => null, + generation: { + notReadyMessage: 'Fake client not initialized - missing API key', + buildRequest: async () => ({}), + }, + }); + await assert.rejects( + () => transport.generate('hello'), + (error) => + error.message === 'Fake client not initialized - missing API key' + ); + }); + + await check( + 'chat generate: no text is "Invalid API response structure"', + async () => { + const { transport } = generationTransport(async () => completion('')); + await assert.rejects( + () => transport.generate('hello'), + (error) => error.message === 'Invalid API response structure' + ); + } + ); + + await check( + 'chat generate: a provider may describe the empty answer', + async () => { + const { transport } = generationTransport(async () => completion(''), { + describeEmptyText: () => 'Invalid API response structure: empty.', + }); + await assert.rejects( + () => transport.generate('hello'), + (error) => error.message === 'Invalid API response structure: empty.' + ); + } + ); + + await check('chat generate: a cut-off answer keeps its prefix', async () => { + const { transport } = generationTransport(async () => + completion('Half an ans', 'length') + ); + await assert.rejects( + () => transport.generate('hello'), + (error) => + error.code === 'ai_response_truncated' && + error.partialText === 'Half an ans' && + /Raise Response Tokens/.test(error.message) + ); + }); + + await check( + 'ollama generate: maps the request onto /api/generate', + async () => { + const posts = []; + const previousThink = config.ollama.think; + config.ollama.think = false; + let text; + try { + text = await ollamaTransport( + [{ data: ollamaData({ response: 'x Hi there. ' }) }], + posts + ).generate('hello', {}, { responseTokens: 300, numCtx: 2048 }); + } finally { + config.ollama.think = previousThink; + } + assert.strictEqual(text, 'Hi there.'); + assert.deepStrictEqual(posts[0].body, { + model: 'llama3', + prompt: 'hello', + system: + "You are a helpful assistant. Generate a clear, concise, and informative response to the user's question or request.", + stream: false, + options: { + temperature: config.aiTemperatureGeneration, + top_p: 0.9, + num_predict: 300, + num_ctx: 2048, + }, + think: false, + }); + } + ); + + await check( + 'ollama generate: options override model, system, temperature, think', + async () => { + const posts = []; + await ollamaTransport( + [{ data: ollamaData({ response: 'ok' }) }], + posts + ).generate( + 'hello', + { + model: 'judge', + systemPrompt: 'Be brief.', + temperature: 0, + reasoning: true, + }, + { responseTokens: 300, numCtx: 2048 } + ); + const { body } = posts[0]; + assert.strictEqual(body.model, 'judge'); + assert.strictEqual(body.system, 'Be brief.'); + assert.strictEqual(body.options.temperature, 0); + assert.strictEqual(body.think, true); + } + ); + + await check('ollama generate: records the usage it was told', async () => { + const transport = ollamaTransport( + [{ data: ollamaData({ response: 'ok' }) }], + [] + ); + await transport.generate( + 'hello', + {}, + { responseTokens: 300, numCtx: 2048 } + ); + assert.deepStrictEqual(transport.lastGenerateTextUsage, { + promptTokens: 9, + completionTokens: 3, + totalTokens: 12, + }); + }); + + await check( + 'ollama generate: a signal rides in the axios config', + async () => { + const posts = []; + const controller = new AbortController(); + await ollamaTransport( + [{ data: ollamaData({ response: 'ok' }) }], + posts + ).generate( + 'hello', + { signal: controller.signal }, + { responseTokens: 300, numCtx: 2048 } + ); + assert.strictEqual(posts[0].requestConfig.signal, controller.signal); + } + ); + + await check( + 'ollama generate: reasoning false overrules OLLAMA_THINK', + async () => { + const posts = []; + const previousThink = config.ollama.think; + config.ollama.think = true; + try { + await ollamaTransport( + [{ data: ollamaData({ response: 'ok' }) }], + posts + ).generate( + 'hello', + { reasoning: false }, + { responseTokens: 300, numCtx: 2048 } + ); + } finally { + config.ollama.think = previousThink; + } + assert.strictEqual(posts[0].body.think, false); + } + ); + + await check( + 'Custom: an empty generated answer says what came back', + async () => { + customService.client = fakeChatClient([], async () => + completion('', 'stop') + ); + await assert.rejects( + () => customService.generateText('hello'), + (error) => + error.message.startsWith( + 'Invalid API response structure: the answer carried no text (finish_reason stop; the response begins {"choices"' + ) + ); + } + ); + + /* --- every provider's generateText() goes through its transport ---------- */ + + for (const [label, service] of [ + ['OpenAI', openaiService], + ['Azure', azureService], + ['Custom', customService], + ['Ollama', ollamaService], + ]) { + await check( + `${label}: generateText() is the transport's generate()`, + async () => { + const original = service.transport.generate; + const seen = []; + service.transport.generate = async function (...args) { + seen.push(args); + this.lastGenerateTextUsage = { totalTokens: 42 }; + return 'from the transport'; + }; + try { + const text = await service.generateText('hello', { temperature: 0 }); + assert.strictEqual(text, 'from the transport'); + assert.strictEqual(seen.length, 1); + assert.strictEqual(seen[0][0], 'hello'); + assert.deepStrictEqual(seen[0][1], { temperature: 0 }); + assert.deepStrictEqual(service.lastGenerateTextUsage, { + totalTokens: 42, + }); + } finally { + service.transport.generate = original; + } + } + ); + } + + await check( + "Ollama: generateText() hands the service's numbers on", + async () => { + const original = ollamaService.transport.generate; + let budget; + ollamaService.transport.generate = async (prompt, options, numbers) => { + budget = numbers; + return 'ok'; + }; + try { + Object.assign(config, { tokenLimit: 1000, responseTokens: 400 }); + await ollamaService.generateText('x'.repeat(2000), { maxTokens: 250 }); + } finally { + ollamaService.transport.generate = original; + Object.assign(config, { tokenLimit: 128000, responseTokens: 1000 }); + } + assert.deepStrictEqual(budget, { responseTokens: 250, numCtx: 1000 }); + } + ); + + await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true }); + + if (failed > 0) { + console.error(`\n${failed} transport request mapping case(s) failed`); + process.exit(1); + } + console.log('\nAll transport request mapping cases passed'); +})(); diff --git a/tests/test-manual-analyze-route.js b/tests/test-manual-analyze-route.js new file mode 100644 index 00000000..e41538c5 --- /dev/null +++ b/tests/test-manual-analyze-route.js @@ -0,0 +1,217 @@ +/** + * Test: POST /manual/analyze in routes/setup.js + * + * The route used to switch on AI_PROVIDER over the provider singletons, and + * the five branches had drifted apart: only OpenAI and Anthropic recorded + * token metrics, OpenAI read `metrics.promptTokens` unguarded (so a failed + * analysis answered 500 with a TypeError instead of the failure record), and + * every branch passed `id || []` — a nonsense id that the thumbnail cache + * rejects. Since every provider has the same analyzeDocument() signature and + * result (docs/llm-service-refactoring.md, step 6), the route goes + * through AIServiceFactory.getService() like the scan loop does. + * + * The router runs for real on a throwaway database (tests/helpers/mount-router); + * only each provider's analyzeDocument() and the metrics store are replaced. + * + * Covers: + * 1. Every provider is reached through the factory with the name lists and + * the id as sent, and its result is answered unchanged + * 2. Every provider's metrics are recorded against the id + * 3. No id: analyzeDocument() gets undefined, nothing is recorded + * 4. A failed analysis (metrics: null) answers 200 with the failure record + * 5. Invalid content is a 400 and reaches no provider + * 6. A provider that throws is a 500 with its message + */ + +'use strict'; + +const assert = require('assert'); +const path = require('path'); + +const { mountRouter, REPO_ROOT } = require('./helpers/mount-router'); + +const API_KEY = 'test-api-key'; +const PROVIDERS = ['openai', 'ollama', 'custom', 'azure', 'anthropic']; +const SERVICE_FILES = { + openai: 'openaiService.js', + ollama: 'ollamaService.js', + custom: 'customService.js', + azure: 'azureService.js', + anthropic: 'anthropicService.js', +}; + +let passed = 0; +let failed = 0; + +async function test(name, fn) { + try { + await fn(); + console.log(`✅ ${name}`); + passed += 1; + } catch (error) { + console.error(`❌ ${name}`); + console.error(` ${error.stack || error.message}`); + failed += 1; + } +} + +function successFor(provider) { + return { + document: { + tags: ['Invoice'], + correspondent: `Acme via ${provider}`, + title: 'Acme invoice', + document_type: 'Invoice', + document_date: '2026-01-01', + language: 'en', + custom_fields: {}, + }, + metrics: { promptTokens: 10, completionTokens: 5, totalTokens: 15 }, + truncated: false, + }; +} + +const FAILURE = { + document: { tags: [], correspondent: null }, + metrics: null, + error: 'Invalid JSON response from API', + errorCode: undefined, +}; + +async function main() { + const harness = await mountRouter({ + stub: ({ paperlessService }) => { + paperlessService.listCorrespondentsNames = async () => [ + { id: 1, name: 'Acme' }, + ]; + paperlessService.listTagNames = async () => [{ id: 2, name: 'Invoice' }]; + paperlessService.listDocumentTypesNames = async () => [ + { id: 3, name: 'Bill' }, + ]; + }, + }); + + const config = require(path.join(REPO_ROOT, 'config', 'config.js')); + const services = {}; + for (const provider of PROVIDERS) { + services[provider] = require( + path.join(REPO_ROOT, 'services', SERVICE_FILES[provider]) + ); + } + + const recordedMetrics = []; + harness.documentModel.addOpenAIMetrics = async (...args) => { + recordedMetrics.push(args); + }; + + const calls = []; + /** Points the app at one provider and decides what it answers. */ + function useProvider(provider, answer) { + config.aiProvider = provider; + process.env.AI_PROVIDER = provider; + calls.length = 0; + recordedMetrics.length = 0; + for (const name of PROVIDERS) { + services[name].analyzeDocument = async (...args) => { + calls.push({ provider: name, args }); + if (answer instanceof Error) throw answer; + return answer; + }; + } + } + + const analyze = async (body) => { + const response = await fetch(harness.base + '/manual/analyze', { + method: 'POST', + headers: { 'x-api-key': API_KEY, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + return { status: response.status, body: await response.json() }; + }; + + try { + for (const provider of PROVIDERS) { + await test(`${provider}: reached through the factory, answered unchanged, metrics recorded`, async () => { + const result = successFor(provider); + useProvider(provider, result); + + const response = await analyze({ content: 'Invoice text', id: '42' }); + + assert.strictEqual(response.status, 200); + assert.deepStrictEqual(response.body, result); + assert.deepStrictEqual( + calls.map((call) => call.provider), + [provider] + ); + assert.deepStrictEqual(calls[0].args, [ + 'Invoice text', + ['Invoice'], + ['Acme'], + ['Bill'], + '42', + ]); + assert.deepStrictEqual(recordedMetrics, [['42', 10, 5, 15]]); + }); + } + + await test('without an id the provider gets undefined and nothing is recorded', async () => { + for (const provider of PROVIDERS) { + useProvider(provider, successFor(provider)); + const response = await analyze({ content: 'Invoice text' }); + assert.strictEqual(response.status, 200, provider); + assert.strictEqual(calls.length, 1, provider); + assert.strictEqual(calls[0].args.length, 5, provider); + assert.strictEqual(calls[0].args[4], undefined, provider); + assert.deepStrictEqual(recordedMetrics, [], provider); + } + }); + + await test('a failed analysis answers 200 with the failure record', async () => { + for (const provider of PROVIDERS) { + useProvider(provider, FAILURE); + const response = await analyze({ content: 'Invoice text', id: '42' }); + assert.strictEqual(response.status, 200, provider); + assert.deepStrictEqual( + response.body, + { + document: { tags: [], correspondent: null }, + metrics: null, + error: 'Invalid JSON response from API', + }, + provider + ); + assert.deepStrictEqual(recordedMetrics, [], provider); + } + }); + + await test('invalid content is a 400 and reaches no provider', async () => { + useProvider('openai', successFor('openai')); + for (const content of [undefined, '', 42]) { + const response = await analyze({ content, id: '42' }); + assert.strictEqual(response.status, 400, String(content)); + assert.deepStrictEqual(response.body, { + error: 'Valid content string is required', + }); + } + assert.strictEqual(calls.length, 0); + }); + + await test('a provider that throws is a 500 with its message', async () => { + useProvider('azure', new Error('provider exploded')); + const response = await analyze({ content: 'Invoice text', id: '42' }); + assert.strictEqual(response.status, 500); + assert.deepStrictEqual(response.body, { error: 'provider exploded' }); + assert.deepStrictEqual(recordedMetrics, []); + }); + } finally { + await harness.close(); + } + + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) process.exitCode = 1; +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/tests/test-native-install-log-paths.js b/tests/test-native-install-log-paths.js index cde4821c..58679078 100644 --- a/tests/test-native-install-log-paths.js +++ b/tests/test-native-install-log-paths.js @@ -24,11 +24,8 @@ function assertMatches(content, regex, message) { function run() { console.log('\n=== Native Install Log Path Checks ==='); - const filesUsingResponseLogPath = [ - 'services/openaiService.js', - 'services/customService.js', - 'services/azureService.js', - ]; + // Every provider writes the response log through the DocumentAnalyzer. + const filesUsingResponseLogPath = ['services/llm/documentAnalyzer.js']; filesUsingResponseLogPath.forEach((relativePath) => { const fullPath = path.join(process.cwd(), relativePath); diff --git a/tests/test-ollama-upstream-error.js b/tests/test-ollama-upstream-error.js new file mode 100644 index 00000000..900f7954 --- /dev/null +++ b/tests/test-ollama-upstream-error.js @@ -0,0 +1,199 @@ +/** + * Surface the upstream cause of Ollama 5xx responses and retry once. + * + * Field report: Ollama answered HTTP 500 { error: 'Post + * "http://127.0.0.1:49496/v1/completions": EOF' } — its own inference + * backend closed the connection on a large document. analyzeDocument only + * reported "Request failed with status code 500", hiding both the fact that + * the failure was upstream of Ollama and any hint about which limit bit. + * + * Covers: + * 1. The upstream error body reaches analysis.error / the log + * 2. A transient 5xx is retried once; a success on the second attempt wins + * 3. A persistent 5xx fails with the upstream detail preserved + */ + +'use strict'; + +const assert = require('assert'); + +process.env.AI_PROVIDER = 'ollama'; +process.env.OLLAMA_API_URL = 'http://ollama.test'; +process.env.OLLAMA_MODEL = 'test-model'; +process.env.SYSTEM_PROMPT = 'Analyse the document.'; +process.env.CUSTOM_FIELDS = JSON.stringify({ custom_fields: [] }); +delete process.env.CONTENT_MAX_LENGTH; + +const ollamaService = require('../services/ollamaService'); + +let passed = 0; +let failed = 0; + +async function test(name, fn) { + try { + await fn(); + console.log(`✅ ${name}`); + passed++; + } catch (error) { + console.error(`❌ ${name}`); + console.error(` ${error.message}`); + failed++; + } +} + +const UPSTREAM_ERROR = 'Post "http://127.0.0.1:49496/v1/completions": EOF'; + +/** An Ollama that answers like the field report: 500 carrying its backend error. */ +function ollamWithResponses(responses) { + const attempts = []; + return { + attempts, + post: async () => { + const response = responses.shift(); + attempts.push(response); + if (response instanceof Error) throw response; + return response; + }, + }; +} + +function successResponse() { + return { + data: { + done_reason: 'stop', + response: JSON.stringify({ + title: 'T', + correspondent: null, + tags: ['a'], + document_type: 'd', + document_date: '2026-01-01', + language: 'de', + }), + eval_count: 20, + prompt_eval_count: 30, + }, + }; +} + +function serverError(body) { + const error = new Error(`Request failed with status code 500`); + error.code = 'ERR_BAD_RESPONSE'; + error.response = { status: 500, data: body }; + return error; +} + +(async () => { + const originalSetTimeout = global.setTimeout; + const originalClient = ollamaService.client; + + try { + // Shorten retries in tests + const originalSetTimeout = setTimeout; + global.setTimeout = (fn) => originalSetTimeout(fn, 1); + + await test('The upstream error body reaches analysis.error', async () => { + ollamaService.client = ollamWithResponses([ + serverError({ error: UPSTREAM_ERROR }), + serverError({ error: UPSTREAM_ERROR }), + ]); + + const analysis = await ollamaService.analyzeDocument( + 'short content', + [], + [], + [], + 370 + ); + + assert.match( + analysis.error, + /v1\/completions.*EOF/s, + 'the operator must see WHICH backend call failed' + ); + assert.strictEqual( + analysis.errorCode, + 'ERR_BAD_RESPONSE', + 'the axios code stays available for classification' + ); + }); + + await test('A transient 5xx is retried once', async () => { + const ollama = ollamWithResponses([ + serverError({ error: UPSTREAM_ERROR }), + successResponse(), + ]); + ollamaService.client = ollama; + + const analysis = await ollamaService.analyzeDocument( + 'short content', + [], + [], + [], + 371 + ); + + assert.strictEqual( + ollama.attempts.length, + 2, + 'exactly one retry after the failed attempt' + ); + assert.strictEqual( + analysis.error, + undefined, + 'the retry succeeded and the answer is used' + ); + assert.deepStrictEqual(analysis.document.tags, ['a']); + }); + + await test('A persistent 5xx fails with the upstream detail kept', async () => { + const ollama = ollamWithResponses([ + serverError({ error: UPSTREAM_ERROR }), + serverError({ error: UPSTREAM_ERROR }), + ]); + ollamaService.client = ollama; + + const analysis = await ollamaService.analyzeDocument( + 'short content', + [], + [], + [], + 372 + ); + + assert.strictEqual(ollama.attempts.length, 2, 'one retry, then give up'); + assert.match(analysis.error, /v1\/completions.*EOF/s); + assert.strictEqual(analysis.errorCode, 'ERR_BAD_RESPONSE'); + }); + + await test('Client errors (4xx) are not retried', async () => { + const notFound = new Error('Request failed with status code 404'); + notFound.code = 'ERR_BAD_REQUEST'; + notFound.response = { status: 404, data: { error: 'model not found' } }; + const ollama = ollamWithResponses([notFound]); + ollamaService.client = ollama; + + const analysis = await ollamaService.analyzeDocument( + 'short content', + [], + [], + [], + 373 + ); + + assert.strictEqual( + ollama.attempts.length, + 1, + 'a 404 will never succeed on retry' + ); + assert.match(analysis.error, /model not found/); + }); + } finally { + global.setTimeout = originalSetTimeout; + ollamaService.client = originalClient; + } + + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) { + process.exitCode = 1; + } +})(); diff --git a/tests/test-predefined-retrigger.js b/tests/test-predefined-retrigger.js new file mode 100644 index 00000000..dd01ef9d --- /dev/null +++ b/tests/test-predefined-retrigger.js @@ -0,0 +1,140 @@ +/** + * Reproduces the broken "Process only specific pre-tagged documents" workflow. + * + * The operator assigns a trigger tag (e.g. perform-ai-scan) in Paperless-ngx, + * expecting Zettelrobbe to process the document — including AGAIN, when the + * tag is set on a document that was already analysed (the documented way to + * re-run documents after fixing a bug). + * + * Two defects made that workflow impossible: + * 1. processDocument() consulted processed_documents before anything else, so + * a re-tagged document was silently skipped forever. + * 2. updateDocument() merged the AI tag list into the current tags, so the + * trigger tag stuck around after processing instead of being removed. + * + * Covers: + * 1. The scan loop re-processes a tagged document that is already recorded + * as processed (source-level guard, since the loop lives in server.js) + * 2. Ignored and permanently-failed documents stay protected + * 3. updateDocument() drops the trigger tag from the merged payload when + * asked to (functional, against a mocked Paperless API) + */ + +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +let passed = 0; +let failed = 0; + +async function test(name, fn) { + try { + await fn(); + console.log(`✅ ${name}`); + passed++; + } catch (error) { + console.error(`❌ ${name}`); + console.error(` ${error.message}`); + failed++; + } +} + +const serverSource = fs.readFileSync( + path.join(process.cwd(), 'server.js'), + 'utf8' +); + +test('A tagged document is re-processed even when already recorded', () => { + // The processed-documents guard must not fire unconditionally: in + // predefined mode the presence of a configured trigger tag is the + // operator's explicit request to run the document again. + assert.match( + serverSource, + /isDocumentProcessed\(doc\.id\)[\s\S]{0,600}?predefined/i, + 'The isProcessed guard must consider the predefined trigger tags' + ); +}); + +test('The re-trigger respects ignored and permanently-failed documents', () => { + const processedIdx = serverSource.indexOf('isDocumentProcessed(doc.id)'); + const ignoredIdx = serverSource.indexOf('isDocumentIgnored(doc.id)'); + const failedIdx = serverSource.indexOf('isDocumentFailed(doc.id)'); + assert.ok(processedIdx !== -1 && ignoredIdx !== -1 && failedIdx !== -1); + // Ignored/failed checks come after the processed check but must NOT sit + // behind the re-trigger bypass, or a re-tagged ignore would resurrect. + const between = serverSource.slice(ignoredIdx, failedIdx + 200); + assert.ok( + !/return null;\s*\}\s*\n\s*\/\/.*predefined|re-?trigger/i.test(between), + 'Ignored/failed guards must stay ahead of any re-trigger logic' + ); +}); + +(async () => { + // ── updateDocument drops the trigger tag on request ────────────────────── + process.env.PAPERLESS_API_URL = 'http://paperless.test'; + process.env.PAPERLESS_API_TOKEN = 'test-token'; + + const calls = []; + const paperlessService = require('../services/paperlessService'); + + const originalClient = paperlessService.client; + paperlessService.client = { + defaults: { baseURL: 'http://paperless.test/api' }, + get: async (url) => { + calls.push({ method: 'get', url }); + if (url === '/documents/315/') { + return { + data: { + id: 315, + tags: [11, 42], // 42 is the trigger tag + correspondent: 7, + }, + }; + } + throw new Error(`unexpected GET ${url}`); + }, + patch: async (url, body) => { + calls.push({ method: 'patch', url, body }); + if (url === '/documents/315/') { + return { data: { ok: true } }; + } + throw new Error(`unexpected PATCH ${url}`); + }, + post: async (url) => { + throw new Error(`unexpected POST ${url}`); + }, + }; + + try { + await test('updateDocument removes configured tag ids from the merged payload', async () => { + await paperlessService.updateDocument(315, { + title: 'New title', + tags: [5], // AI-chosen tags + removeTagIds: [42], // the predefined-mode trigger tag + }); + + const updateCall = calls.find((call) => call.method === 'patch'); + assert.ok(updateCall, 'an update request was sent'); + const sentTags = updateCall.body.tags; + assert.deepStrictEqual( + [...sentTags].sort((a, b) => a - b), + [5, 11], + 'AI tags merge with existing tags, the trigger tag is gone' + ); + assert.strictEqual( + sentTags.includes(42), + false, + 'the trigger tag must not survive the write-back' + ); + }); + } finally { + paperlessService.client = originalClient; + } + + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) { + process.exitCode = 1; + } +})(); diff --git a/tests/test-response-truncation-detection.js b/tests/test-response-truncation-detection.js index ff235d30..006089ed 100644 --- a/tests/test-response-truncation-detection.js +++ b/tests/test-response-truncation-detection.js @@ -7,9 +7,16 @@ * surfaced as "Invalid JSON response from API" a few lines later — and sent * the document to the OCR queue, which re-reads the PDF and re-runs the same * request against the same limit. + * + * analyzeDocument() is exercised per provider: the scan loop reads + * analysis.errorCode, so a service that raises the right error but drops the + * code on the way out records a generic failure and loses the OCR decision. + * The provider client is the only collaborator stubbed; the thumbnail cache is + * pre-seeded so the analysis path reaches the client without a Paperless call. */ const assert = require('assert'); +const fs = require('fs').promises; process.env.AI_PROVIDER = 'openai'; process.env.OPENAI_API_KEY = 'test-key'; @@ -17,6 +24,14 @@ process.env.OPENAI_MODEL = 'gpt-4'; process.env.SYSTEM_PROMPT = 'Analyse the document.'; const { assertCompletionNotTruncated } = require('../services/serviceUtils'); +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('../services/thumbnailCachePaths'); +const openaiService = require('../services/openaiService'); +const azureService = require('../services/azureService'); +const customService = require('../services/customService'); +const ollamaService = require('../services/ollamaService'); let failed = 0; const check = async (label, fn) => { @@ -38,6 +53,16 @@ const COMPLETE_ANSWER = JSON.stringify({ language: 'de', }); +// A prefix of the answer above: valid JSON right up to the point it stops. +const CUT_OFF_ANSWER = COMPLETE_ANSWER.slice(0, -12); + +// Own id so the seeded thumbnail cannot collide with a real cached document. +const DOCUMENT_ID = 'test263truncation'; + +// The response limit the services read from config.responseTokens; eval_count +// reaching it is what tells Ollama's guard which limit bit. +const RESPONSE_TOKENS = 1000; + function completion(finishReason, content = COMPLETE_ANSWER) { return { choices: [{ message: { content }, finish_reason: finishReason }], @@ -45,6 +70,60 @@ function completion(finishReason, content = COMPLETE_ANSWER) { }; } +/** Replaces the SDK client with one that answers whatever the case needs. */ +function stubClient(service, response) { + service.client = { + chat: { completions: { create: async () => response } }, + models: { list: async () => ({ data: [] }) }, + }; +} + +/** Same for Ollama, which answers over axios instead of an SDK. */ +function stubOllamaClient(responseData) { + ollamaService.client = { + post: async () => ({ data: responseData }), + get: async () => ({ status: 200, data: { models: [] } }), + }; +} + +/** + * Per-provider arrangements for the same two events. Kept as thunks so each + * case re-stubs immediately before it runs and cannot inherit a stale client. + */ +function openAiCompatibleProvider(label, service) { + return { + label, + service, + arrangeCutOff: () => + stubClient(service, completion('length', CUT_OFF_ANSWER)), + arrangeComplete: () => stubClient(service, completion('stop')), + }; +} + +const PROVIDERS = [ + openAiCompatibleProvider('OpenAI', openaiService), + openAiCompatibleProvider('Azure', azureService), + openAiCompatibleProvider('Custom', customService), + { + label: 'Ollama', + service: ollamaService, + arrangeCutOff: () => + stubOllamaClient({ + response: CUT_OFF_ANSWER, + done_reason: 'length', + eval_count: RESPONSE_TOKENS, + prompt_eval_count: 900, + }), + arrangeComplete: () => + stubOllamaClient({ + response: COMPLETE_ANSWER, + done_reason: 'stop', + eval_count: 120, + prompt_eval_count: 900, + }), + }, +]; + (async () => { console.log('\n=== Response truncation detection ==='); @@ -89,57 +168,69 @@ function completion(finishReason, content = COMPLETE_ANSWER) { ); }); - /* --- each provider actually consults it -------------------------------- */ - - await check('every OpenAI-compatible analysis path calls the guard', () => { - // Read from source rather than driven through a stub: the analysis path - // caches a thumbnail first, which would drag Paperless-ngx into the test. - // The guard has to sit inside analyzeDocument(), not merely somewhere in - // the file, or a later refactor could move it out without failing here. - const fs = require('fs'); - const path = require('path'); - for (const name of ['openaiService', 'azureService', 'customService']) { - const source = fs.readFileSync( - path.join(__dirname, '..', 'services', `${name}.js`), - 'utf8' - ); - const entry = source.indexOf('async analyzeDocument('); - assert.notStrictEqual( - entry, - -1, - `${name}: analyzeDocument() is gone — the analysis path moved` - ); - assert.ok( - source.indexOf('assertCompletionNotTruncated(', entry) !== -1, - `${name}: analyzeDocument() no longer consults the truncation guard` - ); - } - }); + /* --- each provider actually consults it ------------------------------- */ - await check('every analysis catch block carries the code onward', () => { - // The scan loop reads analysis.errorCode; a service that throws the right - // error but drops the code on the way out records a generic failure. - const fs = require('fs'); - for (const name of [ - 'openaiService', - 'azureService', - 'customService', - 'ollamaService', - ]) { - const source = fs.readFileSync(`services/${name}.js`, 'utf8'); - const guards = (source.match(/errorCode: error\.code/g) || []).length; - const catches = ( - source.match( - /document: \{ tags: \[\], correspondent: null \},\n\s*metrics: null,/g - ) || [] - ).length; - assert.strictEqual( - guards, - catches, - `${name}: ${catches} analysis catch block(s) but ${guards} carry errorCode` - ); - } - }); + /* Pre-seed the thumbnail cache so analyzeDocument() gets past its caching + step without reaching Paperless. Ollama skips the step entirely once the + file is there, the OpenAI-compatible three take their fs.access() hit. */ + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile( + getThumbnailCachePath(DOCUMENT_ID), + Buffer.from([0x89, 0x50, 0x4e, 0x47]) + ); + + /* The failure shape the scan loop branches on: no metadata to write back, + no metrics to record, and the code that decides OCR fallback. Asserted + rather than read out of the source, so it survives the code moving. */ + const assertReportsCutOff = (analysis) => { + assert.strictEqual(analysis.errorCode, 'ai_response_truncated'); + assert.match(analysis.error, /stopped generating/); + assert.deepStrictEqual(analysis.document, { + tags: [], + correspondent: null, + }); + assert.strictEqual(analysis.metrics, null); + }; + + for (const { label, service, arrangeCutOff, arrangeComplete } of PROVIDERS) { + await check( + `${label}: analyzeDocument reports a cut-off answer with its code`, + async () => { + // Truncated JSON would otherwise fail to parse and be blamed on the model. + arrangeCutOff(); + const analysis = await service.analyzeDocument( + 'Rechnung', + [], + [], + [], + DOCUMENT_ID + ); + assertReportsCutOff(analysis); + } + ); + + await check( + `${label}: analyzeDocument passes a complete answer through`, + async () => { + arrangeComplete(); + const analysis = await service.analyzeDocument( + 'Rechnung', + [], + [], + [], + DOCUMENT_ID + ); + assert.strictEqual(analysis.error, undefined); + assert.strictEqual(analysis.errorCode, undefined); + assert.strictEqual( + analysis.document.correspondent, + 'Telekom Deutschland GmbH' + ); + } + ); + } + + await fs.rm(getThumbnailCachePath(DOCUMENT_ID), { force: true }); if (failed > 0) { console.error(`\n${failed} truncation detection case(s) failed`); diff --git a/tests/test-tag-cache-page-size.js b/tests/test-tag-cache-page-size.js index 0debb33c..311d2ba3 100644 --- a/tests/test-tag-cache-page-size.js +++ b/tests/test-tag-cache-page-size.js @@ -48,9 +48,16 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) { return { requests, defaults: { baseURL: BASE_URL }, - get: async (url) => { - requests.push(url); - const parsed = new URL(url, BASE_URL); + get: async (url, options = {}) => { + // Merge axios-style params into the query string, as axios itself would. + const merged = new URL(url, BASE_URL); + if (options.params) { + for (const [key, value] of Object.entries(options.params)) { + merged.searchParams.set(key, String(value)); + } + } + requests.push(merged.pathname + merged.search); + const parsed = merged; const requested = Number(parsed.searchParams.get('page_size')) || 25; const pageSize = Math.min(requested, maxPageSize); const page = Number(parsed.searchParams.get('page')) || 1; @@ -82,12 +89,13 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) { await test('The tag cache is not rebuilt 25 tags at a time', async () => { const server = createTagServer({ tagCount: 1331 }); paperlessService.client = server; - paperlessService.tagCache.clear(); + paperlessService.tags._clear(); + paperlessService.tags.cache_last_fetch_time = 0; await paperlessService.refreshTagCache(); assert.strictEqual( - paperlessService.tagCache.size, + paperlessService.tags.element_list.length, 1331, 'Every tag has to reach the cache' ); @@ -107,7 +115,8 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) { await test('The page size carries into every following page', async () => { const server = createTagServer({ tagCount: 1331 }); paperlessService.client = server; - paperlessService.tagCache.clear(); + paperlessService.tags._clear(); + paperlessService.tags.cache_last_fetch_time = 0; await paperlessService.refreshTagCache(); @@ -125,12 +134,13 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) { // its own maximum and says so only through the shorter result list. const server = createTagServer({ tagCount: 1331, maxPageSize: 100 }); paperlessService.client = server; - paperlessService.tagCache.clear(); + paperlessService.tags._clear(); + paperlessService.tags.cache_last_fetch_time = 0; await paperlessService.refreshTagCache(); assert.strictEqual( - paperlessService.tagCache.size, + paperlessService.tags.element_list.length, 1331, 'A clamped page size must not lose tags' ); @@ -138,8 +148,9 @@ function createTagServer({ tagCount, maxPageSize = 100000 }) { }); } finally { paperlessService.client = originalClient; - paperlessService.tagCache.clear(); - paperlessService.lastTagRefresh = 0; + paperlessService.tags._clear(); + paperlessService.tags.cache_last_fetch_time = 0; + paperlessService.tags.cache_last_fetch_time = 0; } console.log(`\n${passed} passed, ${failed} failed`); diff --git a/tests/test-thumbnail-optional-analysis.js b/tests/test-thumbnail-optional-analysis.js new file mode 100644 index 00000000..39c2b5c7 --- /dev/null +++ b/tests/test-thumbnail-optional-analysis.js @@ -0,0 +1,232 @@ +/** + * Thumbnail caching must not decide whether a document gets analysed. + * + * analyzeDocument() caches the document thumbnail for the UI, but the + * thumbnail never enters the prompt — so every outcome of that side step has + * to leave the analysis itself alone. Two ways it did not: + * + * 1. getThumbnailCachePath() ran outside the try block, so an id it rejects + * ('' after sanitizing, null, undefined) made analyzeDocument() *throw* + * instead of returning a result. The scan loop reads analysis.error and + * never expects a throw; POST /manual/analyze passed `id || []`, which is + * exactly such a value, and answered 500 for every request without an id. + * 2. A thumbnail Paperless does not have returned `undefined` from + * analyzeDocument(), and every caller immediately reads `.document` on it. + * + * Ollama already treated the step as best effort (_handleThumbnailCaching + * returns early on a falsy id); it is in here to keep the four providers from + * drifting apart again. + */ + +const assert = require('assert'); +const fs = require('fs').promises; + +process.env.AI_PROVIDER = 'openai'; +process.env.OPENAI_API_KEY = 'test-key'; +process.env.OPENAI_MODEL = 'gpt-4'; +process.env.AZURE_API_KEY = 'test-key'; +process.env.AZURE_ENDPOINT = 'https://example.invalid'; +process.env.AZURE_DEPLOYMENT_NAME = 'test-deployment'; +process.env.CUSTOM_BASE_URL = 'https://example.invalid/v1'; +process.env.CUSTOM_API_KEY = 'test-key'; +process.env.CUSTOM_MODEL = 'test-model'; +process.env.SYSTEM_PROMPT = 'Analyse the document.'; + +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('../services/thumbnailCachePaths'); +const paperlessService = require('../services/paperlessService'); +const openaiService = require('../services/openaiService'); +const azureService = require('../services/azureService'); +const customService = require('../services/customService'); +const ollamaService = require('../services/ollamaService'); + +let failed = 0; +const check = async (label, fn) => { + try { + await fn(); + console.log(` ✓ ${label}`); + } catch (error) { + failed += 1; + console.error(` ✗ ${label}: ${error.message}`); + } +}; + +const ANSWER = { + title: 'Telekom invoice July', + correspondent: 'Telekom Deutschland GmbH', + tags: ['invoice'], + document_type: 'Rechnung', + document_date: '2026-09-03', + language: 'de', +}; + +const CHAT_ANSWER = { + choices: [ + { message: { content: JSON.stringify(ANSWER) }, finish_reason: 'stop' }, + ], + usage: { prompt_tokens: 900, completion_tokens: 120, total_tokens: 1020 }, +}; + +const OLLAMA_ANSWER = { + response: JSON.stringify(ANSWER), + done_reason: 'stop', + eval_count: 120, + prompt_eval_count: 900, +}; + +const PROVIDERS = [ + { + label: 'OpenAI', + service: openaiService, + stub: () => { + openaiService.client = { + chat: { completions: { create: async () => CHAT_ANSWER } }, + }; + }, + }, + { + label: 'Azure', + service: azureService, + stub: () => { + azureService.client = { + chat: { completions: { create: async () => CHAT_ANSWER } }, + }; + }, + }, + { + label: 'Custom', + service: customService, + stub: () => { + customService.client = { + chat: { completions: { create: async () => CHAT_ANSWER } }, + }; + }, + }, + { + label: 'Ollama', + service: ollamaService, + stub: () => { + ollamaService.client = { post: async () => ({ data: OLLAMA_ANSWER }) }; + }, + }, +]; + +/** The document came back analysed, whatever the thumbnail step did. */ +function assertAnalysed(analysis) { + assert.ok(analysis, 'analyzeDocument returned nothing at all'); + assert.strictEqual(analysis.error, undefined); + assert.strictEqual( + analysis.document.correspondent, + 'Telekom Deutschland GmbH' + ); + assert.deepStrictEqual(analysis.document.tags, ['invoice']); +} + +const originalGetThumbnailImage = paperlessService.getThumbnailImage; +const writtenCachePaths = []; + +(async () => { + console.log('\n=== Thumbnail caching is optional for analysis ==='); + + try { + for (const { label, service, stub } of PROVIDERS) { + /* --- ids the cache path helper rejects ----------------------------- */ + + // `id || []` is what routes/setup.js used to hand over when the request + // omitted one, and String([]) sanitizes to the empty string. + for (const id of [[], undefined, null, '', '///']) { + await check( + `${label}: an unusable id (${JSON.stringify(id)}) still analyses`, + async () => { + stub(); + paperlessService.getThumbnailImage = async () => { + throw new Error( + 'Paperless must not be called for an unusable id' + ); + }; + assertAnalysed( + await service.analyzeDocument('Rechnung', [], [], [], id) + ); + } + ); + } + + /* --- Paperless cannot supply the thumbnail ------------------------- */ + + await check( + `${label}: a thumbnail Paperless does not have still analyses`, + async () => { + stub(); + paperlessService.getThumbnailImage = async () => null; + assertAnalysed( + await service.analyzeDocument( + 'Rechnung', + [], + [], + [], + `thumbmissing${label}` + ) + ); + } + ); + + await check( + `${label}: a failing thumbnail fetch still analyses`, + async () => { + stub(); + paperlessService.getThumbnailImage = async () => { + throw new Error('Paperless unreachable'); + }; + assertAnalysed( + await service.analyzeDocument( + 'Rechnung', + [], + [], + [], + `thumbbroken${label}` + ) + ); + } + ); + + /* --- the feature itself still works ------------------------------- */ + + await check( + `${label}: an available thumbnail is still written to the cache`, + async () => { + stub(); + paperlessService.getThumbnailImage = async () => + Buffer.from([0x89, 0x50, 0x4e, 0x47]); + + const id = `thumbok${label}`; + const cachePath = getThumbnailCachePath(id); + writtenCachePaths.push(cachePath); + await fs.rm(cachePath, { force: true }); + + assertAnalysed( + await service.analyzeDocument('Rechnung', [], [], [], id) + ); + await assert.doesNotReject( + fs.access(cachePath), + `expected a cached thumbnail at ${cachePath}` + ); + } + ); + } + } finally { + paperlessService.getThumbnailImage = originalGetThumbnailImage; + for (const cachePath of writtenCachePaths) { + await fs.rm(cachePath, { force: true }); + } + } + + if (failed > 0) { + console.error(`\n${failed} thumbnail independence case(s) failed`); + process.exit(1); + } + console.log( + `\nAll thumbnail independence cases passed (cache dir: ${THUMBNAIL_CACHE_DIR})` + ); +})();