diff --git a/CLAUDE.md b/CLAUDE.md index 0a0a51a6..40bc92f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this is Paperless-AI **next** — a community fork of clusterzx/paperless-ai. A Node.js/Express app that uses -LLMs (OpenAI, Ollama, Azure, or any OpenAI-compatible "custom" endpoint) plus optional OCR to auto-tag, +LLMs (OpenAI, Ollama, Azure, Anthropic, or any OpenAI-compatible "custom" endpoint) plus optional OCR to auto-tag, title, classify, and extract metadata for documents in a Paperless-ngx instance. Server-rendered EJS UI, no frontend framework. Single SQLite database (better-sqlite3, WAL mode). @@ -54,12 +54,18 @@ the `services/` singletons. `class Foo { ... }; module.exports = new Foo();`. Import and call directly; do not `new` them. **AI provider abstraction:** `services/aiServiceFactory.js` returns the right service based on -`config.aiProvider` (`openai` | `ollama` | `custom` | `azure`). Every provider service implements +`config.aiProvider` (`openai` | `ollama` | `custom` | `azure` | `anthropic`). Every provider service implements `analyzeDocument(content, doc, existingTags, correspondents)`. To add a provider: add a service singleton, wire it into the factory switch, and add its config block in `config/config.js`. -**Token handling:** `services/serviceUtils.js` — `calculateTokens()` (tiktoken for OpenAI, ÷4 char -estimate otherwise) and `truncateToTokenLimit()`. Also holds AI-error → OCR-fallback classification +**Anthropic** is not OpenAI-compatible and has its own service (`services/anthropicService.js`). Per-model +request rules (thinking, effort, structured output, cache minimums), the client factory, usage mapping and +the truncation/refusal check live in `services/anthropicSupport.js` and are shared with Claude OCR; never +send `temperature` to it or read `content[0]`. Its analysis prompt and JSON schema come from +`services/analysisPrompt.js`, which the other providers do not use yet. + +**Token handling:** `services/serviceUtils.js` — `calculateTokens()` (tiktoken for OpenAI, ÷3 char +estimate for `claude-*`, ÷4 otherwise) and `truncateToTokenLimit()`. Also holds AI-error → OCR-fallback classification helpers used by the scan loop. **Prompt restrictions:** `services/restrictionPromptService.js` replaces placeholders like @@ -69,7 +75,7 @@ in prompts. The base prompts live in `config/config.js` (`specialPromptPreDefine normalize them with `toNameList()` from `services/serviceUtils.js` instead of joining them directly. **OCR:** `services/mistralOcrService.js` — provider can be `mistral`, `ollama` (native `/api/chat` -vision), or OpenAI-compatible `/v1`. OCR is a fallback path when AI analysis fails on low-text docs. +vision), OpenAI-compatible `/v1`, or `anthropic` (Claude reads PDFs and images natively). OCR is a fallback path when AI analysis fails on low-text docs. **Database:** `models/document.js` — better-sqlite3 at `data/documents.db`, WAL mode, prepared statements throughout. Key tables: `processed_documents`, `history_documents` (server-side paginated diff --git a/OPENAPI/openapi.json b/OPENAPI/openapi.json index ec2c115c..4205f5f7 100644 --- a/OPENAPI/openapi.json +++ b/OPENAPI/openapi.json @@ -471,14 +471,24 @@ }, "message": { "type": "string", - "description": "Additional status information (for non-healthy states)", + "description": "Additional status information (for non-healthy states). Authenticated callers only.", "example": "Document scan failed 3 time(s) in a row: connect ECONNREFUSED 172.18.0.2:8000" }, "scanner": { - "$ref": "#/components/schemas/ScannerHealth" + "description": "Authenticated callers only.", + "allOf": [ + { + "$ref": "#/components/schemas/ScannerHealth" + } + ] }, "paperless": { - "$ref": "#/components/schemas/PaperlessHealth" + "description": "Authenticated callers only.", + "allOf": [ + { + "$ref": "#/components/schemas/PaperlessHealth" + } + ] } } }, @@ -4609,6 +4619,10 @@ "nullable": true, "description": "Recommended default; absent for providers without model classification" }, + "live": { + "type": "boolean", + "description": "anthropic only — false when the live list failed and the suggested Claude models are returned instead" + }, "resolvedApiUrl": { "type": "string" }, @@ -4697,17 +4711,21 @@ "openai", "ollama", "custom", - "azure" + "azure", + "anthropic" ] }, "apiUrl": { - "type": "string" + "type": "string", + "description": "Not needed for anthropic unless a proxy sits in front of api.anthropic.com" }, "token": { - "type": "string" + "type": "string", + "description": "Omit or leave empty to use the stored key of the selected provider" }, "model": { - "type": "string" + "type": "string", + "description": "For anthropic, checked with a models lookup that spends no tokens" }, "setupValidationTimeoutMs": { "type": "integer" @@ -4727,6 +4745,217 @@ } } }, + "/api/settings/ocr/test": { + "post": { + "summary": "Test OCR provider connectivity from the settings page", + "description": "Runs one OCR request on a generated test image. For `anthropic` the\nkey is the OCR key when one is given or stored for Claude OCR, and\notherwise the Anthropic AI provider's key (`anthropicApiKey`, then\nthe stored ANTHROPIC_API_KEY).\n", + "tags": [ + "Settings" + ], + "security": [ + { + "BearerAuth": [] + }, + { + "ApiKeyAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "enabled": { + "type": "string", + "enum": [ + "yes", + "no" + ] + }, + "provider": { + "type": "string", + "enum": [ + "mistral", + "custom", + "ollama", + "anthropic" + ] + }, + "apiUrl": { + "type": "string", + "description": "Ignored for mistral. For anthropic an optional proxy URL; empty uses the Anthropic AI provider's URL, then api.anthropic.com" + }, + "apiKey": { + "type": "string", + "description": "Omit or leave empty to use the stored key" + }, + "model": { + "type": "string", + "description": "For anthropic, a Claude model; anything else falls back to the AI model" + }, + "setupOcrValidationTimeoutMs": { + "type": "integer" + }, + "anthropicApiKey": { + "type": "string", + "description": "Anthropic AI key typed but not saved yet; Claude OCR falls back to it" + }, + "anthropicApiUrl": { + "type": "string", + "description": "Anthropic AI proxy URL typed but not saved yet" + }, + "anthropicModel": { + "type": "string", + "description": "Anthropic AI model typed but not saved yet" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OCR connectivity result returned" + }, + "401": { + "description": "Unauthorized" + } + } + } + }, + "/api/settings/oidc/test": { + "post": { + "summary": "Test a single sign-on provider from the settings page", + "description": "Reads the provider's discovery document, picks the token endpoint auth\nmethod the login would use, and checks the client credentials by\nredeeming an authorization code that cannot exist: the provider\nanswers invalid_client for wrong credentials and invalid_grant for\nright ones. Nobody is signed in and nothing is saved.\n\nAn empty `clientSecret` uses the stored OIDC_CLIENT_SECRET, but only\nwhen the issuer has the same origin (scheme, host, port) as the\nconfigured OIDC_ISSUER_URL; for any other issuer the test runs\nwithout a secret. Private network addresses are allowed, since\nidentity providers usually run next to the application.\n", + "tags": [ + "Settings" + ], + "security": [ + { + "BearerAuth": [] + }, + { + "ApiKeyAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "issuerUrl", + "clientId" + ], + "properties": { + "issuerUrl": { + "type": "string", + "description": "Issuer URL (http or https); a pasted discovery URL is accepted", + "example": "https://auth.example.com/application/o/zettelrobbe/" + }, + "clientId": { + "type": "string", + "example": "zettelrobbe" + }, + "clientSecret": { + "type": "string", + "description": "Omit or leave empty to use the stored secret (same issuer origin only)", + "example": "" + }, + "tokenAuthMethod": { + "type": "string", + "description": "Empty picks the method automatically", + "enum": [ + "", + "client_secret_basic", + "client_secret_post", + "none" + ], + "example": "" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Test result. success is false (still HTTP 200) when the provider\ncould not be reached, sent no usable discovery document, or refused\nthe client credentials; error then says which.\n", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "success": { + "type": "boolean", + "example": true + }, + "data": { + "type": "object", + "properties": { + "issuer": { + "type": "string", + "description": "The issuer the provider advertises", + "example": "https://auth.example.com/application/o/zettelrobbe/" + }, + "tokenAuthMethods": { + "type": "array", + "description": "Token endpoint auth methods the provider advertises (empty when it lists none)", + "items": { + "type": "string" + }, + "example": [ + "client_secret_basic", + "client_secret_post" + ] + }, + "chosenAuthMethod": { + "type": "string", + "description": "The method a login would use", + "example": "client_secret_basic" + }, + "pkce": { + "type": "boolean", + "description": "Whether the provider advertises PKCE with S256", + "example": true + } + } + }, + "message": { + "type": "string", + "example": "The provider answered and accepted the client." + } + } + }, + { + "$ref": "#/components/schemas/Error" + } + ] + } + } + } + }, + "302": { + "description": "Not signed in; redirected to /login like every other route behind the authentication guard" + }, + "400": { + "description": "Invalid input (issuer not an http(s) URL, empty client ID, unknown auth method, or a method that needs a secret without one)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, "/api/settings/ocr/models": { "post": { "summary": "Discover available OCR models from the settings page", @@ -4754,7 +4983,8 @@ "enum": [ "mistral", "custom", - "ollama" + "ollama", + "anthropic" ] }, "apiUrl": { @@ -4762,10 +4992,22 @@ }, "apiKey": { "type": "string", - "description": "Omit or leave empty to use the stored OCR_API_KEY / MISTRAL_API_KEY" + "description": "Omit or leave empty to use the stored OCR_API_KEY / MISTRAL_API_KEY (for anthropic, the stored Claude OCR key, then the Anthropic AI key)" }, "setupOcrValidationTimeoutMs": { "type": "integer" + }, + "anthropicApiKey": { + "type": "string", + "description": "Anthropic AI key typed but not saved yet; used by anthropic when no OCR key is set" + }, + "anthropicApiUrl": { + "type": "string", + "description": "Anthropic AI proxy URL typed but not saved yet" + }, + "anthropicModel": { + "type": "string", + "description": "Anthropic AI model; anthropic returns it as suggestedModel" } } } @@ -4802,6 +5044,10 @@ "nullable": true, "description": "Recommended default; absent for providers without model classification" }, + "live": { + "type": "boolean", + "description": "anthropic only — false when the live list failed and the suggested Claude models are returned instead" + }, "resolvedApiUrl": { "type": "string" }, @@ -5642,7 +5888,8 @@ "openai", "ollama", "custom", - "azure" + "azure", + "anthropic" ], "example": "openai" }, @@ -5686,6 +5933,58 @@ "description": "Model name for custom LLM provider", "example": "custom-model" }, + "anthropicApiKey": { + "type": "string", + "description": "API key for Anthropic (required when aiProvider is 'anthropic' and none is stored; leave empty to keep the configured value). Saved only with aiProvider 'anthropic'; the anthropic* fields below likewise.", + "example": "sk-ant-api03-abc123" + }, + "anthropicModel": { + "type": "string", + "description": "Claude model for analysis; empty keeps the configured one", + "example": "claude-sonnet-5-5" + }, + "anthropicEffort": { + "type": "string", + "description": "Effort level; empty leaves it to the model. Rejected with 400 when it is not one of the listed levels.", + "enum": [ + "", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "example": "" + }, + "anthropicMaxTokens": { + "type": "integer", + "minimum": 1, + "description": "Completion cap per request, thinking included. Rejected with 400 when it is not a positive whole number; empty keeps the configured value.", + "example": 16000 + }, + "anthropicStructuredOutput": { + "type": "string", + "description": "Constrain the analysis answer to a JSON schema (yes/no)", + "enum": [ + "yes", + "no" + ], + "example": "yes" + }, + "anthropicPromptCaching": { + "type": "string", + "description": "Mark the system prompt cacheable (yes/no)", + "enum": [ + "yes", + "no" + ], + "example": "yes" + }, + "anthropicApiUrl": { + "type": "string", + "description": "Only for a proxy; empty uses api.anthropic.com", + "example": "" + }, "scanInterval": { "type": "number", "description": "Interval in minutes for scanning new documents", @@ -5922,6 +6221,17 @@ "description": "How many entries the vocabulary proposed from the tag names should aim at, types and topics together (5-100, default 25). Out-of-range values are clamped, a non-numeric value keeps the current setting.", "example": 25 }, + "ocrProvider": { + "type": "string", + "description": "OCR fallback provider. For anthropic (Claude OCR) an empty ocrApiKey uses the Anthropic AI key, and ocrApiUrl is an optional proxy URL; a key or URL stored for another provider is cleared when switching to it.", + "enum": [ + "mistral", + "custom", + "ollama", + "anthropic" + ], + "example": "mistral" + }, "ocrAutoProcessEnabled": { "type": "string", "description": "Process queued OCR documents automatically (yes/no)", @@ -5950,6 +6260,91 @@ "YYYY-MM-DD" ], "example": "DD.MM.YYYY" + }, + "oidcEnabled": { + "type": "string", + "description": "Single sign-on switch (OIDC_ENABLED). Every oidc* field is optional; one the body\nleaves out keeps the stored value. When single sign-on ends up switched on, the save\nis refused with 400 unless the issuer URL is an http(s) URL, the client ID is set,\nthe admin e-mail is valid, the redirect URI is empty or an absolute http(s) URL, and\nthe auth method, button icon and colours are valid. Values injected by the container\nare never overwritten.\n", + "enum": [ + "yes", + "no" + ], + "example": "no" + }, + "oidcIssuerUrl": { + "type": "string", + "description": "Issuer URL of the OpenID Connect provider (OIDC_ISSUER_URL); a pasted discovery URL is accepted", + "example": "https://auth.example.com/application/o/zettelrobbe/" + }, + "oidcClientId": { + "type": "string", + "description": "Client ID registered at the provider (OIDC_CLIENT_ID)", + "example": "zettelrobbe" + }, + "oidcClientSecret": { + "type": "string", + "description": "Client secret (OIDC_CLIENT_SECRET); leave empty to keep the stored one", + "example": "" + }, + "oidcAdminEmail": { + "type": "string", + "description": "E-mail address of the identity that signs in as the administrator (OIDC_ADMIN_EMAIL)", + "example": "admin@example.com" + }, + "oidcRedirectUri": { + "type": "string", + "description": "Callback URL registered at the provider (OIDC_REDIRECT_URI); empty derives it from the request", + "example": "" + }, + "oidcScopes": { + "type": "string", + "description": "Space-separated scopes (OIDC_SCOPES); openid is always added", + "example": "openid profile email" + }, + "oidcEmailClaim": { + "type": "string", + "description": "Claim that carries the e-mail address (OIDC_EMAIL_CLAIM)", + "example": "email" + }, + "oidcAllowUnverifiedEmail": { + "type": "string", + "description": "Accept an e-mail address the provider marks unverified (OIDC_ALLOW_UNVERIFIED_EMAIL)", + "enum": [ + "yes", + "no" + ], + "example": "no" + }, + "oidcTokenAuthMethod": { + "type": "string", + "description": "Token endpoint auth method (OIDC_TOKEN_AUTH_METHOD); empty picks it automatically", + "enum": [ + "", + "client_secret_basic", + "client_secret_post", + "none" + ], + "example": "" + }, + "oidcButtonText": { + "type": "string", + "maxLength": 60, + "description": "Label of the login button (OIDC_BUTTON_TEXT); longer text is cut to 60 characters", + "example": "Sign in with Authentik" + }, + "oidcButtonIcon": { + "type": "string", + "description": "Login button icon (OIDC_BUTTON_ICON); empty for the built-in icon, \"none\", an https URL or a path on this server", + "example": "" + }, + "oidcButtonColor": { + "type": "string", + "description": "Login button background (OIDC_BUTTON_COLOR); hex, colour name, rgb() or hsl()", + "example": "#fd4b2d" + }, + "oidcButtonTextColor": { + "type": "string", + "description": "Login button text colour (OIDC_BUTTON_TEXT_COLOR)", + "example": "#ffffff" } } } @@ -6567,7 +6962,7 @@ "/health": { "get": { "summary": "System health check endpoint", - "description": "Reports database connectivity **and** whether the document scanner is\nactually able to work. Used by the Docker healthcheck and monitoring.\n\n`status` is `healthy` when the database is reachable and the scanner is\noperational, `degraded` when automatic processing is enabled but the\nscan loop is not armed or has failed `failureThreshold` runs in a row\n(for example because Paperless-ngx is unreachable), and `database_error`\nwhen the local database cannot be queried.\n\nA `degraded` state answers with HTTP 503 unless `HEALTHCHECK_STRICT=no`\nis configured, in which case the details are reported with HTTP 200.\nWhen automatic processing is switched off via\n`DISABLE_AUTOMATIC_PROCESSING=yes`, a missing scan loop is expected and\nnever reported as degraded.\n", + "description": "Reports database connectivity **and** whether the document scanner is\nactually able to work. Used by the Docker healthcheck and monitoring.\n\n`status` is `healthy` when the database is reachable and the scanner is\noperational, `degraded` when automatic processing is enabled but the\nscan loop is not armed or has failed `failureThreshold` runs in a row\n(for example because Paperless-ngx is unreachable), and `database_error`\nwhen the local database cannot be queried.\n\nA `degraded` state answers with HTTP 503 unless `HEALTHCHECK_STRICT=no`\nis configured, in which case the details are reported with HTTP 200.\nWhen automatic processing is switched off via\n`DISABLE_AUTOMATIC_PROCESSING=yes`, a missing scan loop is expected and\nnever reported as degraded.\n\nWithout authentication the response carries only `status` and\n`database`, with the same HTTP status codes. `message`, `scanner` and\n`paperless` hold raw error messages with internal host names and\nports, so they are returned only with a session (cookie or bearer\ntoken) or the `x-api-key` header.\n", "tags": [ "System" ], @@ -10254,6 +10649,75 @@ } } } + }, + "/auth/oidc/login": { + "get": { + "summary": "Start single sign-on with the configured OpenID Connect provider", + "description": "Redirects the browser to the identity provider's authorization endpoint\n(authorization code flow with PKCE, state and nonce). Only available when\nOIDC_ENABLED=yes and OIDC_ISSUER_URL, OIDC_CLIENT_ID and OIDC_ADMIN_EMAIL\nare set.\n", + "tags": [ + "Authentication" + ], + "responses": { + "302": { + "description": "Redirect to the identity provider" + }, + "404": { + "description": "Single sign-on is not enabled" + }, + "502": { + "description": "The identity provider could not be reached; the login page is rendered with an error" + } + } + } + }, + "/auth/oidc/callback": { + "get": { + "summary": "Complete single sign-on", + "description": "Redirect target registered with the identity provider. Exchanges the\nauthorization code, validates the ID token and signs in as the\nadministrator account when the identity's e-mail address equals\nOIDC_ADMIN_EMAIL. Sets the session cookie.\n", + "tags": [ + "Authentication" + ], + "parameters": [ + { + "in": "query", + "name": "code", + "schema": { + "type": "string" + }, + "description": "Authorization code issued by the provider" + }, + { + "in": "query", + "name": "state", + "schema": { + "type": "string" + }, + "description": "State value echoed back by the provider" + } + ], + "responses": { + "302": { + "description": "Signed in, redirected to the dashboard", + "headers": { + "Set-Cookie": { + "schema": { + "type": "string", + "description": "HTTP-only cookie containing the session JWT" + } + } + } + }, + "400": { + "description": "Missing or expired sign-in transaction, or the provider reported an error" + }, + "403": { + "description": "The identity's e-mail address does not match OIDC_ADMIN_EMAIL" + }, + "404": { + "description": "Single sign-on is not enabled" + } + } + } } }, "tags": [ diff --git a/README.md b/README.md index 6bc639b0..941db9fd 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,23 @@ Then open [http://localhost:3000](http://localhost:3000) to complete setup. - The Mistral OCR provider processes PDFs natively and ignores the `OCR_PDF_RENDER_*` settings - The OCR queue can be worked through automatically instead of pressing **Process All Pending**: enable `OCR_AUTO_PROCESS_ENABLED` (default `no`) and configure `OCR_AUTO_PROCESS_INTERVAL` (cron, default `*/15 * * * *`), `OCR_AUTO_PROCESS_BATCH_SIZE` (documents per run, default `10`) and `OCR_AUTO_ANALYZE` (run AI analysis right after OCR, default `yes`). Runs are skipped while a document scan is active or while Paperless-ngx is unreachable, so queued documents are never marked as failed because of an outage +### Single Sign-On (OIDC) + +Sign in with any OpenID Connect provider (Authentik, Authelia, Keycloak, Kanidm, Pocket ID, Zitadel, Entra ID, Google, ...) through a button on the login page. Zettelrobbe has one administrator account, and the identity whose e-mail address equals `OIDC_ADMIN_EMAIL` signs in as that account. Password login keeps working next to it. + +```yaml +environment: + OIDC_ENABLED: 'yes' + OIDC_ISSUER_URL: https://auth.example.com/application/o/zettelrobbe/ + OIDC_CLIENT_ID: zettelrobbe + OIDC_CLIENT_SECRET: ${ZETTELROBBE_OIDC_CLIENT_SECRET} + OIDC_REDIRECT_URI: https://zettelrobbe.example.com/auth/oidc/callback + OIDC_ADMIN_EMAIL: you@example.com # required + OIDC_BUTTON_TEXT: Sign in with Authentik # optional, also OIDC_BUTTON_ICON / _COLOR / _TEXT_COLOR +``` + +See [docs/sso-oidc.md](docs/sso-oidc.md) for all options, provider examples, button customization, security notes and troubleshooting. + ### Container Images | Image Tag | Size | diff --git a/config/ai-provider-presets.json b/config/ai-provider-presets.json index cd427dc0..3dae96b6 100644 --- a/config/ai-provider-presets.json +++ b/config/ai-provider-presets.json @@ -8,6 +8,14 @@ "model": "gpt-4o-mini", "tokenPlaceholder": "sk-..." }, + { + "id": "anthropic-default", + "label": "Anthropic (Claude)", + "provider": "anthropic", + "apiUrl": "", + "model": "claude-haiku-5-5", + "tokenPlaceholder": "sk-ant-..." + }, { "id": "lmstudio-local", "label": "LM Studio (OpenAI compatible)", diff --git a/config/changelog.js b/config/changelog.js index 3f6fe20c..0bf3de73 100644 --- a/config/changelog.js +++ b/config/changelog.js @@ -143,6 +143,20 @@ const RELEASES = [ 'New: Settings sections for Duplicates and Simplify tags', ], }, + { + version: 'v2026.10.01', + entries: [ + 'New: Anthropic (Claude) as AI provider. Settings and the setup wizard take an API key and list the Claude models it can use; Claude Haiku 5.5 is the default. The analysis comes back as structured output, and prompt caching keeps the repeated instructions cheap', + 'New: Claude as OCR provider. It reads PDFs and images directly and uses the API key and model of the AI provider unless OCR has its own', + 'Fix: Saving the settings no longer signs everybody out. The restart after a save created a new sign-in key, so every open session ended, and a page left open only showed vague errors such as "Model discovery failed". A session that ends now sends the page to the login screen', + 'Fix: The manual tagging page uses the current design again. Its buttons, cards, document preview and tag chips were still built on styles that no longer exist', + 'New: Single sign-on with any OpenID Connect provider (Authentik, Keycloak, Entra ID and others). Contributed by @krauseerl in #372. E-mail addresses the provider marks as unverified are refused unless OIDC_ALLOW_UNVERIFIED_EMAIL=yes', + 'New: Single sign-on can be set up on the settings page, including a "Test connection" button. Values set by the container stay locked there', + 'Improvement: The settings page shows one topic at a time, grouped in a sidebar, with a search field and a save bar that counts unsaved changes across all topics', + 'Improvement: The Claude model is picked from a dropdown filled by "Fetch Models", like every other model', + 'Security: The login no longer reveals whether a username exists, a two-factor code is accepted only once, and the health endpoint keeps scanner and Paperless-ngx details behind authentication. Contributed by @krauseerl', + ], + }, ]; const latestRelease = RELEASES[RELEASES.length - 1]; diff --git a/config/config.js b/config/config.js index 61eeb338..96c412f9 100644 --- a/config/config.js +++ b/config/config.js @@ -368,6 +368,32 @@ const getCookieSecureMode = () => { return 'auto'; }; +/* Single sign-on through OpenID Connect. Read on every call rather than + snapshotted at load, like the other auth getters above, so a runtime + override or a test can change it without reloading the module. Off unless + OIDC_ENABLED is set; password login keeps working either way. */ +const getOidcConfig = () => ({ + enabled: parseEnvBoolean(process.env.OIDC_ENABLED, 'no') === 'yes', + issuerUrl: String(process.env.OIDC_ISSUER_URL || '').trim(), + clientId: String(process.env.OIDC_CLIENT_ID || '').trim(), + clientSecret: String(process.env.OIDC_CLIENT_SECRET || ''), + // Required: the identity whose e-mail matches signs in as the admin. + adminEmail: String(process.env.OIDC_ADMIN_EMAIL || '').trim(), + redirectUri: String(process.env.OIDC_REDIRECT_URI || '').trim(), + scopes: String(process.env.OIDC_SCOPES || 'openid profile email').trim(), + emailClaim: String(process.env.OIDC_EMAIL_CLAIM || 'email').trim() || 'email', + // An address the provider marks unverified is refused unless this is set. + allowUnverifiedEmail: + parseEnvBoolean(process.env.OIDC_ALLOW_UNVERIFIED_EMAIL, 'no') === 'yes', + // client_secret_basic | client_secret_post | none; empty = automatic. + tokenAuthMethod: String(process.env.OIDC_TOKEN_AUTH_METHOD || '').trim(), + // Login button appearance. + buttonText: String(process.env.OIDC_BUTTON_TEXT || '').trim(), + buttonIcon: String(process.env.OIDC_BUTTON_ICON || '').trim(), + buttonColor: String(process.env.OIDC_BUTTON_COLOR || '').trim(), + buttonTextColor: String(process.env.OIDC_BUTTON_TEXT_COLOR || '').trim(), +}); + // Initialize limit functions with defaults const limitFunctions = { activateTagging: parseEnvBoolean(process.env.ACTIVATE_TAGGING, 'yes'), @@ -433,13 +459,14 @@ startupLog(logLevel, 'info', 'Configuration loaded:', { }); module.exports = { - PAPERLESS_AI_VERSION: 'v2026.09.03', + PAPERLESS_AI_VERSION: 'v2026.10.01', CONFIGURED: false, configSourceMode: CONFIG_SOURCE_MODE, getApiKey, getJwtSecret, getTrustProxy, getCookieSecureMode, + getOidcConfig, isProtectedRuntimeEnvKey, get apiKey() { return getApiKey(); @@ -648,6 +675,33 @@ module.exports = { deploymentName: process.env.AZURE_DEPLOYMENT_NAME || '', apiVersion: process.env.AZURE_API_VERSION || '2023-05-15', }, + // The Claude API speaks its own Messages dialect, so it is a provider of + // its own rather than a preset of `custom`. Request rules per model live in + // services/anthropicSupport.js. + anthropic: { + apiKey: process.env.ANTHROPIC_API_KEY || '', + // Empty means api.anthropic.com; set only for a proxy in front of it. + apiUrl: process.env.ANTHROPIC_API_URL || '', + model: process.env.ANTHROPIC_MODEL || 'claude-haiku-5-5', + // low | medium | high | xhigh | max; empty leaves the model's default. + effort: String(process.env.ANTHROPIC_EFFORT || '') + .trim() + .toLowerCase(), + // Completion cap per request. Thinking counts toward it, which is why it + // is separate from RESPONSE_TOKENS. + maxTokens: parseTokenCount( + process.env.ANTHROPIC_MAX_TOKENS, + 16000, + 'ANTHROPIC_MAX_TOKENS' + ), + // Constrain the analysis answer to a JSON schema (output_config.format). + structuredOutput: parseEnvBoolean( + process.env.ANTHROPIC_STRUCTURED_OUTPUT, + 'yes' + ), + // Mark the system prompt cacheable; it repeats for every document of a scan. + promptCaching: parseEnvBoolean(process.env.ANTHROPIC_PROMPT_CACHING, 'yes'), + }, mistralOcr: { enabled: parseEnvBoolean(process.env.MISTRAL_OCR_ENABLED, 'no'), provider: String( diff --git a/docker-compose.yml b/docker-compose.yml index 861893ff..0d282a99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,6 +67,15 @@ services: # Update check (optional) – asks GitHub once a day whether a newer release # exists. Set to 'no' for air-gapped installations; nothing else changes. # - UPDATE_CHECK_ENABLED=no + # Single sign-on through OpenID Connect (optional), see docs/sso-oidc.md. + # The identity whose e-mail equals OIDC_ADMIN_EMAIL signs in as the admin. + # - OIDC_ENABLED=yes + # - OIDC_ISSUER_URL=https://auth.example.com/application/o/zettelrobbe/ + # - OIDC_CLIENT_ID=your_client_id + # - OIDC_CLIENT_SECRET=your_client_secret + # - OIDC_REDIRECT_URI=https://zettelrobbe.example.com/auth/oidc/callback + # - OIDC_ADMIN_EMAIL=you@example.com + # - OIDC_BUTTON_TEXT=Sign in with SSO ports: - '3000:${PAPERLESS_AI_PORT:-3000}' volumes: diff --git a/docs/sso-oidc.md b/docs/sso-oidc.md new file mode 100644 index 00000000..8b187e59 --- /dev/null +++ b/docs/sso-oidc.md @@ -0,0 +1,244 @@ +# Single Sign-On with OpenID Connect + +Zettelrobbe can sign you in through any standards-compliant OpenID Connect (OIDC) provider, for +example Authentik, Authelia, Keycloak, Kanidm, Pocket ID, Zitadel, Microsoft Entra ID or Google. +When it is configured, the login page shows an extra button next to the password form. One click +takes you to your provider and back into Zettelrobbe. + +- [How it works](#how-it-works) +- [Configuration](#configuration) +- [Setting it up](#setting-it-up) +- [Provider examples](#provider-examples) +- [Customizing the login button](#customizing-the-login-button) +- [Security notes](#security-notes) +- [Troubleshooting](#troubleshooting) + +## How it works + +Zettelrobbe has exactly one account: the administrator created by the setup wizard. Single sign-on +does not add users, it is a second way to sign in to that one account. + +1. You click the SSO button. Zettelrobbe sends you to your provider using the standard + authorization code flow with PKCE (S256), `state` and `nonce`. +2. You sign in at the provider, which sends you back to `/auth/oidc/callback`. +3. Zettelrobbe exchanges the code for tokens and validates the ID token (signature, issuer, + audience, expiry, nonce). +4. It reads the e-mail address from the `email` claim (ID token, completed from the userinfo + endpoint) and compares it with `OIDC_ADMIN_EMAIL`, ignoring upper and lower case. +5. If they match, you are signed in as the administrator, whatever your username is at the + provider. Any other identity is refused. + +Nothing is provider-specific. Endpoints, signing keys and the supported client authentication +methods come from the provider's discovery document (`/.well-known/openid-configuration`). + +The password login keeps working next to SSO, so you can still get in when the provider is down. + +## Configuration + +SSO can be set on the settings page (**Settings › Sign-in and security › Single sign-on**) or +through the environment variables below, usually in the `environment:` section of your +`docker-compose.yml`. Both write the same `OIDC_*` values. + +- **On the settings page**, a value the container injects through the environment is shown locked + and cannot be changed there; remove it from the compose file to manage it on the page. The + client secret is never shown again after saving: leave the field empty to keep it. Saving with + SSO switched on is refused while the issuer URL, client ID, admin e-mail or redirect URI is + missing or invalid, so a broken setup cannot hide the button unnoticed. Switched off, the page + saves whatever you typed. Like every save, it restarts the application. +- **Test connection** checks the values in the form before you save: it loads the provider's + discovery document, shows the token endpoint auth method the login would use and whether the + provider offers PKCE, and checks the client ID and secret at the token endpoint (with a made-up + authorization code, so nobody is signed in). With the secret field empty the test uses the + stored secret, but only for an issuer with the same scheme, host and port as the saved one. + +Single sign-on only switches on when +`OIDC_ENABLED=yes` **and** the three required values are set. If one is missing or invalid, the +button stays hidden and the log says what is missing at startup: + +```text +[WARN] OIDC_ENABLED=yes but single sign-on stays off: OIDC_ADMIN_EMAIL is missing or not an e-mail address. +``` + +### Required + +| Variable | Example | Purpose | +| ------------------ | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `OIDC_ENABLED` | `yes` | Turns single sign-on on. Default `no` | +| `OIDC_ISSUER_URL` | `https://auth.example.com/application/o/zettelrobbe/` | The provider's issuer URL (see [Provider examples](#provider-examples)). The full discovery URL also works | +| `OIDC_CLIENT_ID` | `zettelrobbe` | Client ID of the application registered at the provider | +| `OIDC_ADMIN_EMAIL` | `you@example.com` | Your e-mail address at the provider. Only the identity with this address can sign in | + +### Usually needed + +| Variable | Default | Purpose | +| -------------------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `OIDC_CLIENT_SECRET` | – | Client secret of a confidential client. Leave empty for a public client (PKCE is always used) | +| `OIDC_REDIRECT_URI` | derived from the request | The callback URL registered at the provider: `https:///auth/oidc/callback`. Set it explicitly behind a reverse proxy | + +### Optional + +| Variable | Default | Purpose | +| ----------------------------- | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `OIDC_SCOPES` | `openid profile email` | Requested scopes. `openid` is always added. Keep `email`, it carries the address | +| `OIDC_EMAIL_CLAIM` | `email` | Claim that holds the e-mail address, for providers that put it elsewhere | +| `OIDC_ALLOW_UNVERIFIED_EMAIL` | `no` | Accept an address the provider marks `email_verified: false`. Only for providers that never confirm addresses; restrict the application at the provider then | +| `OIDC_TOKEN_AUTH_METHOD` | automatic | How the client authenticates at the token endpoint: `client_secret_basic`, `client_secret_post` or `none`. Automatic picks `none` without a secret, otherwise `client_secret_basic` unless the provider only lists `client_secret_post` | +| `OIDC_BUTTON_TEXT` | `Sign in with SSO` | Label of the login button (up to 60 characters) | +| `OIDC_BUTTON_ICON` | key icon | Icon on the button: an `https://` image URL, a path on this server such as `/favicon.png`, or `none` | +| `OIDC_BUTTON_COLOR` | theme default | Background colour of the button: `#rrggbb`, `#rgb`, a colour name, `rgb(...)` or `hsl(...)` | +| `OIDC_BUTTON_TEXT_COLOR` | theme default | Text colour of the button, same formats | + +`OIDC_CLIENT_SECRET` is treated like the other secrets: it is masked in the settings page and in +logs. + +## Setting it up + +These steps work for every provider. The [examples](#provider-examples) below fill in the +provider-specific parts. + +1. **Register an application at your provider.** + - Flow: authorization code. Client type: confidential (with a secret) or public. + - Redirect URI: `https:///auth/oidc/callback`, exactly as the browser + reaches Zettelrobbe. + - Scopes: `openid profile email`. +2. **Restrict access at the provider.** Allow only your own user (or a group with only you in it) + to use the application. See [Security notes](#security-notes). +3. **Add the variables to your compose file.** + + ```yaml + services: + zettelrobbe: + environment: + OIDC_ENABLED: 'yes' + OIDC_ISSUER_URL: https://auth.example.com/application/o/zettelrobbe/ + OIDC_CLIENT_ID: zettelrobbe + OIDC_CLIENT_SECRET: ${ZETTELROBBE_OIDC_CLIENT_SECRET} + OIDC_REDIRECT_URI: https://zettelrobbe.example.com/auth/oidc/callback + OIDC_ADMIN_EMAIL: you@example.com + # Optional: make the button look like your provider + OIDC_BUTTON_TEXT: Sign in with Authentik + OIDC_BUTTON_COLOR: '#fd4b2d' + OIDC_BUTTON_TEXT_COLOR: white + # Behind a reverse proxy + TRUST_PROXY: '1' + ``` + + Keep the secret out of the compose file by putting it in the `.env` file next to it, as shown + with `${ZETTELROBBE_OIDC_CLIENT_SECRET}`. + +4. **Recreate the container** (`docker compose up -d`) and check the log for + `[INFO] Single sign-on enabled with issuer ...`. +5. **Open the login page.** The SSO button appears below the password form. + +The setup wizard has to be finished first: SSO signs in to the administrator account, so that +account must exist. + +## Provider examples + +### Authentik + +1. **Applications → Providers → Create → OAuth2/OpenID Provider.** + - Authorization flow: the default explicit or implicit consent flow. + - Client type: _Confidential_. Note the client ID and secret. + - Redirect URIs: `Strict`, `https://zettelrobbe.example.com/auth/oidc/callback`. + - Signing key: select a certificate (for example _authentik Self-signed Certificate_). +2. **Applications → Applications → Create.** Name it, choose a slug (for example `zettelrobbe`) + and select the provider from step 1. +3. **Restrict access:** on the application, open _Policy / Group / User Bindings_ and bind your + user or a group. Without a binding, every Authentik user may use the application. +4. Issuer URL: `https:///application/o//`. The provider's overview page + shows it as _OpenID Configuration Issuer_. + +### Authelia + +Add a client to the Authelia configuration (Authelia 4.38 or later syntax): + +```yaml +identity_providers: + oidc: + clients: + - client_id: 'zettelrobbe' + client_name: 'Zettelrobbe' + client_secret: '$pbkdf2-sha512$310000$...' # hashed secret + public: false + authorization_policy: 'two_factor' + redirect_uris: + - 'https://zettelrobbe.example.com/auth/oidc/callback' + scopes: ['openid', 'profile', 'email'] + require_pkce: true + pkce_challenge_method: 'S256' + token_endpoint_auth_method: 'client_secret_basic' +``` + +Issuer URL: `https://`. Restrict the client with an access control rule or a +dedicated authorization policy. + +### Keycloak + +1. In your realm: **Clients → Create client**, type _OpenID Connect_, client ID `zettelrobbe`. +2. Turn _Client authentication_ on and keep _Standard flow_ enabled. +3. _Valid redirect URIs_: `https://zettelrobbe.example.com/auth/oidc/callback`. +4. Copy the secret from the _Credentials_ tab. + +Issuer URL: `https:///realms/`. + +### Other providers + +| Provider | `OIDC_ISSUER_URL` | Notes | +| ------------------ | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Kanidm | `https:///oauth2/openid/` | The issuer is per client | +| Pocket ID | `https://` | | +| Zitadel | `https://` | | +| Microsoft Entra ID | `https://login.microsoftonline.com//v2.0` | `email` is an optional claim in Entra ID. Add it to the token configuration, or set `OIDC_EMAIL_CLAIM=preferred_username` when the UPN is your e-mail address | +| Google | `https://accounts.google.com` | | + +Any other provider works the same way as long as it publishes a discovery document and sends the +e-mail address in the ID token or from its userinfo endpoint. + +## Customizing the login button + +With no customization the button is a neutral secondary button reading _Sign in with SSO_ with a +key icon. Four variables change it: + +```yaml +OIDC_BUTTON_TEXT: Sign in with Authentik +OIDC_BUTTON_ICON: https://auth.example.com/media/zettelrobbe-icon.svg +OIDC_BUTTON_COLOR: '#fd4b2d' +OIDC_BUTTON_TEXT_COLOR: white +``` + +- `OIDC_BUTTON_ICON` accepts an `http(s)` URL or a path on the Zettelrobbe server. `none` removes + the icon. Anything else falls back to the key icon. +- The colours accept `#rgb`, `#rrggbb` (with optional alpha), colour names, `rgb(...)` and + `hsl(...)`. Other values are ignored and the theme colour is used. Quote hex colours in YAML, + because `#` starts a comment. + +## Security notes + +- **Whoever passes the check is the administrator.** Zettelrobbe checks that the e-mail address + equals `OIDC_ADMIN_EMAIL` and refuses it when the provider marks it unverified + (`email_verified: false`, see `OIDC_ALLOW_UNVERIFIED_EMAIL`). Many providers let users change + their own e-mail address, so always restrict the application at the provider to your own account + or a group with only you in it. +- **MFA belongs to the provider.** An SSO login does not ask for Zettelrobbe's own TOTP code. + Enforce MFA at the provider instead. +- **Use HTTPS.** Plain-HTTP issuers work but log a warning. Use them only on a trusted network. +- **Logging out ends the Zettelrobbe session only.** Your session at the provider stays open, so + the next click on the SSO button signs you in again without a password prompt. Log out at the + provider as well on a shared computer. +- The sign-in transaction (state, nonce, PKCE verifier) lives in a signed, HTTP-only cookie that + expires after 10 minutes and can never be used as a session. +- The SSO endpoints share the login rate limit (`LOGIN_RATE_LIMIT_MAX`). + +## Troubleshooting + +| Symptom | Cause and fix | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No SSO button on the login page | Check the startup log for `OIDC_ENABLED=yes but single sign-on stays off: ...` and set what it names | +| _Single sign-on is unavailable: the identity provider could not be reached_ | Zettelrobbe could not load the discovery document. Check `OIDC_ISSUER_URL` and that the container can reach the provider (DNS, firewall, certificates). The log has the exact error | +| The provider reports an invalid or mismatched redirect URI | The redirect URI must match the registered one character for character, including `https://`. Set `OIDC_REDIRECT_URI` to the full URL, and behind a reverse proxy also `TRUST_PROXY` | +| _Single sign-on failed_ after returning from the provider | The token exchange failed. With `invalid_client` in the log, set `OIDC_TOKEN_AUTH_METHOD` to the method registered at the provider, and check the client secret | +| _The identity provider did not send the "email" claim_ | Add `email` to `OIDC_SCOPES` and allow that scope for the client at the provider, or point `OIDC_EMAIL_CLAIM` at the claim that holds the address | +| _This account may not sign in to Zettelrobbe_ | The e-mail address at the provider differs from `OIDC_ADMIN_EMAIL`. The log line `[FAILED LOGIN] OIDC sign-in refused for ...` shows the address the provider sent | +| _Your single sign-on session expired_ | More than 10 minutes passed at the provider, or the browser dropped the cookie. Start again from the login page. If it keeps happening behind HTTPS, check `COOKIE_SECURE_MODE` and `TRUST_PROXY` | +| _Zettelrobbe has no administrator account yet_ | Finish the setup wizard first | diff --git a/package-lock.json b/package-lock.json index 5d854c6d..2088145c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "@anthropic-ai/sdk": "^0.133.0", "axios": "^1.20.0", "bcryptjs": "^3.0.2", "better-sqlite3": "^12.11.1", @@ -23,6 +24,7 @@ "jsonwebtoken": "^9.0.3", "node-cron": "^4.6.0", "openai": "^7.23.0", + "openid-client": "^6.8.8", "qrcode": "^1.5.4", "swagger-jsdoc": "^6.3.0", "swagger-ui-express": "^5.0.1", @@ -40,6 +42,27 @@ "stylelint-config-standard": "^40.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.133.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.133.0.tgz", + "integrity": "sha512-jlqUN8p8ESTJj+GJnA4KjzbEfF1waEoGzRT8jwytlYGFFgv+/DUQOxlYGvYw10UzhD1kwLqYzxvA3FrTB2YWuQ==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "14.0.1", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-14.0.1.tgz", @@ -149,6 +172,15 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/runtime": { + "version": "7.29.10", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.10.tgz", + "integrity": "sha512-NpugTzbKgGrNe45Ui32cN8vCFFMxVtP/9rtfwv0nP5FRDAjvYBxJcLVpYA9mr1jxP6cnIT7x9LUzuDTbWYYNRA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@cacheable/memory": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.2.0.tgz", @@ -660,6 +692,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "license": "MIT" + }, "node_modules/@types/esrecurse": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", @@ -1130,9 +1168,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -1610,9 +1648,9 @@ } }, "node_modules/dotenv": { - "version": "18.0.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-18.0.3.tgz", - "integrity": "sha512-/8CJTyL817qPnfbQrJgOEcrU3fDrJkRIoXBE566wHuvIXKOCHUZhPqGPbCPi9SogjdGnreF0TlvnTgK6J/8zpA==", + "version": "18.0.7", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-18.0.7.tgz", + "integrity": "sha512-QEvMEF8ca3GNZJJnAbOgxpj5ULhBvnMcURp+XKrtvqFNPgjBvYoswQ0Jl2cO7JzCjxZjJhr9ZuIEhlrRxMft/Q==", "license": "BSD-2-Clause", "bin": { "dotenv": "dist/index.cjs" @@ -2165,6 +2203,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "license": "Unlicense" + }, "node_modules/fast-uri": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", @@ -2830,9 +2874,9 @@ "license": "ISC" }, "node_modules/ip-address": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", - "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", + "version": "10.7.4", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.4.tgz", + "integrity": "sha512-DWBMKyeM25D70nrIiTkgQth59/G+naKOTep6huF9j+GmeK3DMD60Zvxg5d5qQnz+38AHgou6ean2LECX+lrKDg==", "license": "MIT", "engines": { "node": ">= 12" @@ -2949,6 +2993,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -2999,6 +3052,19 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -3441,6 +3507,15 @@ "node": ">=0.10.0" } }, + "node_modules/oauth4webapi": { + "version": "3.8.8", + "resolved": "https://registry.npmjs.org/oauth4webapi/-/oauth4webapi-3.8.8.tgz", + "integrity": "sha512-8N28E+a/oxfXWBgOMt+ZP/JUf/XR+IFbvkAEPP3gznXOMv9BpAAwiIj0TFNz3tGTPc0ZQ8zmWBNgN1nAys0gng==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -3534,6 +3609,19 @@ "license": "MIT", "peer": true }, + "node_modules/openid-client": { + "version": "6.8.8", + "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-6.8.8.tgz", + "integrity": "sha512-ZsucJA5Ad04Uv7YN4ql+s4GXNmb9uAYQwyJTsJx7CH/MX3JZioLE2pVKi1SC+YrbSLA4Px3Gi30dMjgOZtb6pA==", + "license": "MIT", + "dependencies": { + "jose": "^6.2.12", + "oauth4webapi": "^3.8.8" + }, + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -3800,9 +3888,9 @@ } }, "node_modules/postcss-selector-parser": { - "version": "7.1.5", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.5.tgz", - "integrity": "sha512-KvvtD7SrlBP7dlgkBghEE3r84CABm5SmV2aNcG4oCA+qDnJ/tvKonFVvwWAyyWUEwxuNawdfEAZKP9zM3oZ2Uw==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.6.tgz", + "integrity": "sha512-7qASPzhKF2l2KLboRZux8CCTRMdGiV08vWmyKzPz22qZ7ZjQBOeY7rNzNoCLSUiftJ7HUq0GERHmxw/t0dCdMw==", "dev": true, "license": "MIT", "dependencies": { @@ -3874,9 +3962,9 @@ } }, "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", "license": "MIT", "dependencies": { "forwarded": "0.2.0", @@ -3884,6 +3972,10 @@ }, "engines": { "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/proxy-from-env": { @@ -4646,9 +4738,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -4680,6 +4772,16 @@ "dev": true, "license": "CC0-1.0" }, + "node_modules/standardwebhooks": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.1.1.tgz", + "integrity": "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==", + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -4732,9 +4834,9 @@ } }, "node_modules/stylelint": { - "version": "17.15.0", - "resolved": "https://registry.npmjs.org/stylelint/-/stylelint-17.15.0.tgz", - "integrity": "sha512-mWIkesYQvQjf4Kvdeu9ns0IL8/K/wtjaGxPqsPd6DlLZvaQxGysJsocxUDrBcyHQ5VleAk4uSMat4yMrVED7PA==", + "version": "17.16.0", + "resolved": "https://registry.npmjs.org/stylelint/-/stylelint-17.16.0.tgz", + "integrity": "sha512-ejlWnbJ13x1GU32+zZXa6sYQJsKKkwSdv2B4rAdO3OWdEmvhSJKcXi5pOphQYSzyaj1kXpuGJr7pcMBNfDIuPw==", "dev": true, "funding": [ { @@ -5184,6 +5286,12 @@ "nodetouch": "bin/nodetouch.js" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", diff --git a/package.json b/package.json index e92ab614..71e18f07 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "test:setup-remote-guard": "node scripts/run-tests.js --test setup-remote-guard", "test:area:security": "node scripts/run-tests.js --area security", "test:area:duplicates": "node scripts/run-tests.js --area duplicates", + "test:area:anthropic": "node scripts/run-tests.js --area anthropic", "test:restriction-service": "node scripts/run-tests.js --test restriction-service", "test:updated-service": "node scripts/run-tests.js --test updated-service", "test:list": "node scripts/run-tests.js --list", @@ -57,6 +58,7 @@ "author": "admonstrator", "license": "MIT", "dependencies": { + "@anthropic-ai/sdk": "^0.133.0", "axios": "^1.20.0", "bcryptjs": "^3.0.2", "better-sqlite3": "^12.11.1", @@ -71,6 +73,7 @@ "jsonwebtoken": "^9.0.3", "node-cron": "^4.6.0", "openai": "^7.23.0", + "openid-client": "^6.8.8", "qrcode": "^1.5.4", "swagger-jsdoc": "^6.3.0", "swagger-ui-express": "^5.0.1", diff --git a/public/css/badges.css b/public/css/badges.css index b6925366..bc875958 100644 --- a/public/css/badges.css +++ b/public/css/badges.css @@ -159,4 +159,10 @@ opacity: 1; background: color-mix(in srgb, var(--zr-brand) 18%, transparent); } + /* The default 16px icon is wider than its 15px button and stuck out past + the chip's rounded end. */ + .zr-chip button .zr-icon { + width: 11px; + height: 11px; + } } diff --git a/public/css/pages/auth.css b/public/css/pages/auth.css index ecd63c6b..97e8a6d1 100644 --- a/public/css/pages/auth.css +++ b/public/css/pages/auth.css @@ -108,4 +108,39 @@ .login-back-button { align-self: center; } + + /* login: "or" rule between the password form and the single sign-on + button. */ + .login-sso__divider { + display: flex; + align-items: center; + gap: var(--zr-3); + color: var(--zr-text-faint); + } + .login-sso__divider::before, + .login-sso__divider::after { + content: ''; + flex: 1 1 auto; + height: 1px; + background: var(--zr-line); + } + + /* login: the single sign-on button. OIDC_BUTTON_COLOR and + OIDC_BUTTON_TEXT_COLOR arrive as these two custom properties on the + element; without them it is a regular secondary button. */ + .login-sso__button { + background: var(--login-sso-bg, var(--zr-surface)); + border-color: var(--login-sso-bg, var(--zr-line-strong)); + color: var(--login-sso-fg, var(--zr-text)); + } + .login-sso__button:hover { + background: var(--login-sso-bg, var(--zr-surface-2)); + border-color: var(--login-sso-bg, var(--zr-text-faint)); + filter: brightness(0.95); + } + .login-sso__icon { + width: 18px; + height: 18px; + object-fit: contain; + } } diff --git a/public/css/pages/settings.css b/public/css/pages/settings.css index 854f9ea8..6a3150ef 100644 --- a/public/css/pages/settings.css +++ b/public/css/pages/settings.css @@ -23,12 +23,9 @@ /* --- settings ------------------------------------------------------------ */ - /* The section nav navigates with plain #hash links, and the browser scrolls the - target flush to the top of the viewport — where the sticky topbar sits on top - of it. Every landing target gets the topbar plus the gap the section nav - itself keeps below it, so a section lands where the nav says it is. - js/modules/section-nav.js approximates the same offset for the scroll-spy. */ - .zr-settings :is([id$='-tab'], .zr-module[id^='sec-']) { + /* An old link to a section (#sec-provider) opens its topic and scrolls the + section into view; it lands below the sticky topbar, not under it. */ + .zr-settings .zr-module[id^='sec-'] { scroll-margin-top: calc(var(--zr-topbar-h) + var(--zr-4)); } @@ -38,7 +35,7 @@ band — drawn the way .zr-table draws them, from card edge to card edge, which the body's own padding has to be undone for. - Scoped to a card's own body: #changelog-tab uses .zr-module__body as a bare + Scoped to a card's own body: the changelog uses .zr-module__body as a bare layout wrapper outside any .zr-module, where the release timeline brings its own rhythm and there is no card box for a full-bleed band to bleed into. */ .zr-settings .zr-module > .zr-module__body.zr-col--loose { @@ -122,4 +119,228 @@ display: none; } } + + /* --- one topic per view (js/modules/settings-nav.js) ------------------- */ + + /* The topic list is wider than the old section list: the entries are names + ("Schedule and tag filters"), not single words. */ + .zr-settings { + grid-template-columns: 220px minmax(0, 1fr); + } + + .zr-settings-nav { + gap: var(--zr-3); + } + .zr-settings-nav__group { + display: flex; + flex-direction: column; + gap: 1px; + } + .zr-settings-nav__label { + display: flex; + align-items: center; + gap: var(--zr-2); + padding: 0 var(--zr-2) var(--zr-1); + font-size: var(--zr-fs-xs); + font-weight: 600; + letter-spacing: 0.05em; + text-transform: uppercase; + color: var(--zr-text-faint); + } + .zr-settings-nav a { + font-weight: 500; + } + /* A topic with edits that are not saved yet, wherever they are. */ + .zr-settings-nav__dirty { + flex: none; + width: 7px; + height: 7px; + border-radius: var(--zr-r-full); + background: var(--zr-brand); + } + /* While searching, topics without a match step back but stay clickable. */ + .zr-settings-nav__miss { + opacity: 0.4; + } + + .zr-settings-picker .zr-input { + width: 100%; + } + + /* The search sits above both columns; its results float over the page. */ + .zr-settings-search__box { + position: relative; + flex: 1 1 320px; + max-width: 520px; + } + .zr-settings-search__box > .zr-icon { + position: absolute; + left: 10px; + top: 50%; + transform: translateY(-50%); + color: var(--zr-text-faint); + pointer-events: none; + } + .zr-settings-search__box > .zr-input { + width: 100%; + padding-left: 32px; + } + .zr-settings-search__results { + position: absolute; + z-index: 30; + top: calc(100% + var(--zr-1)); + left: 0; + right: 0; + max-height: min(420px, 60dvh); + overflow-y: auto; + padding: var(--zr-1); + background: var(--zr-surface); + border: 1px solid var(--zr-line); + border-radius: var(--zr-r-lg); + box-shadow: var(--zr-shadow-lg); + } + .zr-settings-search__item { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 0 var(--zr-2); + width: 100%; + padding: var(--zr-2); + border: 0; + border-radius: var(--zr-r-md); + background: transparent; + color: var(--zr-text); + text-align: left; + cursor: pointer; + } + .zr-settings-search__item:hover, + .zr-settings-search__item[aria-selected='true'] { + background: var(--zr-brand-soft); + } + .zr-settings-search__name { + font-weight: 600; + } + .zr-settings-search__path { + grid-column: 1; + font-size: var(--zr-fs-xs); + color: var(--zr-text-muted); + } + .zr-settings-search__key { + grid-column: 2; + grid-row: 1 / span 2; + align-self: center; + font-family: var(--zr-font-mono); + font-size: var(--zr-fs-xs); + color: var(--zr-text-muted); + } + .zr-settings-search__empty { + padding: var(--zr-2); + } + + /* The topic's own head: group label, then the topic as the page heading. */ + .zr-settings-entry__head { + display: flex; + flex-direction: column; + gap: 2px; + } + .zr-settings-entry__titlerow { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: var(--zr-2); + } + .zr-settings-entry__title { + font-family: var(--zr-font-brand); + font-size: var(--zr-fs-xl); + font-weight: 700; + letter-spacing: -0.01em; + } + /* "On, but incomplete: ..." can be a sentence; it wraps instead of + running out of the column. */ + .zr-settings-entry__status { + white-space: normal; + max-width: 100%; + } + + /* The variable a field writes, named at the start of its hint. */ + .zr-envkey { + font-family: var(--zr-font-mono); + font-size: 0.95em; + color: var(--zr-text-muted); + } + + /* The row a search result or an error opened, for a moment. */ + .zr-settings-flash { + animation: zr-settings-flash 1.6s var(--zr-ease); + } + @keyframes zr-settings-flash { + 0%, + 40% { + background: var(--zr-brand-soft); + } + 100% { + background: transparent; + } + } + + /* --- single sign-on ---------------------------------------------------- */ + .zr-sso-preview { + display: flex; + flex-direction: column; + gap: var(--zr-2); + } + /* A slice of the login card, on the login page's own canvas. */ + .zr-sso-preview__card { + max-width: 360px; + padding: var(--zr-4); + border: 1px solid var(--zr-line); + border-radius: var(--zr-r-lg); + background: var(--zr-canvas); + } + .zr-sso-preview__card .login-sso__button { + pointer-events: none; + } + .zr-sso-swatch { + flex: none; + align-self: stretch; + width: 30px; + border: 1px solid var(--zr-line-strong); + border-radius: var(--zr-r-md); + background: repeating-conic-gradient( + var(--zr-surface-2) 0% 25%, + var(--zr-surface) 0% 50% + ) + 50% / 10px 10px; + } + .zr-sso-swatch[data-state='invalid'] { + border-color: var(--zr-danger); + } + .zr-input[aria-invalid='true'] { + border-color: var(--zr-danger); + } + + .zr-sso-advanced > summary { + cursor: pointer; + list-style: none; + } + .zr-sso-advanced > summary::-webkit-details-marker { + display: none; + } + .zr-sso-advanced:not([open]) > summary { + border-bottom: 0; + } + .zr-sso-advanced[open] > summary > .zr-icon { + transform: rotate(180deg); + } + + @media (max-width: 860px) { + .zr-settings { + grid-template-columns: minmax(0, 1fr); + } + .zr-settings-search__box { + max-width: none; + } + .zr-settings-entry__title { + font-size: var(--zr-fs-lg); + } + } } diff --git a/public/css/pages/shared.css b/public/css/pages/shared.css index 94fa05a4..0aea7d3a 100644 --- a/public/css/pages/shared.css +++ b/public/css/pages/shared.css @@ -1,5 +1,5 @@ /* ========================================================================== - Cross-page legacy pieces: the pre-framework modal system, .stat-box, + Cross-page legacy pieces: the pre-framework modal system, the manual document search, the queue toolbar and the odds and ends the page scripts of history, settings and manual generate at runtime. Not one page — several, which is why they are not in any @@ -163,23 +163,37 @@ transition: filter 0.3s ease; } - /* Stats Box */ - .stat-box { - background: var(--zr-surface-2); - padding: 1rem; - border-radius: 8px; - border: 1px solid var(--zr-line); - transition: all 0.2s ease; - color: var(--zr-text); - } - /* The manual preview claims its height up front, so picking a document does - not make the rest of the page jump. Frame and padding come from .stat-box. */ - /* Tall enough to balance the analysis column once content loads; while empty + not make the rest of the page jump. Frame and padding come from .zr-panel. + Tall enough to balance the analysis column once content loads; while empty the #previewEmpty hint fills it instead of a bare grey area. */ .manual-preview { min-height: 360px; } + /* Document text keeps its own line breaks and scrolls inside the panel: a + long document used to run together into one paragraph and stretch the + page to its full length. */ + .manual-preview #contentPreview { + max-height: 600px; + overflow-y: auto; + white-space: pre-wrap; + overflow-wrap: anywhere; + font-size: var(--zr-fs-sm); + } + + /* An empty tag list says so instead of leaving its label over nothing. */ + .manual-tags:empty::before { + content: attr(data-empty); + font-size: var(--zr-fs-sm); + color: var(--zr-text-faint); + } + /* A suggestion is not on the document yet: outlined rather than filled. The + padding gives back the border's pixel so both chips stand equally tall. */ + .manual-chip--suggested { + padding: 1px 3px 1px 7px; + background: transparent; + border: 1px dashed color-mix(in srgb, var(--zr-brand) 55%, transparent); + } /* The empty hint shows only while the preview has no content. */ .manual-preview #contentPreview:not(:empty) + #previewEmpty { display: none; @@ -190,12 +204,9 @@ .manual-preview { min-height: 160px; } - } - - .stat-box:hover { - transform: translateY(-2px); - box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1); - background: var(--zr-surface-2); + .manual-preview #contentPreview { + max-height: 320px; + } } /* Modal Styles */ diff --git a/public/js/bug-report-link.js b/public/js/bug-report-link.js index d1d905ac..476d7890 100644 --- a/public/js/bug-report-link.js +++ b/public/js/bug-report-link.js @@ -78,6 +78,8 @@ `AZURE_ENDPOINT=${supportInfo.azureEndpoint || 'unknown'}`, `AZURE_DEPLOYMENT_NAME=${supportInfo.azureDeploymentName || 'unknown'}`, `AZURE_API_VERSION=${supportInfo.azureApiVersion || 'unknown'}`, + `ANTHROPIC_MODEL=${supportInfo.anthropicModel || 'unknown'}`, + `ANTHROPIC_API_URL=${supportInfo.anthropicApiUrl || 'unknown'}`, `MISTRAL_OCR_MODEL=${supportInfo.mistralOcrModel || 'unknown'}`, `TRUST_PROXY=${supportInfo.trustProxy || 'unknown'}`, `USE_EXISTING_DATA=${supportInfo.useExistingData || 'no'}`, @@ -88,6 +90,7 @@ `OPENAI_API_KEY=${keyState(supportInfo.openAiKeySet)}`, `CUSTOM_API_KEY=${keyState(supportInfo.customKeySet)}`, `AZURE_API_KEY=${keyState(supportInfo.azureKeySet)}`, + `ANTHROPIC_API_KEY=${keyState(supportInfo.anthropicKeySet)}`, `MISTRAL_API_KEY=${keyState(supportInfo.mistralKeySet)}`, `API_KEY=${keyState(supportInfo.apiKeySet)}`, ].join('\n') diff --git a/public/js/duplicates.js b/public/js/duplicates.js index b216973b..6bd0d1be 100644 --- a/public/js/duplicates.js +++ b/public/js/duplicates.js @@ -2125,7 +2125,7 @@ async function reattachReview() { */ function htmlReviewFailure(message) { const text = String(message == null ? '' : message); - return `
${htmlIcons.danger}
Run failed

${esc(text)}

Duplicates settings

`; + return `
${htmlIcons.danger}
Run failed

${esc(text)}

Duplicates settings

`; } /** What a review has to admit about itself, or '' when it went through. */ diff --git a/public/js/failed.js b/public/js/failed.js index ea797579..150dda26 100644 --- a/public/js/failed.js +++ b/public/js/failed.js @@ -147,6 +147,9 @@ 'Insufficient content (< 10 chars)', ai_response_truncated: 'AI answer cut off (token limit)', + // A Claude model declined the request (stop_reason "refusal"). + ai_refused: + 'Declined by the AI model', }; if (map[reason]) return map[reason]; diff --git a/public/js/modules/settings-form.js b/public/js/modules/settings-form.js index 12c4f767..abf080ff 100644 --- a/public/js/modules/settings-form.js +++ b/public/js/modules/settings-form.js @@ -35,7 +35,9 @@ export default function settingsForm(root, { confirmDialog }) { // A field needs a restart when its own hint says so — the hints are the single // source of truth for that in this form. function needsRestart(field) { - const scope = field.closest('.zr-field, .zr-formgrid, .zr-row'); + const scope = field.closest( + '.zr-field, .zr-switchrow, .zr-formgrid, .zr-row' + ); return Boolean(scope && /restart required/i.test(scope.textContent || '')); } @@ -54,6 +56,12 @@ export default function settingsForm(root, { confirmDialog }) { if (!row) return; row.toggleAttribute('data-dirty', dirty.includes(field)); }); + + // The topic list marks the topics holding these, since only one topic is + // visible at a time (modules/settings-nav.js). + document.dispatchEvent( + new CustomEvent('zr:settings-dirty', { detail: { fields: dirty } }) + ); } form.addEventListener('input', sync); @@ -76,14 +84,22 @@ export default function settingsForm(root, { confirmDialog }) { sync(); }); - // settings.js handles the request; once it succeeds the current values become - // the new baseline. - form.addEventListener('submit', () => { - setTimeout(() => { - fields.forEach((field) => initial.set(field, valueOf(field))); - sync(); - }, 0); + // settings.js handles the request and announces a successful save; only then + // do the current values become the new baseline. A failed save keeps them + // counted as unsaved. + form.addEventListener('zr:settings-saved', () => { + fields.forEach((field) => initial.set(field, valueOf(field))); + sync(); }); + // Edits in any topic, visible or not, would be lost by leaving the page. + const warnOnLeave = (event) => { + if (!dirtyFields().length) return; + event.preventDefault(); + // Older browsers only show the prompt when returnValue is set. + event.returnValue = ''; + }; + window.addEventListener('beforeunload', warnOnLeave); + sync(); } diff --git a/public/js/modules/settings-nav.js b/public/js/modules/settings-nav.js new file mode 100644 index 00000000..1e390259 --- /dev/null +++ b/public/js/modules/settings-nav.js @@ -0,0 +1,486 @@ +/** + * Settings: one topic per view. + * + * Every entry (.zr-settings-entry) stays inside the one settings form; this + * module only decides which of them is visible. Hidden entries keep their + * inputs, so saving still sends every field. The current entry lives in the + * URL hash (#sso, #ai-provider), which makes reload and Back work, and the + * hashes the page used before the regrouping still land on the right entry. + * + * It also owns the settings search, the unsaved-change dots in the topic list + * (fed by modules/settings-form.js) and switching to the entry of a field the + * save names in an error or the browser refuses as invalid. + */ + +// Hashes and section ids from before the regrouping. Section ids (#sec-...) +// need no entry here: any element id inside an entry resolves to that entry. +const LEGACY_HASHES = { + 'system-tab': 'connection', + 'mfa-tab': 'account', + 'ai-tab': 'ai-provider', + 'duplicates-tab': 'duplicates', + 'simplify-tab': 'simplify', + 'ocr-tab': 'ocr', + 'troubleshooting-tab': 'caching', + 'env-tab': 'environment', + 'changelog-tab': 'changelog', +}; + +const MAX_RESULTS = 12; +const ROW_SELECTOR = '.zr-field, .zr-switchrow, .zr-checktile'; +const CONTROL_SELECTOR = + 'input:not([type="hidden"]), select, textarea, button, [tabindex]'; + +const norm = (text) => + String(text || '') + .replace(/\s+/g, ' ') + .trim(); + +export default function settingsNav(root) { + const entries = [...root.querySelectorAll('.zr-settings-entry')]; + if (!entries.length) return undefined; + + const byId = new Map(entries.map((entry) => [entry.dataset.entry, entry])); + const links = [...root.querySelectorAll('[data-entry-link]')]; + const select = root.querySelector('[data-el="select"]'); + const search = root.querySelector('[data-el="search"]'); + const results = root.querySelector('[data-el="results"]'); + const form = root.querySelector('#setupForm'); + const firstId = entries[0].dataset.entry; + let current = null; + + /* --- navigation -------------------------------------------------------- */ + + /** + * Maps a hash onto an entry and, for a hash that named something inside + * one (a section id from an old link), the element to scroll to. + */ + function resolve(hash) { + const key = decodeURIComponent(String(hash || '').replace(/^#/, '')); + if (!key) return { id: firstId, target: null }; + if (byId.has(key)) return { id: key, target: null }; + if (LEGACY_HASHES[key]) return { id: LEGACY_HASHES[key], target: null }; + const element = document.getElementById(key); + const entry = element && element.closest('.zr-settings-entry'); + if (entry) { + return { + id: entry.dataset.entry, + target: element === entry ? null : element, + }; + } + return { id: firstId, target: null }; + } + + function show(id, { scroll = true } = {}) { + const entry = byId.get(id) || byId.get(firstId); + const changed = entry !== current; + entries.forEach((item) => item.classList.toggle('hidden', item !== entry)); + links.forEach((link) => + link.setAttribute( + 'aria-current', + link.dataset.entryLink === entry.dataset.entry ? 'page' : 'false' + ) + ); + if (select && select.value !== entry.dataset.entry) { + select.value = entry.dataset.entry; + } + current = entry; + if (changed && scroll) { + // A new topic starts at its top, not at the scroll position the last one + // was left at. + window.scrollTo({ top: 0, behavior: 'auto' }); + } + return entry; + } + + function hashFor(id) { + return id === firstId && !location.hash ? '' : `#${id}`; + } + + /** Navigates like a link would: one history step per topic. */ + function go(id) { + if (!byId.has(id)) return; + if (location.hash !== `#${id}`) { + history.pushState(null, '', `#${id}`); + } + show(id); + } + + function syncFromLocation() { + const { id, target } = resolve(location.hash); + show(id, { scroll: !target }); + // An old hash is rewritten to the new one, so the address bar and any + // bookmark made from it name the topic as the list does. + const key = location.hash.replace(/^#/, ''); + if (key && key !== id && !target) { + history.replaceState(null, '', hashFor(id) || location.pathname); + } + if (target) reveal(target, { focus: false }); + } + + window.addEventListener('hashchange', syncFromLocation); + window.addEventListener('popstate', syncFromLocation); + + if (select) { + select.addEventListener('change', () => go(select.value)); + } + + /** + * Opens the entry an element sits in and brings the element into view. + * Optional sub-blocks the element sits in (a provider's fields while another + * provider is selected) are not forced open: the nearest visible ancestor is + * shown instead. + */ + function reveal(element, { focus = true } = {}) { + const entry = element.closest('.zr-settings-entry'); + if (!entry) return; + if (entry !== current) { + if (location.hash !== `#${entry.dataset.entry}`) { + history.pushState(null, '', `#${entry.dataset.entry}`); + } + show(entry.dataset.entry, { scroll: false }); + } + const advanced = element.closest('details'); + if (advanced) advanced.open = true; + + let visible = element; + while (visible && visible !== entry && visible.closest('.hidden')) { + visible = visible.parentElement; + } + const row = (visible || entry).closest(ROW_SELECTOR) || visible || entry; + row.scrollIntoView({ block: 'center', behavior: 'auto' }); + flash(row); + + if (focus) { + const control = controlOf(element); + if (control && !control.closest('.hidden') && !control.disabled) { + control.focus({ preventScroll: true }); + } + } + } + + function flash(row) { + row.classList.remove('zr-settings-flash'); + // Restart the animation when the same row is picked twice. + void row.offsetWidth; + row.classList.add('zr-settings-flash'); + setTimeout(() => row.classList.remove('zr-settings-flash'), 1600); + } + + /** The control a person can actually focus for a field or a row. */ + function controlOf(element) { + if (element.matches('input[type="hidden"]')) { + return root.querySelector(`[data-switch-target="${element.id}"]`) || null; + } + if (element.matches(CONTROL_SELECTOR)) return element; + return element.querySelector(CONTROL_SELECTOR); + } + + /* --- unsaved-change dots ------------------------------------------------ */ + + document.addEventListener('zr:settings-dirty', (event) => { + const fields = (event.detail && event.detail.fields) || []; + const dirtyEntries = new Set( + fields + .map((field) => field.closest('.zr-settings-entry')) + .filter(Boolean) + .map((entry) => entry.dataset.entry) + ); + root.querySelectorAll('[data-entry-dirty]').forEach((dot) => { + dot.classList.toggle('hidden', !dirtyEntries.has(dot.dataset.entryDirty)); + }); + }); + + /* --- fields the save or the browser refuses ------------------------------ */ + + if (form) { + // A required field in a hidden entry cannot show the browser's message, + // and the submit would fail without a word. Opening its entry first lets + // the browser point at it. + form.addEventListener( + 'invalid', + (event) => { + const field = event.target; + if (field && field.closest('.zr-settings-entry') !== current) { + reveal(field, { focus: false }); + } + }, + true + ); + } + + document.addEventListener('zr:settings-save-error', (event) => { + const message = String((event.detail && event.detail.message) || ''); + const field = fieldNamedIn(message); + if (field) reveal(field); + }); + + /** + * The field an error message names: by its env key (OIDC_ISSUER_URL), by + * the body key the form sends (oidcIssuerUrl), or by its label. + */ + function fieldNamedIn(message) { + if (!message || !form) return null; + const text = message.toLowerCase(); + const named = [...form.querySelectorAll('[name], [data-env-key]')]; + + const byEnvKey = named + .filter((field) => field.dataset.envKey) + .sort((a, b) => b.dataset.envKey.length - a.dataset.envKey.length) + .find((field) => + new RegExp(`\\b${field.dataset.envKey}\\b`).test(message) + ); + if (byEnvKey) return byEnvKey; + + const byName = named.find( + (field) => field.name && new RegExp(`\\b${field.name}\\b`).test(message) + ); + if (byName) return byName; + + let best = null; + let bestLength = 0; + indexRows().forEach((item) => { + const label = item.label.toLowerCase(); + if ( + label.length >= 6 && + label.length > bestLength && + text.includes(label) + ) { + best = item.row; + bestLength = label.length; + } + }); + return best; + } + + /* --- search ------------------------------------------------------------- */ + + /** + * Every searchable row, built on demand: settings.js adds the env keys after + * this module mounts, and rows can appear later (custom fields). + */ + function indexRows() { + const items = []; + entries.forEach((entry) => { + entry.querySelectorAll(ROW_SELECTOR).forEach((row) => { + // A field that wraps other fields (the OCR block) is a container, not + // a setting of its own. + if (row.matches('.zr-field') && row.querySelector(ROW_SELECTOR)) return; + const labelEl = row.querySelector('label, .zr-field__name') || row; + const label = norm( + [...labelEl.childNodes] + .filter( + (node) => + node.nodeType === Node.TEXT_NODE || + !node.matches?.('.zr-badge, button') + ) + .map((node) => node.textContent) + .join(' ') + ); + if (!label) return; + const hint = norm( + [...row.querySelectorAll('.zr-field__hint')] + .map((p) => p.textContent) + .join(' ') + ); + const envKeys = [ + ...new Set( + [...row.querySelectorAll('[data-env-key]')] + .map((field) => field.dataset.envKey) + .concat(hint.match(/\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b/g) || []) + ), + ]; + items.push({ entry, row, label, hint, envKeys }); + }); + }); + return items; + } + + function matches(query) { + const words = norm(query).toLowerCase().split(' ').filter(Boolean); + if (!words.length) return []; + const scored = []; + + entries.forEach((entry) => { + const title = + `${entry.dataset.title} ${entry.dataset.group}`.toLowerCase(); + if (words.every((word) => title.includes(word))) { + scored.push({ kind: 'entry', entry, score: 3 }); + } + }); + + indexRows().forEach((item) => { + const label = item.label.toLowerCase(); + const keys = item.envKeys.join(' ').toLowerCase(); + // SCAN_INTERVAL is found by "scan interval" as well as by its own name. + const keysSpaced = keys.replace(/_/g, ' '); + const haystack = `${label} ${item.hint.toLowerCase()} ${keys} ${keysSpaced}`; + if (!words.every((word) => haystack.includes(word))) return; + const inLabel = words.every((word) => label.includes(word)); + const inKey = words.every( + (word) => keys.includes(word) || keysSpaced.includes(word) + ); + scored.push({ + kind: 'field', + ...item, + score: inLabel ? 2 : inKey ? 1.5 : 1, + }); + }); + + return scored.sort((a, b) => b.score - a.score).slice(0, MAX_RESULTS); + } + + let active = -1; + let shown = []; + + function closeResults() { + if (!results) return; + results.classList.add('hidden'); + search.setAttribute('aria-expanded', 'false'); + search.removeAttribute('aria-activedescendant'); + active = -1; + } + + function renderResults() { + const query = search.value; + shown = matches(query); + filterNav(query); + results.replaceChildren(); + if (!norm(query)) { + closeResults(); + return; + } + + if (!shown.length) { + const empty = document.createElement('p'); + empty.className = 'zr-settings-search__empty zr-sm zr-muted'; + empty.textContent = + 'No setting matches. Try a word from its label or its environment variable.'; + results.append(empty); + } + + shown.forEach((item, index) => { + const option = document.createElement('button'); + option.type = 'button'; + option.className = 'zr-settings-search__item'; + option.id = `settingsSearchOption${index}`; + option.setAttribute('role', 'option'); + option.dataset.index = String(index); + + const name = document.createElement('span'); + name.className = 'zr-settings-search__name'; + name.textContent = + item.kind === 'entry' ? item.entry.dataset.title : item.label; + const path = document.createElement('span'); + path.className = 'zr-settings-search__path'; + path.textContent = + item.kind === 'entry' + ? item.entry.dataset.group + : `${item.entry.dataset.group} › ${item.entry.dataset.title}`; + option.append(name, path); + + if (item.kind === 'field' && item.envKeys.length) { + const key = document.createElement('code'); + key.className = 'zr-settings-search__key'; + key.textContent = item.envKeys[0]; + option.append(key); + } + results.append(option); + }); + + results.classList.remove('hidden'); + search.setAttribute('aria-expanded', 'true'); + setActive(shown.length ? 0 : -1); + } + + function setActive(index) { + active = index; + [...results.querySelectorAll('[role="option"]')].forEach((option, i) => + option.setAttribute('aria-selected', i === index ? 'true' : 'false') + ); + if (index >= 0) { + search.setAttribute( + 'aria-activedescendant', + `settingsSearchOption${index}` + ); + results + .querySelector(`#settingsSearchOption${index}`) + ?.scrollIntoView({ block: 'nearest' }); + } else { + search.removeAttribute('aria-activedescendant'); + } + } + + function pick(index) { + const item = shown[index]; + if (!item) return; + closeResults(); + if (item.kind === 'entry') { + go(item.entry.dataset.entry); + return; + } + reveal(item.row); + } + + /** Dims the topics that hold nothing matching the query. */ + function filterNav(query) { + const hasQuery = Boolean(norm(query)); + const hit = new Set(shown.map((item) => item.entry.dataset.entry)); + links.forEach((link) => + link.classList.toggle( + 'zr-settings-nav__miss', + hasQuery && !hit.has(link.dataset.entryLink) + ) + ); + } + + if (search && results) { + search.addEventListener('input', renderResults); + search.addEventListener('focus', () => { + if (norm(search.value)) renderResults(); + }); + search.addEventListener('keydown', (event) => { + const count = shown.length; + if (event.key === 'ArrowDown' && count) { + event.preventDefault(); + if (results.classList.contains('hidden')) renderResults(); + setActive((active + 1) % count); + } else if (event.key === 'ArrowUp' && count) { + event.preventDefault(); + setActive((active - 1 + count) % count); + } else if (event.key === 'Enter') { + // Enter in the search field must never submit anything. + event.preventDefault(); + if (active >= 0) pick(active); + } else if (event.key === 'Escape') { + if (!results.classList.contains('hidden')) { + event.preventDefault(); + closeResults(); + } else if (search.value) { + search.value = ''; + renderResults(); + } + } + }); + results.addEventListener('mousedown', (event) => { + // Keeps the focus in the field until the click has picked something. + event.preventDefault(); + }); + results.addEventListener('click', (event) => { + const option = event.target.closest('[role="option"]'); + if (option) pick(Number(option.dataset.index)); + }); + document.addEventListener('click', (event) => { + if (!event.target.closest('.zr-settings-search')) closeResults(); + }); + } + + syncFromLocation(); + + return { + destroy() { + window.removeEventListener('hashchange', syncFromLocation); + window.removeEventListener('popstate', syncFromLocation); + }, + }; +} diff --git a/public/js/modules/settings-sso.js b/public/js/modules/settings-sso.js new file mode 100644 index 00000000..fe53f23e --- /dev/null +++ b/public/js/modules/settings-sso.js @@ -0,0 +1,271 @@ +/** + * Settings › Single sign-on: the connection test, copying the redirect URI and + * the live preview of the login button. + * + * The fields themselves are ordinary members of the settings form and are + * saved with it; nothing here writes configuration. + */ + +const DEFAULT_BUTTON_TEXT = 'Sign in with SSO'; +const CALLBACK_PATH = '/auth/oidc/callback'; + +/** The colour syntax services/oidcService.js accepts for the button. */ +export function isSafeCssColor(value) { + const text = String(value || '').trim(); + return ( + /^#(?:[0-9a-f]{3,4}|[0-9a-f]{6}|[0-9a-f]{8})$/i.test(text) || + /^[a-z]{3,30}$/i.test(text) || + /^(?:rgb|rgba|hsl|hsla)\([0-9a-z.,%/\s]{1,60}\)$/i.test(text) + ); +} + +/** Mirrors resolveButtonIcon() in services/oidcService.js. */ +export function resolveButtonIcon(value) { + const text = String(value || '').trim(); + if (!text) return { type: 'default' }; + if (text.toLowerCase() === 'none') return { type: 'none' }; + if (/^\/(?!\/)\S*$/.test(text)) return { type: 'image', src: text }; + try { + const url = new URL(text); + if (url.protocol === 'https:' || url.protocol === 'http:') { + return { type: 'image', src: url.href }; + } + } catch { + // Not a URL; the login page shows the default icon then. + } + return { type: 'default' }; +} + +export default function settingsSso(root, { toast }) { + const $ = (selector) => root.querySelector(selector); + const field = (id) => document.getElementById(id); + + /* --- redirect URI -------------------------------------------------------- */ + const redirect = field('oidcRedirectUri'); + if (redirect && !redirect.dataset.detected) { + // Rendered without the route's derived value: this page's own origin is + // what the server derives from the same request. + redirect.dataset.detected = `${window.location.origin}${CALLBACK_PATH}`; + redirect.placeholder = redirect.dataset.detected; + } + + const copyButton = $('[data-el="copy-redirect"]'); + if (copyButton && redirect) { + copyButton.addEventListener('click', async () => { + const value = redirect.value.trim() || redirect.dataset.detected || ''; + try { + await navigator.clipboard.writeText(value); + toast(`Copied ${value}`, { tone: 'ok' }); + } catch { + // No clipboard (plain http, or permission refused): select the text so + // it can be copied by hand. + redirect.focus(); + if (!redirect.value) redirect.value = value; + redirect.select(); + toast('Copy it from the selected field.', { tone: 'warn' }); + } + }); + } + + /* --- connection test ------------------------------------------------------ */ + const testButton = $('[data-el="test"]'); + const testResult = $('[data-el="test-result"]'); + + function showResult(tone, title, lines) { + if (!testResult) return; + const icon = tone === 'ok' ? 'i-check-circle' : 'i-alert'; + const box = document.createElement('div'); + box.className = `zr-alert zr-alert--${tone === 'ok' ? 'ok' : 'danger'}`; + box.innerHTML = `
`; + box.querySelector('.zr-alert__title').textContent = title; + const body = box.querySelector('.zr-alert__body'); + lines.forEach(([label, value]) => { + const p = document.createElement('p'); + p.className = 'zr-sm'; + if (label) { + const strong = document.createElement('span'); + strong.className = 'zr-strong'; + strong.textContent = `${label}: `; + p.append(strong); + } + if (value) { + const code = document.createElement(label ? 'code' : 'span'); + code.textContent = value; + p.append(code); + } + body.append(p); + }); + testResult.replaceChildren(box); + testResult.classList.remove('hidden'); + } + + async function runTest() { + const issuerUrl = (field('oidcIssuerUrl')?.value || '').trim(); + if (!issuerUrl) { + showResult('danger', 'Nothing to test yet', [ + ['', 'Enter the issuer URL of your provider first.'], + ]); + field('oidcIssuerUrl')?.focus(); + return; + } + + const payload = { + issuerUrl, + clientId: (field('oidcClientId')?.value || '').trim(), + // Empty asks the server to use the stored secret, which it does only for + // the issuer it was stored for. + clientSecret: field('oidcClientSecret')?.value || '', + tokenAuthMethod: field('oidcTokenAuthMethod')?.value || '', + }; + + const label = testButton.querySelector('span'); + const original = label ? label.textContent : ''; + testButton.disabled = true; + testButton.querySelector('.zr-icon')?.classList.add('zr-icon--spin'); + if (label) label.textContent = 'Testing…'; + + try { + const response = await fetch('/api/settings/oidc/test', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + const isJson = /application\/json/i.test( + response.headers.get('content-type') || '' + ); + const result = isJson ? await response.json().catch(() => null) : null; + + // An ended session answers with a redirect to /login, which fetch + // follows: the reply is the login page, not JSON. session-guard.js + // already sends the browser there; this only says why. + if ( + response.redirected || + response.status === 401 || + (!result && response.ok) + ) { + showResult('danger', 'Session expired', [ + ['', 'Your session has ended. Sign in again and repeat the test.'], + ]); + return; + } + if (!result) { + showResult('danger', 'The test could not run', [ + ['', `The server answered with HTTP ${response.status}.`], + ]); + return; + } + if (!response.ok || !result.success) { + showResult('danger', 'The provider could not be used', [ + [ + '', + result.error || + `The test failed with HTTP ${response.status}. Check the issuer URL.`, + ], + ]); + return; + } + + const data = result.data || {}; + const lines = [ + ['Issuer', data.issuer || issuerUrl], + ['Token authentication', data.chosenAuthMethod || 'automatic'], + ]; + if ( + Array.isArray(data.tokenAuthMethods) && + data.tokenAuthMethods.length + ) { + lines.push(['Provider offers', data.tokenAuthMethods.join(', ')]); + } + if (data.pkce === false) { + lines.push([ + '', + 'The provider does not list PKCE (S256); sign-in still sends it.', + ]); + } + showResult('ok', result.message || 'The provider answered', lines); + } catch (error) { + showResult('danger', 'The test could not run', [ + ['', error.message || 'The server did not answer.'], + ]); + } finally { + testButton.disabled = false; + testButton.querySelector('.zr-icon')?.classList.remove('zr-icon--spin'); + if (label) label.textContent = original; + } + } + + if (testButton) testButton.addEventListener('click', runTest); + + /* --- login button preview ------------------------------------------------- */ + const preview = $('[data-el="preview"]'); + const previewText = $('[data-el="preview-text"]'); + const previewImage = $('[data-el="preview-image"]'); + const previewDefaultIcon = $('[data-el="preview-default-icon"]'); + const previewFields = [ + 'oidcButtonText', + 'oidcButtonIcon', + 'oidcButtonColor', + 'oidcButtonTextColor', + ] + .map(field) + .filter(Boolean); + + function colourOf(id) { + const value = (field(id)?.value || '').trim(); + return isSafeCssColor(value) ? value : ''; + } + + function paintSwatch(id) { + const input = field(id); + const swatch = root.querySelector(`[data-swatch-for="${id}"]`); + if (!input || !swatch) return; + const raw = input.value.trim(); + const valid = !raw || isSafeCssColor(raw); + swatch.style.background = valid && raw ? raw : ''; + swatch.dataset.state = raw ? (valid ? 'set' : 'invalid') : 'empty'; + input.setAttribute('aria-invalid', valid ? 'false' : 'true'); + } + + function renderPreview() { + if (!preview) return; + const text = + (field('oidcButtonText')?.value || '').trim().slice(0, 60) || + DEFAULT_BUTTON_TEXT; + previewText.textContent = text; + + const icon = resolveButtonIcon(field('oidcButtonIcon')?.value); + previewDefaultIcon.classList.toggle('hidden', icon.type !== 'default'); + previewImage.classList.toggle('hidden', icon.type !== 'image'); + if (icon.type === 'image') { + previewImage.src = icon.src; + } else { + previewImage.removeAttribute('src'); + } + + const background = colourOf('oidcButtonColor'); + const foreground = colourOf('oidcButtonTextColor'); + if (background) preview.style.setProperty('--login-sso-bg', background); + else preview.style.removeProperty('--login-sso-bg'); + if (foreground) preview.style.setProperty('--login-sso-fg', foreground); + else preview.style.removeProperty('--login-sso-fg'); + + paintSwatch('oidcButtonColor'); + paintSwatch('oidcButtonTextColor'); + } + + previewFields.forEach((input) => { + input.addEventListener('input', renderPreview); + input.addEventListener('change', renderPreview); + }); + // A broken image URL falls back to nothing rather than a torn image icon. + previewImage?.addEventListener('error', () => + previewImage.classList.add('hidden') + ); + renderPreview(); + + return { + destroy() { + if (testButton) testButton.removeEventListener('click', runTest); + }, + }; +} diff --git a/public/js/session-guard.js b/public/js/session-guard.js new file mode 100644 index 00000000..cd0a0f3d --- /dev/null +++ b/public/js/session-guard.js @@ -0,0 +1,48 @@ +/** + * Sends the browser to the login page once a request finds the session gone. + * + * Every signed-in route answers a missing or expired session with a redirect + * to /login, and fetch follows redirects silently: the caller receives the + * login page with status 200, fails to read it as JSON and reports something + * generic such as "Model discovery failed". The page itself kept looking + * signed in, so every further action failed the same way and nothing pointed + * at the session. + * + * Loaded in right after csrf.js, so it wraps that patch and sees a + * request's final answer, after any CSRF token retry. + */ +(function () { + 'use strict'; + + const LOGIN_PATH = '/login'; + const innerFetch = window.fetch; + let leaving = false; + + function endedAtLogin(response) { + if (!response?.redirected || !response.url) return false; + try { + const target = new URL(response.url, window.location.href); + return ( + target.origin === window.location.origin && + target.pathname === LOGIN_PATH + ); + } catch { + return false; + } + } + + window.fetch = async (...args) => { + const response = await innerFetch(...args); + + if ( + !leaving && + window.location.pathname !== LOGIN_PATH && + endedAtLogin(response) + ) { + leaving = true; + window.location.assign(LOGIN_PATH); + } + + return response; + }; +})(); diff --git a/public/js/settings.js b/public/js/settings.js index 3a728d66..9a0bf3b4 100644 --- a/public/js/settings.js +++ b/public/js/settings.js @@ -114,6 +114,7 @@ class FormManager { const ollamaSettings = document.getElementById('ollamaSettings'); const customSettings = document.getElementById('customSettings'); const azureSettings = document.getElementById('azureSettings'); + const anthropicSettings = document.getElementById('anthropicSettings'); // Get all provider-specific fields const openaiKey = document.getElementById('openaiKey'); @@ -141,6 +142,13 @@ class FormManager { ollamaSettings.classList.add('hidden'); customSettings.classList.add('hidden'); azureSettings.classList.add('hidden'); + anthropicSettings?.classList.add('hidden'); + + // Claude models reject sampling parameters, so the temperature fields + // stay (they apply again after switching back) but say they are ignored. + document.querySelectorAll('[data-anthropic-hint]').forEach((hint) => { + hint.classList.toggle('hidden', provider !== 'anthropic'); + }); // Reset all required fields openaiKey.required = false; @@ -175,6 +183,11 @@ class FormManager { azureDeploymentName.required = true; azureApiVersion.required = true; break; + case 'anthropic': + // Nothing is required here: an empty key keeps the stored one (the + // save route refuses when there is none), and the model has a default. + anthropicSettings?.classList.remove('hidden'); + break; } } @@ -501,6 +514,15 @@ function initializeFormHandlers() { const aiProviderPresets = Array.isArray(settingsBootstrap.aiProviderPresets) ? settingsBootstrap.aiProviderPresets : []; + // Claude models and defaults as services/anthropicSupport.js defines them. + const anthropicOptions = settingsBootstrap.anthropic || {}; + const anthropicSuggestedModelIds = ( + Array.isArray(anthropicOptions.suggestedModels) + ? anthropicOptions.suggestedModels + : [] + ) + .map((entry) => String(entry?.id || '').trim()) + .filter(Boolean); const aiPresetSelect = document.getElementById('aiPreset'); const aiPresetHint = document.getElementById('aiPresetHint'); @@ -515,6 +537,12 @@ function initializeFormHandlers() { const fetchCustomAiModelsBtn = document.getElementById( 'fetchCustomAiModelsBtn' ); + const anthropicApiKeyInput = document.getElementById('anthropicApiKey'); + const anthropicModelInput = document.getElementById('anthropicModel'); + const anthropicApiUrlInput = document.getElementById('anthropicApiUrl'); + const fetchAnthropicModelsBtn = document.getElementById( + 'fetchAnthropicModelsBtn' + ); const ocrEnabledSelect = document.getElementById('mistralOcrEnabled'); const ocrFieldsContainer = document.getElementById('ocrFieldsContainer'); @@ -672,6 +700,16 @@ function initializeFormHandlers() { ).trim(), }; } + if (provider === 'anthropic') { + // An empty URL is api.anthropic.com, which is what the preset says. + return { + provider, + apiUrl: String(anthropicApiUrlInput?.value || '') + .trim() + .replace(/\/+$/, ''), + model: String(anthropicModelInput?.value || '').trim(), + }; + } return { provider: 'custom', @@ -757,6 +795,27 @@ function initializeFormHandlers() { azureEndpoint.value = String(preset.apiUrl || '').trim(); if (azureDeploymentName) azureDeploymentName.value = String(preset.model || '').trim(); + } else if (provider === 'anthropic') { + if (anthropicApiUrlInput) + anthropicApiUrlInput.value = String(preset.apiUrl || '').trim(); + if (anthropicModelInput) { + const presetModel = + String(preset.model || '').trim() || + String(anthropicOptions.defaultModel || ''); + populateModelSelect( + anthropicModelInput, + [...anthropicSuggestedModelIds, presetModel], + 'Select Claude model', + { preferred: presetModel } + ); + } + if ( + anthropicApiKeyInput && + preset.tokenPlaceholder && + anthropicApiKeyInput.placeholder !== 'Configured' + ) { + anthropicApiKeyInput.placeholder = preset.tokenPlaceholder; + } } if (aiPresetHint) { @@ -824,6 +883,8 @@ function initializeFormHandlers() { document.getElementById('openaiModel'), document.getElementById('azureEndpoint'), document.getElementById('azureDeploymentName'), + anthropicModelInput, + anthropicApiUrlInput, ].forEach((element) => { if (element) { element.addEventListener('change', () => refreshAiPresetSelection()); @@ -1002,6 +1063,83 @@ function initializeFormHandlers() { }); } + /* The live list needs a key; without one (or when the call fails) the route + answers with the suggested models and says so in its message. The list is + laid out like the OCR model select, and the model chosen before stays + selected and listed even when the answer does not name it. */ + const fillAnthropicModelSelect = (models) => { + const current = String(anthropicModelInput?.value || '').trim(); + const preferred = current || String(anthropicOptions.defaultModel || ''); + populateModelSelect( + anthropicModelInput, + [...models, preferred], + 'Select Claude model', + { preferred } + ); + }; + + if (fetchAnthropicModelsBtn) { + fetchAnthropicModelsBtn.addEventListener('click', async () => { + setButtonLoading(fetchAnthropicModelsBtn, true); + try { + const response = await fetch('/api/settings/ai/models', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + aiProvider: 'anthropic', + apiUrl: String(anthropicApiUrlInput?.value || '').trim(), + // Empty means the stored key; the route resolves it. + token: String(anthropicApiKeyInput?.value || '').trim(), + }), + }); + const result = await response.json().catch(() => ({})); + if (!response.ok || !result.success) { + throw new Error( + result.error || result.message || 'Model discovery failed' + ); + } + + const models = Array.isArray(result.models) ? result.models : []; + fillAnthropicModelSelect( + models.length > 0 ? models : anthropicSuggestedModelIds + ); + await zrDialog({ + icon: result.live === false ? 'warning' : 'success', + title: + result.live === false ? 'Suggested models shown' : 'Models loaded', + text: result.message || `Found ${models.length} model(s).`, + }); + } catch (error) { + fillAnthropicModelSelect(anthropicSuggestedModelIds); + await zrDialog({ + icon: 'error', + title: 'Loading failed', + text: String(error?.message || error), + }); + } finally { + setButtonLoading(fetchAnthropicModelsBtn, false); + } + }); + } + + /* What Claude OCR falls back to when its own fields are empty: the + Anthropic AI provider's key, URL and model — but only while that provider + is selected, because the save route stores them only then. */ + const getAnthropicFallbackForOcr = () => { + if ( + String(aiProviderSelect?.value || '') + .trim() + .toLowerCase() !== 'anthropic' + ) { + return {}; + } + return { + anthropicApiKey: String(anthropicApiKeyInput?.value || '').trim(), + anthropicApiUrl: String(anthropicApiUrlInput?.value || '').trim(), + anthropicModel: String(anthropicModelInput?.value || '').trim(), + }; + }; + const setOcrTestPill = (state, text) => { if (!ocrTestState) return; ocrTestState.textContent = text; @@ -1017,9 +1155,13 @@ function initializeFormHandlers() { const normalizeOcrProviderForApi = (provider) => { const normalized = String(provider || 'mistral').toLowerCase(); - return normalized === 'custom' ? 'custom' : 'mistral'; + return ['custom', 'anthropic'].includes(normalized) + ? normalized + : 'mistral'; }; + // Mistral has no URL field. For Claude the field is an optional proxy URL, + // so whatever is in it was typed for Claude (switching to Claude empties it). const normalizeOcrApiUrlForProvider = (provider, rawUrl) => { const normalizedProvider = normalizeOcrProviderForApi(provider); if (normalizedProvider === 'mistral') { @@ -1029,6 +1171,27 @@ function initializeFormHandlers() { return String(rawUrl || '').trim(); }; + const isClaudeModelName = (model) => /^claude-/i.test(String(model || '')); + + /* The OCR model select for Claude: the suggested models, with the current + choice kept when it is a Claude model and the AI provider's model (or the + default) preselected otherwise. */ + const offerAnthropicOcrModels = () => { + if (!ocrModelInput) return; + const current = String(ocrModelInput.value || '').trim(); + const aiModel = getAnthropicFallbackForOcr().anthropicModel || ''; + const preferred = isClaudeModelName(current) + ? current + : (isClaudeModelName(aiModel) && aiModel) || + String(anthropicOptions.defaultModel || ''); + populateModelSelect( + ocrModelInput, + [...anthropicSuggestedModelIds, preferred], + 'Select Claude model', + { preferred } + ); + }; + const getOcrValidationTimeoutMs = () => { const rawSeconds = Number.parseInt( String(ocrValidationTimeoutInput?.value || '30').trim(), @@ -1040,6 +1203,13 @@ function initializeFormHandlers() { return normalizedSeconds * 1000; }; + // The provider the OCR fields were last laid out for, so a switch to + // Claude can be told apart from a re-render (the quickstart sets the select + // without a change event and then calls toggleOcrFields()). + let previousOcrProvider = String( + ocrProviderSelect?.value || 'mistral' + ).toLowerCase(); + const toggleOcrFields = () => { if (!ocrEnabledSelect || !ocrProviderSelect) return; @@ -1057,10 +1227,31 @@ function initializeFormHandlers() { if (ocrApiUrlContainer) { ocrApiUrlContainer.classList.toggle( 'hidden', - !enabled || provider !== 'custom' + !enabled || !['custom', 'anthropic'].includes(provider) ); } + // Claude OCR: the key is optional (the AI provider's key is the + // fallback) and the URL is only for a proxy. Placeholders and hints for + // both cases are rendered by the server, which knows what is stored. + const isAnthropicOcr = provider === 'anthropic'; + [ocrApiKeyInput, ocrApiUrlInput].forEach((input) => { + if (!input) return; + const placeholder = isAnthropicOcr + ? input.dataset.placeholderAnthropic + : input.dataset.placeholderDefault; + if (placeholder) input.placeholder = placeholder; + }); + document + .getElementById('ocrApiKeyHint') + ?.classList.toggle('hidden', isAnthropicOcr); + document + .getElementById('ocrApiKeyAnthropicHint') + ?.classList.toggle('hidden', !isAnthropicOcr); + document + .getElementById('ocrApiUrlAnthropicHint') + ?.classList.toggle('hidden', !isAnthropicOcr); + // The recommended/other grouping in the model dropdown only exists on the // classification path, which the Mistral provider does not take. const ocrModelVisionHint = document.getElementById('ocrModelVisionHint'); @@ -1071,8 +1262,8 @@ function initializeFormHandlers() { ); } - // PDF page rendering only applies to local vision models; the Mistral - // provider handles PDFs natively. + // PDF page rendering only applies to local vision models; Mistral and + // Claude read PDFs natively. const ocrPdfRenderContainer = document.getElementById( 'ocrPdfRenderContainer' ); @@ -1086,6 +1277,8 @@ function initializeFormHandlers() { if (testOcrBtn) { testOcrBtn.disabled = !enabled; } + + previousOcrProvider = provider; }; if (ocrProviderSelect) { @@ -1095,8 +1288,29 @@ function initializeFormHandlers() { if (initialProvider === 'ollama') { ocrProviderSelect.value = 'custom'; } + if (initialProvider === 'anthropic') { + offerAnthropicOcrModels(); + } ocrProviderSelect.addEventListener('change', () => { + const provider = String( + ocrProviderSelect.value || 'mistral' + ).toLowerCase(); + if (provider === 'anthropic' && previousOcrProvider !== 'anthropic') { + // A URL left from another provider (an auto-detected Ollama URL, + // say) would be used as a Claude proxy. + if (ocrApiUrlInput) ocrApiUrlInput.value = ''; + offerAnthropicOcrModels(); + } else if ( + provider === 'mistral' && + isClaudeModelName(ocrModelInput?.value) + ) { + populateModelSelect( + ocrModelInput, + ['mistral-ocr-latest'], + 'Select OCR model' + ); + } toggleOcrFields(); setOcrTestPill('default', 'Not tested'); }); @@ -1625,9 +1839,15 @@ function initializeFormHandlers() { ocrApiUrlInput?.value ), apiKey: String(ocrApiKeyInput?.value || '').trim(), + // Claude OCR picks the AI model when this is empty; the Mistral + // default would only be replaced again. model: - String(ocrModelInput?.value || '').trim() || 'mistral-ocr-latest', + String(ocrModelInput?.value || '').trim() || + (ocrProviderSelect?.value === 'anthropic' + ? '' + : 'mistral-ocr-latest'), setupOcrValidationTimeoutMs: getOcrValidationTimeoutMs(), + ...getAnthropicFallbackForOcr(), }; const originalHtml = testOcrBtn.innerHTML; @@ -1713,6 +1933,7 @@ function initializeFormHandlers() { apiUrl, apiKey, setupOcrValidationTimeoutMs: getOcrValidationTimeoutMs(), + ...getAnthropicFallbackForOcr(), }), }); @@ -1727,6 +1948,14 @@ function initializeFormHandlers() { ocrApiUrlInput.value = String(result.resolvedApiUrl).trim(); } + // A Claude model already chosen stays chosen; the route suggests the + // AI provider's model otherwise. + const currentOcrModel = String(ocrModelInput?.value || '').trim(); + const preferredOcrModel = + provider === 'anthropic' && isClaudeModelName(currentOcrModel) + ? currentOcrModel + : result.suggestedModel; + const models = Array.isArray(result.models) ? result.models : []; const visionModels = Array.isArray(result.visionModels) ? result.visionModels @@ -1738,11 +1967,14 @@ function initializeFormHandlers() { recommended: visionModels, recommendedLabel: 'Recommended (vision detected)', otherLabel: 'Other models', - preferred: result.suggestedModel, + preferred: preferredOcrModel, }); await zrDialog({ - icon: 'success', - title: 'OCR models loaded', + icon: result.live === false ? 'warning' : 'success', + title: + result.live === false + ? 'Suggested models shown' + : 'OCR models loaded', text: models.length > 0 ? result.message : 'No models found.', }); } catch (error) { @@ -2382,6 +2614,7 @@ function initializeFormHandlers() { ), apiKey: String(formData.get('ocrApiKey') || '').trim(), setupOcrValidationTimeoutMs: getOcrValidationTimeoutMs(), + ...getAnthropicFallbackForOcr(), }; const response = await fetch('/api/settings/ocr/models', { @@ -2442,6 +2675,9 @@ function initializeFormHandlers() { const result = await response.json(); if (result.success) { + // The saved values are the new baseline for the unsaved-change count + // and the leave warning (modules/settings-form.js). + setupForm.dispatchEvent(new CustomEvent('zr:settings-saved')); await zrDialog({ icon: 'success', title: 'Success!', @@ -2465,6 +2701,13 @@ function initializeFormHandlers() { title: 'Error', text: error.message, }); + // Only one settings topic is visible at a time; this opens the one + // holding the field the message names (modules/settings-nav.js). + document.dispatchEvent( + new CustomEvent('zr:settings-save-error', { + detail: { message: String(error.message || '') }, + }) + ); } finally { submitBtn.disabled = false; submitBtn.innerHTML = originalBtnText; @@ -2691,6 +2934,75 @@ function initializeTooltipAndValidation() { initializeFieldHelp(); } +/** + * Names each field's environment variable in its hint ("SCAN_INTERVAL · + * Restart required") and on the field itself as data-env-key, unless the + * markup already names it. A switch carries the key on its hidden yes/no input + * and on the visible switch alike. + */ +function annotateEnvKeys(fieldMappings) { + fieldMappings.forEach(({ selector, envKey }) => { + const field = document.querySelector(selector); + if (!field) return; + field.dataset.envKey = envKey; + const visibleSwitch = field.id + ? document.querySelector(`[data-switch-target="${field.id}"]`) + : null; + if (visibleSwitch) visibleSwitch.dataset.envKey = envKey; + + // A tile is a single checkbox with its label; the key goes in its tooltip. + const tile = field.closest('.zr-checktile'); + if (tile) { + tile.title = envKey; + return; + } + const container = field.closest('.zr-field, .zr-switchrow'); + if (!container) return; + const hints = [...container.querySelectorAll('.zr-field__hint')].filter( + (hint) => !hint.hasAttribute('data-anthropic-hint') + ); + const keyElement = () => { + const code = document.createElement('code'); + code.className = 'zr-envkey'; + code.textContent = envKey; + return code; + }; + + // A hint that names the key in its markup already gets it set like the + // added ones, instead of a second copy. + const naming = hints.filter((hint) => hint.textContent.includes(envKey)); + if (naming.length) { + naming.forEach((hint) => { + if (hint.querySelector('.zr-envkey')) return; + const node = [...hint.childNodes].find( + (child) => + child.nodeType === Node.TEXT_NODE && + child.textContent.includes(envKey) + ); + if (!node) return; + const at = node.textContent.indexOf(envKey); + const rest = node.splitText(at); + rest.textContent = rest.textContent.slice(envKey.length); + hint.insertBefore(keyElement(), rest); + }); + return; + } + if (!hints.length) { + const hint = document.createElement('p'); + hint.className = 'zr-field__hint'; + hint.append(keyElement()); + container.append(hint); + return; + } + // Hints that swap with the provider (the OCR key) each name the key, so it + // does not vanish with the one that is shown first. + const targets = hints.some((hint) => hint.id) + ? hints.filter((hint) => hint.id) + : [hints[0]]; + targets.forEach((hint) => hint.prepend(keyElement(), ' · ')); + }); +} + function initializeRuntimeOverridePills() { const resetLocalOverridesBtn = document.getElementById( 'resetLocalOverridesBtn' @@ -2740,26 +3052,9 @@ function initializeRuntimeOverridePills() { } } - const overrideKeys = new Set( - Array.isArray(parsedOverrideKeys) ? parsedOverrideKeys : [] - ); - const lockedEnvKeys = new Set( - Array.isArray(parsedLockedEnvKeys) ? parsedLockedEnvKeys : [] - ); - if (overrideKeys.size === 0 && lockedEnvKeys.size === 0) { - return; - } - - // Kept local because classic scripts cannot import ES modules; keep in sync - // with modules/text-utils.js. - const escapeHtml = (value) => - String(value) - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); - + // Every field with the variable it writes. Besides the pills below, this + // names the variable in each hint and gives the settings search + // (modules/settings-nav.js) something to find a field by. const fieldMappings = [ { selector: '#paperlessUrl', envKey: 'PAPERLESS_API_URL' }, { selector: '#paperlessPublicUrl', envKey: 'PAPERLESS_PUBLIC_URL' }, @@ -2786,6 +3081,16 @@ function initializeRuntimeOverridePills() { { selector: '#azureApiKey', envKey: 'AZURE_API_KEY' }, { selector: '#azureDeploymentName', envKey: 'AZURE_DEPLOYMENT_NAME' }, { selector: '#azureApiVersion', envKey: 'AZURE_API_VERSION' }, + { selector: '#anthropicApiKey', envKey: 'ANTHROPIC_API_KEY' }, + { selector: '#anthropicModel', envKey: 'ANTHROPIC_MODEL' }, + { selector: '#anthropicEffort', envKey: 'ANTHROPIC_EFFORT' }, + { selector: '#anthropicMaxTokens', envKey: 'ANTHROPIC_MAX_TOKENS' }, + { + selector: '#anthropicStructuredOutput', + envKey: 'ANTHROPIC_STRUCTURED_OUTPUT', + }, + { selector: '#anthropicPromptCaching', envKey: 'ANTHROPIC_PROMPT_CACHING' }, + { selector: '#anthropicApiUrl', envKey: 'ANTHROPIC_API_URL' }, { selector: '#tokenLimit', envKey: 'TOKEN_LIMIT' }, { selector: '#responseTokens', envKey: 'RESPONSE_TOKENS' }, { selector: '#aiTemperatureAnalysis', envKey: 'AI_TEMPERATURE_ANALYSIS' }, @@ -2933,8 +3238,47 @@ function initializeRuntimeOverridePills() { selector: '#externalApiAllowPrivateIps', envKey: 'EXTERNAL_API_ALLOW_PRIVATE_IPS', }, + { selector: '#oidcEnabled', envKey: 'OIDC_ENABLED' }, + { selector: '#oidcIssuerUrl', envKey: 'OIDC_ISSUER_URL' }, + { selector: '#oidcClientId', envKey: 'OIDC_CLIENT_ID' }, + { selector: '#oidcClientSecret', envKey: 'OIDC_CLIENT_SECRET' }, + { selector: '#oidcAdminEmail', envKey: 'OIDC_ADMIN_EMAIL' }, + { selector: '#oidcRedirectUri', envKey: 'OIDC_REDIRECT_URI' }, + { selector: '#oidcScopes', envKey: 'OIDC_SCOPES' }, + { selector: '#oidcEmailClaim', envKey: 'OIDC_EMAIL_CLAIM' }, + { + selector: '#oidcAllowUnverifiedEmail', + envKey: 'OIDC_ALLOW_UNVERIFIED_EMAIL', + }, + { selector: '#oidcTokenAuthMethod', envKey: 'OIDC_TOKEN_AUTH_METHOD' }, + { selector: '#oidcButtonText', envKey: 'OIDC_BUTTON_TEXT' }, + { selector: '#oidcButtonIcon', envKey: 'OIDC_BUTTON_ICON' }, + { selector: '#oidcButtonColor', envKey: 'OIDC_BUTTON_COLOR' }, + { selector: '#oidcButtonTextColor', envKey: 'OIDC_BUTTON_TEXT_COLOR' }, ]; + annotateEnvKeys(fieldMappings); + + const overrideKeys = new Set( + Array.isArray(parsedOverrideKeys) ? parsedOverrideKeys : [] + ); + const lockedEnvKeys = new Set( + Array.isArray(parsedLockedEnvKeys) ? parsedLockedEnvKeys : [] + ); + if (overrideKeys.size === 0 && lockedEnvKeys.size === 0) { + return; + } + + // Kept local because classic scripts cannot import ES modules; keep in sync + // with modules/text-utils.js. + const escapeHtml = (value) => + String(value) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + const pills = []; fieldMappings.forEach(({ selector, envKey }) => { @@ -2990,6 +3334,15 @@ function initializeRuntimeOverridePills() { // The :disabled styling in the framework carries the visual state. fieldElement.disabled = true; fieldElement.setAttribute('aria-disabled', 'true'); + // A switch is clicked on its visible half; the hidden yes/no input alone + // being disabled left the switch movable with nothing to show for it. + const lockedSwitch = fieldElement.id + ? document.querySelector(`[data-switch-target="${fieldElement.id}"]`) + : null; + if (lockedSwitch) { + lockedSwitch.disabled = true; + lockedSwitch.setAttribute('aria-disabled', 'true'); + } if (!targetLabel.querySelector('.locked-pill')) { const lockedPill = document.createElement('span'); diff --git a/public/js/setup.js b/public/js/setup.js index 40c88ce2..6711a53e 100644 --- a/public/js/setup.js +++ b/public/js/setup.js @@ -7,6 +7,15 @@ class SetupWizard { this.presets = Array.isArray(this.bootstrap.aiProviderPresets) ? this.bootstrap.aiProviderPresets : []; + // Claude models and defaults as services/anthropicSupport.js defines them. + this.anthropic = this.bootstrap.anthropic || {}; + this.anthropicSuggestedModelIds = ( + Array.isArray(this.anthropic.suggestedModels) + ? this.anthropic.suggestedModels + : [] + ) + .map((entry) => String(entry?.id || '').trim()) + .filter(Boolean); this.steps = Array.from(document.querySelectorAll('.zr-steppane')); this.stepLabel = document.getElementById('setupStepLabel'); @@ -115,6 +124,7 @@ class SetupWizard { this.aiPresetHint = document.getElementById('aiPresetHint'); this.aiProvider = document.getElementById('aiProvider'); this.aiApiUrl = document.getElementById('aiApiUrl'); + this.aiApiUrlHint = document.getElementById('aiApiUrlHint'); this.aiToken = document.getElementById('aiToken'); this.aiModel = document.getElementById('aiModel'); this.fetchAiModelsBtn = document.getElementById('fetchAiModelsBtn'); @@ -277,6 +287,11 @@ class SetupWizard { ); this.aiModel.value = this.config.AZURE_DEPLOYMENT_NAME || ''; this.aiToken.value = this.config.AZURE_API_KEY || ''; + } else if (provider === 'anthropic') { + // The key never reaches the page (sanitizeConfigForBootstrap drops it). + this.aiApiUrl.value = this.config.ANTHROPIC_API_URL || ''; + this.offerAnthropicAiModels(this.config.ANTHROPIC_MODEL); + this.aiToken.value = ''; } else { this.aiProvider.value = 'custom'; this.aiApiUrl.value = this.config.CUSTOM_BASE_URL || ''; @@ -288,6 +303,88 @@ class SetupWizard { this.aiModel.value = this.config.CUSTOM_MODEL || ''; this.aiToken.value = this.config.CUSTOM_API_KEY || ''; } + this.updateAiProviderFields(); + // After the AI values, whose model Claude OCR prefers. + if (this.ocrProvider.value === 'anthropic') { + this.offerAnthropicOcrModels(); + } + } + + isClaudeModelName(model) { + return /^claude-/i.test(String(model || '')); + } + + /* The model select for Anthropic: the suggested models, preselecting the + given one (or the default). A live list replaces it on "Fetch models". */ + offerAnthropicAiModels(model) { + const preferred = + String(model || '').trim() || String(this.anthropic.defaultModel || ''); + this.setModelSelectOptions( + this.aiModel, + [...this.anthropicSuggestedModelIds, preferred], + 'Select model', + { preferred } + ); + } + + /* Anthropic is the one provider whose API URL is optional: the SDK knows + api.anthropic.com and the field only serves a proxy. */ + updateAiProviderFields() { + const isAnthropic = + String(this.aiProvider.value || '') + .trim() + .toLowerCase() === 'anthropic'; + this.aiApiUrl.required = !isAnthropic; + this.aiApiUrl.placeholder = isAnthropic ? 'https://api.anthropic.com' : ''; + this.aiApiUrlHint?.classList.toggle('hidden', !isAnthropic); + } + + aiProviderNeedsApiUrl(provider) { + return String(provider || '').toLowerCase() !== 'anthropic'; + } + + /* What Claude OCR falls back to when its own key is empty: the AI step's + key, URL and model, while Anthropic is the AI provider. */ + getAnthropicFallbackForOcr() { + if ( + String(this.aiProvider.value || '') + .trim() + .toLowerCase() !== 'anthropic' + ) { + return {}; + } + return { + anthropicApiKey: this.aiToken.value.trim(), + anthropicApiUrl: this.aiApiUrl.value.trim(), + anthropicModel: this.aiModel.value.trim(), + }; + } + + /* The OCR model sent to the server. Claude OCR picks the AI model itself + when this is empty, so the Mistral default is only filled in for others. */ + getOcrModelForRequest() { + const model = this.mistralOcrModel.value.trim(); + if (model) return model; + return (this.ocrProvider.value || 'mistral').toLowerCase() === 'anthropic' + ? '' + : 'mistral-ocr-latest'; + } + + /* The OCR model select for Claude: the suggested models, keeping a Claude + model already chosen and preferring the AI step's model otherwise. */ + offerAnthropicOcrModels() { + const current = this.mistralOcrModel.value.trim(); + const aiModel = this.getAnthropicFallbackForOcr().anthropicModel || ''; + const preferred = this.isClaudeModelName(current) + ? current + : (this.isClaudeModelName(aiModel) && aiModel) || + String(this.anthropic.defaultModel || ''); + this.setModelSelectOptions( + this.mistralOcrModel, + [...this.anthropicSuggestedModelIds, preferred], + 'Select Claude model', + { preferred } + ); } findMatchingPreset() { @@ -893,12 +990,52 @@ class SetupWizard { this.ocrApiKeyContainer.classList.toggle('hidden', !enabled); } + // Switching to Claude: a URL left from another provider (an + // auto-detected Ollama URL, say) would be used as a Claude proxy, and the + // model select needs the Claude models. Back to Mistral, a Claude model + // would be sent to Mistral. + const previousProvider = this.previousOcrProvider || provider; + if (provider === 'anthropic' && previousProvider !== 'anthropic') { + this.ocrApiUrl.value = ''; + this.offerAnthropicOcrModels(); + } else if ( + provider === 'mistral' && + previousProvider !== 'mistral' && + this.isClaudeModelName(this.mistralOcrModel.value) + ) { + this.setModelSelectOptions( + this.mistralOcrModel, + ['mistral-ocr-latest'], + 'Select OCR model' + ); + } + this.previousOcrProvider = provider; + + const isAnthropic = provider === 'anthropic'; if (this.ocrApiUrlContainer) { this.ocrApiUrlContainer.classList.toggle( 'hidden', - provider !== 'custom' || !enabled + !['custom', 'anthropic'].includes(provider) || !enabled ); } + document + .getElementById('ocrApiUrlHint') + ?.classList.toggle('hidden', isAnthropic); + document + .getElementById('ocrApiUrlAnthropicHint') + ?.classList.toggle('hidden', !isAnthropic); + document + .getElementById('ocrApiKeyHint') + ?.classList.toggle('hidden', isAnthropic); + document + .getElementById('ocrApiKeyAnthropicHint') + ?.classList.toggle('hidden', !isAnthropic); + [this.ocrApiKey, this.ocrApiUrl].forEach((input) => { + const placeholder = isAnthropic + ? input?.dataset.placeholderAnthropic + : input?.dataset.placeholderDefault; + if (placeholder) input.placeholder = placeholder; + }); // The recommended/other grouping in the model dropdown only exists on the // classification path, which the Mistral provider does not take. @@ -929,8 +1066,9 @@ class SetupWizard { this.ocrApiUrl.value ), apiKey: this.ocrApiKey.value.trim(), - model: this.mistralOcrModel.value.trim() || 'mistral-ocr-latest', + model: this.getOcrModelForRequest(), setupOcrValidationTimeoutMs: this.getOcrValidationTimeoutMs(), + ...this.getAnthropicFallbackForOcr(), }; if (!payload.enabled) { @@ -1316,6 +1454,7 @@ class SetupWizard { applyPreset(preset) { if (!preset) { this.aiProvider.value = 'custom'; + this.updateAiProviderFields(); this.aiPresetHint.textContent = 'Manual mode: choose provider and enter values yourself. Token is optional for custom endpoints.'; return; @@ -1323,7 +1462,13 @@ class SetupWizard { this.aiProvider.value = preset.provider || 'custom'; this.aiApiUrl.value = preset.apiUrl || ''; - this.aiModel.value = preset.model || ''; + if (this.aiProvider.value === 'anthropic') { + // The select has to offer the model before it can show it. + this.offerAnthropicAiModels(preset.model); + } else { + this.aiModel.value = preset.model || ''; + } + this.updateAiProviderFields(); this.aiToken.placeholder = preset.tokenPlaceholder || 'Enter API token if required'; this.aiPresetHint.textContent = `Preset "${preset.label}" selected.${this.aiProvider.value === 'custom' ? ' Token can stay empty if your endpoint allows anonymous access.' : ''}`; @@ -1501,6 +1646,7 @@ class SetupWizard { (selectedAiModel && this.aiModel.value !== selectedAiModel); this.aiProvider.value = detection.aiProvider || 'custom'; + this.updateAiProviderFields(); this.aiApiUrl.value = detection.resolvedAiApiUrl || ''; this.aiToken.value = quickstartKey; if (selectedAiModel) { @@ -1833,7 +1979,10 @@ class SetupWizard { return []; } - if (!['openai', 'azure'].includes(payload.aiProvider) && !payload.apiUrl) { + if ( + !['openai', 'azure', 'anthropic'].includes(payload.aiProvider) && + !payload.apiUrl + ) { if (!silent) { await this.showPopup({ icon: 'warning', @@ -1862,8 +2011,15 @@ class SetupWizard { const resolvedInfo = result.resolvedApiUrl ? `\nResolved API URL: ${String(result.resolvedApiUrl).trim()}` : ''; + // Anthropic answers with the suggested models when the live list + // fails (live: false); the message says so. await this.showPopup({ - icon: models.length > 0 ? 'success' : 'info', + icon: + result.live === false + ? 'warning' + : models.length > 0 + ? 'success' + : 'info', title: 'AI models loaded', text: `${result.message || (models.length > 0 ? 'Models discovered successfully.' : 'No models found.')}${resolvedInfo}`, }); @@ -1900,6 +2056,7 @@ class SetupWizard { ), apiKey: this.ocrApiKey.value.trim(), setupOcrValidationTimeoutMs: this.getOcrValidationTimeoutMs(), + ...this.getAnthropicFallbackForOcr(), }; // No local refusal on an empty field: an operator who already has @@ -1907,6 +2064,7 @@ class SetupWizard { // resolves it. Whether a key exists is the server's question to answer. this.setButtonLoading(this.fetchOcrModelsBtn, true, 'Loading...'); try { + const previousOcrModel = this.mistralOcrModel.value.trim(); const result = await this.request('/api/setup/ocr/models', payload); // The result dialog is awaited below; reset the button first so it // does not keep claiming the test is still running. @@ -1930,13 +2088,23 @@ class SetupWizard { : [], recommendedLabel: 'Recommended (vision detected)', otherLabel: 'Other models', - preferred: result.suggestedModel, + // A Claude model already chosen stays chosen. + preferred: + payload.provider === 'anthropic' && + this.isClaudeModelName(previousOcrModel) + ? previousOcrModel + : result.suggestedModel, } ); if (!silent) { await this.showPopup({ - icon: models.length > 0 ? 'success' : 'info', + icon: + result.live === false + ? 'warning' + : models.length > 0 + ? 'success' + : 'info', title: 'OCR models loaded', text: result.message || @@ -1981,11 +2149,17 @@ class SetupWizard { setupValidationTimeoutMs: this.getAiValidationTimeoutMs(), }; - if (!payload.aiProvider || !payload.model || !payload.apiUrl) { + if ( + !payload.aiProvider || + !payload.model || + (!payload.apiUrl && this.aiProviderNeedsApiUrl(payload.aiProvider)) + ) { await this.showPopup({ icon: 'warning', title: 'Missing values', - text: 'Provider, API URL, and model are required.', + text: this.aiProviderNeedsApiUrl(payload.aiProvider) + ? 'Provider, API URL, and model are required.' + : 'Provider and model are required.', }); return; } @@ -2201,21 +2375,26 @@ class SetupWizard { await this.fetchAiModels(true); } + const providerNeedsUrl = this.aiProviderNeedsApiUrl( + this.aiProvider.value.trim() + ); if ( !this.aiProvider.value.trim() || - !this.aiApiUrl.value.trim() || + (providerNeedsUrl && !this.aiApiUrl.value.trim()) || !this.aiModel.value.trim() ) { await this.showPopup({ icon: 'warning', title: 'Missing values', - text: 'Provider, API URL, and model are required.', + text: providerNeedsUrl + ? 'Provider, API URL, and model are required.' + : 'Provider and model are required.', }); return false; } const provider = this.aiProvider.value.trim().toLowerCase(); - const tokenRequired = provider === 'openai' || provider === 'azure'; + const tokenRequired = ['openai', 'azure', 'anthropic'].includes(provider); if (tokenRequired && !this.aiToken.value.trim()) { await this.showPopup({ icon: 'warning', @@ -2283,11 +2462,11 @@ class SetupWizard { await this.fetchOcrModels(true); } - if (!['mistral', 'custom'].includes(provider)) { + if (!['mistral', 'custom', 'anthropic'].includes(provider)) { await this.showPopup({ icon: 'warning', title: 'Invalid OCR provider', - text: 'Choose either mistral or custom for OCR fallback.', + text: 'Choose Mistral, custom or Claude for OCR fallback.', }); return false; } @@ -2342,6 +2521,15 @@ class SetupWizard { preview.push(`AZURE_API_KEY=${this.aiToken.value.trim()}`); preview.push(`AZURE_DEPLOYMENT_NAME=${this.aiModel.value.trim()}`); preview.push('AZURE_API_VERSION=2023-05-15'); + } else if (provider === 'anthropic') { + preview.push(`ANTHROPIC_API_KEY=${this.aiToken.value.trim()}`); + preview.push( + `ANTHROPIC_MODEL=${this.aiModel.value.trim() || this.anthropic.defaultModel || ''}` + ); + // Written only when set, like the server does: empty is api.anthropic.com. + if (this.aiApiUrl.value.trim()) { + preview.push(`ANTHROPIC_API_URL=${this.aiApiUrl.value.trim()}`); + } } else { preview.push(`CUSTOM_BASE_URL=${this.aiApiUrl.value.trim()}`); preview.push(`CUSTOM_API_KEY=${this.aiToken.value.trim()}`); @@ -2362,14 +2550,14 @@ class SetupWizard { this.ocrProvider.value || 'mistral' ).toLowerCase(); preview.push( - `OCR_PROVIDER=${normalizedOcrProvider === 'custom' ? 'custom' : 'mistral'}` + `OCR_PROVIDER=${['custom', 'anthropic'].includes(normalizedOcrProvider) ? normalizedOcrProvider : 'mistral'}` ); preview.push( `OCR_API_URL=${this.normalizeOcrApiUrlForProvider(normalizedOcrProvider, this.ocrApiUrl.value)}` ); preview.push(`OCR_API_KEY=${this.ocrApiKey.value.trim()}`); preview.push( - `MISTRAL_OCR_MODEL=${this.mistralOcrModel.value.trim() || 'mistral-ocr-latest'}` + `MISTRAL_OCR_MODEL=${this.getOcrModelForRequest() || this.getAnthropicFallbackForOcr().anthropicModel || this.anthropic.defaultModel || ''}` ); return preview.join('\n'); @@ -2425,7 +2613,7 @@ class SetupWizard { this.ocrApiUrl.value ), this.ocrApiKey.value.trim(), - this.mistralOcrModel.value.trim() || 'mistral-ocr-latest', + this.getOcrModelForRequest(), ]), }; } @@ -2478,8 +2666,7 @@ class SetupWizard { this.ocrApiUrl.value ), ocrApiKey: this.ocrApiKey.value.trim(), - mistralOcrModel: - this.mistralOcrModel.value.trim() || 'mistral-ocr-latest', + mistralOcrModel: this.getOcrModelForRequest(), }; } diff --git a/routes/oidc.js b/routes/oidc.js new file mode 100644 index 00000000..e8d375bc --- /dev/null +++ b/routes/oidc.js @@ -0,0 +1,228 @@ +const express = require('express'); +const router = express.Router(); +const rateLimit = require('express-rate-limit'); +const jwt = require('jsonwebtoken'); +const config = require('../config/config'); +const documentModel = require('../models/document'); +const oidcService = require('../services/oidcService'); +const { SESSION_TOKEN_TYPE } = require('./auth'); + +// Holds state, nonce and PKCE verifier between the redirect to the provider +// and the callback. Signed like the MFA challenge, with its own type claim so +// it can never pass for a session and a session can never pass for it. +const OIDC_TRANSACTION_COOKIE = 'zr_oidc_tx'; +const OIDC_TRANSACTION_TYPE = 'oidc-login'; +const OIDC_TRANSACTION_TTL_SECONDS = 10 * 60; +const SESSION_TTL_MS = 24 * 60 * 60 * 1000; + +function shouldUseSecureCookies(req) { + const mode = config.getCookieSecureMode(); + if (mode === 'always') return true; + if (mode === 'never') return false; + + const forwardedProto = String(req.headers['x-forwarded-proto'] || '') + .split(',')[0] + .trim() + .toLowerCase(); + return Boolean(req.secure || forwardedProto === 'https'); +} + +function renderLoginError(res, status, error) { + return res.status(status).render('login', { + error, + mfaRequired: false, + username: '', + sso: oidcService.getLoginButton(), + }); +} + +// Same budget as the password form: both start a sign-in. +const oidcLimiter = rateLimit({ + windowMs: 15 * 60 * 1000, + max: parseInt(process.env.LOGIN_RATE_LIMIT_MAX || '10', 10), + standardHeaders: true, + legacyHeaders: false, + handler: (_req, res) => + renderLoginError( + res, + 429, + 'Too many login attempts. Please wait a few minutes and try again.' + ), +}); + +/** + * @swagger + * /auth/oidc/login: + * get: + * summary: Start single sign-on with the configured OpenID Connect provider + * description: | + * Redirects the browser to the identity provider's authorization endpoint + * (authorization code flow with PKCE, state and nonce). Only available when + * OIDC_ENABLED=yes and OIDC_ISSUER_URL, OIDC_CLIENT_ID and OIDC_ADMIN_EMAIL + * are set. + * tags: + * - Authentication + * responses: + * 302: + * description: Redirect to the identity provider + * 404: + * description: Single sign-on is not enabled + * 502: + * description: The identity provider could not be reached; the login page is rendered with an error + */ +router.get('/auth/oidc/login', oidcLimiter, async (req, res) => { + if (!oidcService.isEnabled()) { + return res.status(404).send('Single sign-on is not enabled'); + } + + const jwtSecret = config.getJwtSecret(); + if (!jwtSecret) { + return res.status(500).send('Server misconfiguration: JWT secret missing'); + } + + try { + const { url, transaction } = await oidcService.startLogin(req); + const transactionToken = jwt.sign( + { challengeType: OIDC_TRANSACTION_TYPE, ...transaction }, + jwtSecret, + { expiresIn: OIDC_TRANSACTION_TTL_SECONDS } + ); + // SameSite=Lax: the provider sends the browser back with a top-level GET, + // which carries Lax cookies. + res.cookie(OIDC_TRANSACTION_COOKIE, transactionToken, { + httpOnly: true, + secure: shouldUseSecureCookies(req), + sameSite: 'lax', + path: '/', + maxAge: OIDC_TRANSACTION_TTL_SECONDS * 1000, + }); + return res.redirect(url); + } catch (error) { + console.error('[ERROR] OIDC login could not start:', error.message); + return renderLoginError( + res, + 502, + 'Single sign-on is unavailable: the identity provider could not be reached. Check OIDC_ISSUER_URL.' + ); + } +}); + +/** + * @swagger + * /auth/oidc/callback: + * get: + * summary: Complete single sign-on + * description: | + * Redirect target registered with the identity provider. Exchanges the + * authorization code, validates the ID token and signs in as the + * administrator account when the identity's e-mail address equals + * OIDC_ADMIN_EMAIL. Sets the session cookie. + * tags: + * - Authentication + * parameters: + * - in: query + * name: code + * schema: + * type: string + * description: Authorization code issued by the provider + * - in: query + * name: state + * schema: + * type: string + * description: State value echoed back by the provider + * responses: + * 302: + * description: Signed in, redirected to the dashboard + * headers: + * Set-Cookie: + * schema: + * type: string + * description: HTTP-only cookie containing the session JWT + * 400: + * description: Missing or expired sign-in transaction, or the provider reported an error + * 403: + * description: The identity's e-mail address does not match OIDC_ADMIN_EMAIL + * 404: + * description: Single sign-on is not enabled + */ +router.get('/auth/oidc/callback', oidcLimiter, async (req, res) => { + if (!oidcService.isEnabled()) { + return res.status(404).send('Single sign-on is not enabled'); + } + + const jwtSecret = config.getJwtSecret(); + if (!jwtSecret) { + return res.status(500).send('Server misconfiguration: JWT secret missing'); + } + + const transactionToken = req.cookies?.[OIDC_TRANSACTION_COOKIE]; + res.clearCookie(OIDC_TRANSACTION_COOKIE, { path: '/' }); + + let transaction; + try { + transaction = jwt.verify(transactionToken || '', jwtSecret); + if (transaction.challengeType !== OIDC_TRANSACTION_TYPE) { + throw new Error('Invalid transaction type'); + } + } catch { + return renderLoginError( + res, + 400, + 'Your single sign-on session expired. Please try again.' + ); + } + + const query = new URLSearchParams(req.originalUrl.split('?')[1] || ''); + + let claims; + try { + claims = await oidcService.completeLogin(query, transaction); + } catch (error) { + console.error('[ERROR] OIDC callback failed:', error.message); + return renderLoginError( + res, + 400, + 'Single sign-on failed. Please try again or sign in with your password.' + ); + } + + try { + const users = await documentModel.getUsers(); + const { user, error, email } = oidcService.authorize(claims, users); + + if (!user) { + console.warn( + '[FAILED LOGIN] OIDC sign-in refused for %s: %s', + email || '(no e-mail claim)', + error + ); + return renderLoginError(res, 403, error); + } + + const token = jwt.sign( + { + id: user.id, + username: user.username, + typ: SESSION_TOKEN_TYPE, + }, + jwtSecret, + { expiresIn: '24h' } + ); + res.cookie('jwt', token, { + httpOnly: true, + secure: shouldUseSecureCookies(req), + sameSite: 'lax', + path: '/', + maxAge: SESSION_TTL_MS, + }); + + console.log('[LOGIN] OIDC sign-in for user:', user.username); + return res.redirect('/dashboard'); + } catch (error) { + console.error('[ERROR] OIDC sign-in error:', error); + return renderLoginError(res, 500, 'An error occurred during login'); + } +}); + +module.exports = router; +module.exports.OIDC_TRANSACTION_COOKIE = OIDC_TRANSACTION_COOKIE; diff --git a/routes/setup.js b/routes/setup.js index d41598cb..7d41ea02 100644 --- a/routes/setup.js +++ b/routes/setup.js @@ -44,6 +44,7 @@ const entityNameMatcher = require('../services/entityNameMatcher'); const entityMatchAiService = require('../services/entityMatchAiService'); const duplicateReviewJobService = require('../services/duplicateReviewJobService'); const tagSimplifyService = require('../services/tagSimplifyService'); +const oidcService = require('../services/oidcService'); const { THUMBNAIL_CACHE_DIR, getThumbnailCachePath, @@ -51,6 +52,13 @@ const { const { sanitizeConfigForBootstrap, } = require('../services/bootstrapConfigSanitizer'); +const { + ANTHROPIC_DEFAULT_MODEL, + ANTHROPIC_SUGGESTED_MODELS, + ANTHROPIC_EFFORT_LEVELS, + ANTHROPIC_DEFAULT_MAX_TOKENS, + isConfiguredAnthropicHost, +} = require('../services/anthropicSupport'); const config = require('../config/config.js'); require('dotenv').config({ path: '../data/.env' }); @@ -86,9 +94,11 @@ const SETTINGS_SECRET_FIELDS = [ 'OLLAMA_API_KEY', 'CUSTOM_API_KEY', 'AZURE_API_KEY', + 'ANTHROPIC_API_KEY', 'OCR_API_KEY', 'MISTRAL_API_KEY', 'API_KEY', + 'OIDC_CLIENT_SECRET', ]; function formatBytes(bytes) { @@ -395,8 +405,17 @@ const loginLimiter = rateLimit({ * example: "#FF5733" */ -// Routes that don't require authentication -let PUBLIC_ROUTES = ['/health', '/login', '/logout', '/setup', '/api/setup']; +// Routes that don't require authentication. /auth/oidc is served by +// routes/oidc.js, mounted ahead of this router; it is listed so the sign-in +// callback stays reachable whatever the mount order. +let PUBLIC_ROUTES = [ + '/health', + '/login', + '/logout', + '/setup', + '/api/setup', + '/auth/oidc', +]; /** * Returns true if the incoming request originates from localhost. @@ -579,6 +598,20 @@ function hasValidSession(req) { return Boolean(jwtSecret && token && verifySessionToken(token, jwtSecret)); } +/** + * True when a public route is called with a usable session or the API key. + * Like hasValidSession(), for routes the authentication guard lets through. + */ +function isAuthenticatedCaller(req) { + const currentApiKey = config.getApiKey(); + const apiKey = req.headers['x-api-key']; + if (currentApiKey && apiKey && apiKey === currentApiKey) { + return true; + } + + return hasValidSession(req); +} + /** * @swagger * /login: @@ -712,18 +745,24 @@ function getAuthenticatedSettingsUsername(req) { return null; } -function verifyTotpToken(secret, inputToken) { +/** + * Returns the TOTP time step the code belongs to, or null when the code does + * not match any step inside the accepted window. + * + * @param {string} secret - base32 TOTP secret + * @param {string} inputToken - the code the user typed + * @returns {number|null} + */ +function findTotpStep(secret, inputToken) { const normalizedInput = String(inputToken || '').replace(/\s+/g, ''); if (!/^\d{6,8}$/.test(normalizedInput)) { - return false; + return null; } const now = Math.floor(Date.now() / 1000); for (let offset = -TOTP_WINDOW; offset <= TOTP_WINDOW; offset += 1) { - const expected = generateTotpToken( - secret, - now + offset * TOTP_STEP_SECONDS - ); + const unixTimeSeconds = now + offset * TOTP_STEP_SECONDS; + const expected = generateTotpToken(secret, unixTimeSeconds); if (!expected || expected.length !== normalizedInput.length) { continue; } @@ -734,11 +773,47 @@ function verifyTotpToken(secret, inputToken) { Buffer.from(normalizedInput) ) ) { - return true; + return Math.floor(unixTimeSeconds / TOTP_STEP_SECONDS); } } - return false; + return null; +} + +function verifyTotpToken(secret, inputToken) { + return findTotpStep(secret, inputToken) !== null; +} + +// Last accepted TOTP time step per username. RFC 6238 section 5.2: once a +// code has been accepted, the verifier must not accept it again. Kept in +// memory: a restart forgets it, and the window it protects is 90 seconds. +const lastAcceptedTotpSteps = new Map(); + +/** + * Verifies a TOTP code and consumes it, so the same code (or an older one) + * is refused for that user until a newer time step comes up. For actions a + * captured code must not be able to repeat: signing in, enabling and + * disabling MFA. + * + * @param {string} username - the account the code is for + * @param {string} secret - base32 TOTP secret + * @param {string} inputToken - the code the user typed + * @returns {boolean} + */ +function consumeTotpToken(username, secret, inputToken) { + const step = findTotpStep(secret, inputToken); + if (step === null) { + return false; + } + + const key = String(username || '').toLowerCase(); + const lastStep = lastAcceptedTotpSteps.get(key); + if (lastStep !== undefined && step <= lastStep) { + return false; + } + + lastAcceptedTotpSteps.set(key, step); + return true; } function renderLoginView(res, options = {}) { @@ -746,6 +821,7 @@ function renderLoginView(res, options = {}) { error: options.error || null, mfaRequired: Boolean(options.mfaRequired), username: options.username || '', + sso: oidcService.getLoginButton(), }); } @@ -753,6 +829,48 @@ function isMfaEnabledForUser(user) { return Boolean(user && (user.mfa_enabled || user.mfaEnabled)); } +/* Cost the setup wizard hashes the administrator password with. */ +const ADMIN_PASSWORD_BCRYPT_ROUNDS = 15; +const dummyPasswordHashes = new Map(); + +/** + * Spends the same bcrypt work on a failed username lookup as on a real + * password check, so the response time no longer tells an attacker whether + * the username exists (CWE-208). The dummy hash uses the cost of the stored + * account when there is one, because the comparison time grows with it. + * + * @param {*} password - the submitted password + * @returns {Promise} + */ +async function spendPasswordCheckTime(password) { + let rounds = ADMIN_PASSWORD_BCRYPT_ROUNDS; + try { + const users = await documentModel.getUsers(); + const storedHash = users?.[0]?.password; + if (storedHash) { + rounds = bcrypt.getRounds(storedHash); + } + } catch { + // Keep the default cost. + } + + if (!dummyPasswordHashes.has(rounds)) { + dummyPasswordHashes.set( + rounds, + bcrypt.hash(crypto.randomBytes(32).toString('hex'), rounds) + ); + } + + try { + await bcrypt.compare( + String(password ?? ''), + await dummyPasswordHashes.get(rounds) + ); + } catch { + // Only the elapsed time matters here. + } +} + router.get('/login', (req, res) => { //check if a user exists beforehand documentModel.getUsers().then((users) => { @@ -875,7 +993,7 @@ router.post('/login', loginLimiter, async (req, res) => { }); } - if (!verifyTotpToken(mfaSecret, mfaToken)) { + if (!consumeTotpToken(user.username, mfaSecret, mfaToken)) { return renderLoginView(res, { error: 'Invalid authentication code. Please try again.', mfaRequired: true, @@ -907,6 +1025,7 @@ router.post('/login', loginLimiter, async (req, res) => { const user = await documentModel.getUser(username); if (!user || !user.password) { + await spendPasswordCheckTime(password); console.log('[FAILED LOGIN] User not found or invalid data:', username); return renderLoginView(res, { error: 'Invalid credentials', username }); } @@ -3343,8 +3462,33 @@ const DEFAULT_AI_PROVIDER_PRESETS = [ model: 'meta-llama/llama-3.3-70b-instruct', tokenPlaceholder: 'ionos-api-key', }, + { + // No URL: the SDK knows api.anthropic.com, and ANTHROPIC_API_URL is only + // for a proxy in front of it. + id: 'anthropic-default', + label: 'Anthropic (Claude)', + provider: 'anthropic', + apiUrl: '', + model: ANTHROPIC_DEFAULT_MODEL, + tokenPlaceholder: 'sk-ant-...', + }, ]; +/* What the setup wizard and the settings page need to offer the Claude + models and effort levels without a request: the same constants the + provider itself runs with. */ +function getAnthropicFormOptions() { + return { + defaultModel: ANTHROPIC_DEFAULT_MODEL, + suggestedModels: ANTHROPIC_SUGGESTED_MODELS.map((entry) => ({ + id: entry.id, + label: entry.label, + })), + effortLevels: ANTHROPIC_EFFORT_LEVELS.slice(), + defaultMaxTokens: ANTHROPIC_DEFAULT_MAX_TOKENS, + }; +} + function cleanupExpiredSetupMfaChallenges() { const now = Date.now(); for (const [challengeId, challenge] of setupMfaChallenges.entries()) { @@ -3505,6 +3649,13 @@ const ENV_EXPORT_GROUPS = [ 'AZURE_API_KEY', 'AZURE_DEPLOYMENT_NAME', 'AZURE_API_VERSION', + 'ANTHROPIC_API_KEY', + 'ANTHROPIC_API_URL', + 'ANTHROPIC_MODEL', + 'ANTHROPIC_EFFORT', + 'ANTHROPIC_MAX_TOKENS', + 'ANTHROPIC_STRUCTURED_OUTPUT', + 'ANTHROPIC_PROMPT_CACHING', ], }, { @@ -3595,6 +3746,25 @@ const ENV_EXPORT_GROUPS = [ 'CONFIG_SOURCE_MODE', ], }, + { + title: 'Single sign-on (OIDC)', + keys: [ + 'OIDC_ENABLED', + 'OIDC_ISSUER_URL', + 'OIDC_CLIENT_ID', + 'OIDC_CLIENT_SECRET', + 'OIDC_ADMIN_EMAIL', + 'OIDC_REDIRECT_URI', + 'OIDC_SCOPES', + 'OIDC_EMAIL_CLAIM', + 'OIDC_ALLOW_UNVERIFIED_EMAIL', + 'OIDC_TOKEN_AUTH_METHOD', + 'OIDC_BUTTON_TEXT', + 'OIDC_BUTTON_ICON', + 'OIDC_BUTTON_COLOR', + 'OIDC_BUTTON_TEXT_COLOR', + ], + }, { title: 'Maintenance', keys: [ @@ -3668,6 +3838,9 @@ function toEnvPreviewLines(config) { 'AZURE_API_KEY', 'AZURE_DEPLOYMENT_NAME', 'AZURE_API_VERSION', + 'ANTHROPIC_API_KEY', + 'ANTHROPIC_API_URL', + 'ANTHROPIC_MODEL', 'MISTRAL_OCR_ENABLED', 'OCR_PROVIDER', 'OCR_API_URL', @@ -3717,7 +3890,9 @@ async function loadAiProviderPresets() { ), })) .filter((item) => - ['openai', 'ollama', 'custom', 'azure'].includes(item.provider) + ['openai', 'ollama', 'custom', 'azure', 'anthropic'].includes( + item.provider + ) ); } catch (error) { console.warn( @@ -3739,12 +3914,21 @@ function resolveStoredAiToken(aiProvider) { if (provider === 'custom') return process.env.CUSTOM_API_KEY || ''; if (provider === 'openai') return process.env.OPENAI_API_KEY || ''; if (provider === 'azure') return process.env.AZURE_API_KEY || ''; + if (provider === 'anthropic') return process.env.ANTHROPIC_API_KEY || ''; return ''; } -function resolveSettingsAiToken(aiProvider, token) { +function resolveSettingsAiToken(aiProvider, token, apiUrl) { const normalizedToken = String(token || '').trim(); - return normalizedToken || resolveStoredAiToken(aiProvider); + if (normalizedToken) return normalizedToken; + // The saved Claude key only goes to a Claude endpoint already configured. + const provider = String(aiProvider || '') + .trim() + .toLowerCase(); + if (provider === 'anthropic' && !isConfiguredAnthropicHost(apiUrl)) { + return ''; + } + return resolveStoredAiToken(aiProvider); } /* Accepts what the row menu sends (one id) and what the bulk menu sends (a @@ -3796,6 +3980,147 @@ function resolveSettingsQuickstartApiKey(baseUrl, apiKey) { : ''; } +/* Single sign-on fields of the settings page: form key -> OIDC_* key. */ +const OIDC_SETTINGS_FIELDS = { + oidcEnabled: 'OIDC_ENABLED', + oidcIssuerUrl: 'OIDC_ISSUER_URL', + oidcClientId: 'OIDC_CLIENT_ID', + oidcClientSecret: 'OIDC_CLIENT_SECRET', + oidcAdminEmail: 'OIDC_ADMIN_EMAIL', + oidcRedirectUri: 'OIDC_REDIRECT_URI', + oidcScopes: 'OIDC_SCOPES', + oidcEmailClaim: 'OIDC_EMAIL_CLAIM', + oidcAllowUnverifiedEmail: 'OIDC_ALLOW_UNVERIFIED_EMAIL', + oidcTokenAuthMethod: 'OIDC_TOKEN_AUTH_METHOD', + oidcButtonText: 'OIDC_BUTTON_TEXT', + oidcButtonIcon: 'OIDC_BUTTON_ICON', + oidcButtonColor: 'OIDC_BUTTON_COLOR', + oidcButtonTextColor: 'OIDC_BUTTON_TEXT_COLOR', +}; +const OIDC_SWITCH_FIELDS = new Set(['oidcEnabled', 'oidcAllowUnverifiedEmail']); + +/* Turns the single sign-on fields of a settings save into OIDC_* values and + checks the result. A field the body does not carry keeps what is stored, + so a client that predates the section still saves. An empty client secret + keeps the stored one, like every other secret on the page. + + The checks only run when single sign-on ends up switched on, and they look + at what will be in force after the save: a value injected by the container + wins over a typed one, because saveConfig() never writes over it. Switched + off, whatever was typed is saved, so a half-finished setup can be parked. + + Returns { updates } or { error }. */ +function buildOidcSettingsUpdate(body) { + const updates = {}; + for (const [field, envKey] of Object.entries(OIDC_SETTINGS_FIELDS)) { + const raw = body?.[field]; + if (raw === undefined || raw === null) continue; + if (OIDC_SWITCH_FIELDS.has(field)) { + const normalized = String(raw).trim().toLowerCase(); + if (['yes', 'on', 'true', '1'].includes(normalized)) { + updates[envKey] = 'yes'; + } else if (['no', 'off', 'false', '0', ''].includes(normalized)) { + updates[envKey] = 'no'; + } + continue; + } + if (field === 'oidcClientSecret') { + const secret = String(raw).trim(); + if (secret) updates[envKey] = secret; + continue; + } + let value = String(raw).trim(); + if (field === 'oidcScopes') value = value.split(/\s+/).join(' '); + if (field === 'oidcTokenAuthMethod') value = value.toLowerCase(); + if (field === 'oidcButtonText') { + value = value.slice(0, oidcService.MAX_BUTTON_TEXT_LENGTH); + } + updates[envKey] = value; + } + + const effective = (envKey) => + String( + !configFile.isProtectedRuntimeEnvKey(envKey) && envKey in updates + ? updates[envKey] + : process.env[envKey] || '' + ).trim(); + + const enabled = ['yes', 'true', '1', 'on'].includes( + effective('OIDC_ENABLED').toLowerCase() + ); + if (!enabled) { + return { updates }; + } + + const errors = oidcService.getConfigErrors({ + enabled, + issuerUrl: effective('OIDC_ISSUER_URL'), + clientId: effective('OIDC_CLIENT_ID'), + redirectUri: effective('OIDC_REDIRECT_URI'), + adminEmail: effective('OIDC_ADMIN_EMAIL'), + }); + const tokenAuthMethod = effective('OIDC_TOKEN_AUTH_METHOD').toLowerCase(); + if ( + tokenAuthMethod && + !oidcService.TOKEN_AUTH_METHODS.includes(tokenAuthMethod) + ) { + errors.push( + `OIDC_TOKEN_AUTH_METHOD must be empty (automatic) or one of ${oidcService.TOKEN_AUTH_METHODS.join(', ')}` + ); + } + const buttonIcon = effective('OIDC_BUTTON_ICON'); + if ( + buttonIcon && + oidcService.resolveButtonIcon(buttonIcon).type === 'default' + ) { + errors.push( + 'OIDC_BUTTON_ICON must be empty, "none", an https:// URL or a path starting with /' + ); + } + for (const envKey of ['OIDC_BUTTON_COLOR', 'OIDC_BUTTON_TEXT_COLOR']) { + const color = effective(envKey); + if (color && !oidcService.isSafeCssColor(color)) { + errors.push( + `${envKey} must be a CSS colour such as #1f6feb, a colour name, or rgb()/hsl()` + ); + } + } + + if (errors.length > 0) { + return { + error: `Single sign-on is switched on but cannot work: ${errors.join('; ')}. Fix these or switch single sign-on off.`, + }; + } + return { updates }; +} + +/* The origin (scheme, host and port) of a URL, or null. */ +function urlOrigin(value) { + try { + const parsed = new URL(String(value || '').trim()); + return parsed.origin.toLowerCase(); + } catch { + return null; + } +} + +/* The client secret for the single sign-on test. A typed secret is always + used. An empty field means "the stored one", but only for the provider it + was stored for: the page never shows the saved secret, and a test endpoint + that sent it to any issuer the caller names would hand it out again. */ +function resolveOidcTestClientSecret(issuerUrl, clientSecret) { + const typed = String(clientSecret || '').trim(); + if (typed) return typed; + + const requested = urlOrigin(oidcService.normalizeIssuerUrl(issuerUrl)); + const configured = urlOrigin( + oidcService.normalizeIssuerUrl(process.env.OIDC_ISSUER_URL) + ); + return requested && configured && requested === configured + ? String(process.env.OIDC_CLIENT_SECRET || '') + : ''; +} + /* Compares the detection target with the configured AI server. The Quickstart field is written by hand and normalizeBaseUrls() strips /v1 and trailing slashes, so "http://host:1234/v1/" and "http://host:1234" are the same @@ -3830,6 +4155,33 @@ function resolveOcrApiKey(apiKey) { ); } +/* resolveOcrApiKey() for a request that names its OCR provider. Claude OCR + must never receive a stored Mistral key: MISTRAL_API_KEY never belongs to + it, and OCR_API_KEY only when Claude is the OCR provider it was saved for. + An empty answer is fine — setupService falls back to the Anthropic AI key. + `apiUrl` is where the request goes; the stored key is withheld from any + host that is not already configured for Claude. */ +function resolveOcrApiKeyForProvider(provider, apiKey, apiUrl = '') { + const normalizedProvider = String(provider || '') + .trim() + .toLowerCase(); + if (normalizedProvider !== 'anthropic') { + return resolveOcrApiKey(apiKey); + } + + const normalizedApiKey = String(apiKey || '').trim(); + if (normalizedApiKey) { + return normalizedApiKey; + } + + const storedProvider = String(process.env.OCR_PROVIDER || '') + .trim() + .toLowerCase(); + return storedProvider === 'anthropic' && isConfiguredAnthropicHost(apiUrl) + ? process.env.OCR_API_KEY || '' + : ''; +} + async function validatePaperlessConnectionForSetup( paperlessUrl, paperlessToken @@ -3897,7 +4249,7 @@ async function validateAiConnectionForSetup({ if ( !provider || - !['openai', 'ollama', 'custom', 'azure'].includes(provider) + !['openai', 'ollama', 'custom', 'azure', 'anthropic'].includes(provider) ) { return { success: false, @@ -3985,6 +4337,29 @@ async function validateAiConnectionForSetup({ }; } + if (provider === 'anthropic') { + if (!normalizedToken) { + return { + success: false, + message: 'An API key is required for Anthropic.', + }; + } + + // The URL stays optional: empty means api.anthropic.com. + const valid = await setupService.validateAnthropicConfig( + normalizedToken, + normalizedModel || ANTHROPIC_DEFAULT_MODEL, + normalizedApiUrl + ); + + return { + success: valid, + message: valid + ? 'Anthropic API key and model are valid.' + : 'Anthropic test failed. Check the API key, the model and, if set, the proxy URL.', + }; + } + if (!normalizedApiUrl || !normalizedModel) { return { success: false, @@ -4031,6 +4406,9 @@ async function validateOcrConnectionForSetup({ apiKey, model, setupOcrValidationTimeoutMs, + anthropicApiKey, + anthropicApiUrl, + anthropicModel, }) { return setupService.withTemporaryValidationTimeout( setupOcrValidationTimeoutMs, @@ -4072,7 +4450,17 @@ async function validateOcrConnectionForSetup({ provider: normalizedProvider, apiUrl: resolvedApiUrl, apiKey: String(apiKey || '').trim(), - model: String(model || '').trim() || 'mistral-ocr-latest', + // Claude OCR picks its own model when this one is not a Claude model + // (see setupService.resolveAnthropicOcrModel), so the Mistral default + // must not be filled in for it. + model: + String(model || '').trim() || + (normalizedProvider === 'anthropic' ? '' : 'mistral-ocr-latest'), + // The Anthropic AI provider's key, URL and model, which Claude OCR + // falls back to; the environment covers them when they are saved. + anthropicApiKey, + anthropicApiUrl, + anthropicModel, }); return { @@ -4086,6 +4474,21 @@ async function validateOcrConnectionForSetup({ ); } +/* The model-discovery answer for Anthropic. The list is never empty — it + falls back to the suggested models — so the message has to say which list + the picker now shows, or a refused key would look like a working one. */ +function describeAnthropicModelDiscovery(discovery, apiUrl) { + return { + success: true, + models: discovery.models, + live: discovery.live, + resolvedApiUrl: apiUrl, + message: discovery.live + ? `Discovered ${discovery.models.length} model(s).` + : `Could not load the live model list (${discovery.error}). Showing the suggested Claude models instead.`, + }; +} + async function discoverAiModelsForSetup({ aiProvider, apiUrl, @@ -4101,6 +4504,16 @@ async function discoverAiModelsForSetup({ const normalizedApiUrl = String(apiUrl || '').trim(); const normalizedToken = String(token || '').trim(); + if (provider === 'anthropic') { + return describeAnthropicModelDiscovery( + await setupService.discoverAnthropicModels({ + apiKey: normalizedToken, + apiUrl: normalizedApiUrl, + }), + normalizedApiUrl + ); + } + // For local providers, classify models via quickstart detection so // embedding-only models are excluded from the AI model dropdown. if (['custom', 'ollama'].includes(provider) && normalizedApiUrl) { @@ -4162,6 +4575,9 @@ async function discoverOcrModelsForSetup({ apiUrl, apiKey, setupOcrValidationTimeoutMs, + anthropicApiKey, + anthropicApiUrl, + anthropicModel, }) { return setupService.withTemporaryValidationTimeout( setupOcrValidationTimeoutMs, @@ -4172,6 +4588,27 @@ async function discoverOcrModelsForSetup({ const normalizedApiUrl = String(apiUrl || '').trim(); const normalizedApiKey = String(apiKey || '').trim(); + // Claude OCR lists the same models as the AI provider, reached with the + // OCR key or the AI key, and preselects the model OCR would run with. + if (normalizedProvider === 'anthropic') { + const connection = setupService.resolveAnthropicOcrConnection({ + apiKey: normalizedApiKey, + apiUrl: normalizedApiUrl, + anthropicApiKey, + anthropicApiUrl, + }); + return { + ...describeAnthropicModelDiscovery( + await setupService.discoverAnthropicModels(connection), + normalizedApiUrl + ), + suggestedModel: setupService.resolveAnthropicOcrModel( + '', + anthropicModel + ), + }; + } + // For local providers, classify models via quickstart detection so // embedding-only models are excluded from the OCR dropdown // (metadata-first via LM Studio /api/v0/models or Ollama /api/show, @@ -4391,6 +4828,10 @@ router.get('/setup', async (req, res) => { AZURE_API_KEY: process.env.AZURE_API_KEY || '', AZURE_DEPLOYMENT_NAME: process.env.AZURE_DEPLOYMENT_NAME || '', AZURE_API_VERSION: process.env.AZURE_API_VERSION || '', + // The key is dropped by sanitizeConfigForBootstrap() before rendering. + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || '', + ANTHROPIC_API_URL: process.env.ANTHROPIC_API_URL || '', + ANTHROPIC_MODEL: process.env.ANTHROPIC_MODEL || ANTHROPIC_DEFAULT_MODEL, MISTRAL_OCR_ENABLED: process.env.MISTRAL_OCR_ENABLED || 'no', OCR_PROVIDER: process.env.OCR_PROVIDER || 'mistral', OCR_API_URL: process.env.OCR_API_URL || '', @@ -4464,6 +4905,7 @@ router.get('/setup', async (req, res) => { config: sanitizedConfig, success: successMessage, aiProviderPresets, + anthropicOptions: getAnthropicFormOptions(), defaults: { scanInterval: getDefaultScanInterval(), }, @@ -4475,6 +4917,7 @@ router.get('/setup', async (req, res) => { config: {}, error: 'An error occurred while loading the setup page.', aiProviderPresets, + anthropicOptions: getAnthropicFormOptions(), defaults: { scanInterval: getDefaultScanInterval(), }, @@ -4855,6 +5298,9 @@ router.post('/api/setup/ocr/test', express.json(), async (req, res) => { setupOcrValidationTimeoutMs: req.body?.setupOcrValidationTimeoutMs ?? req.body?.setupValidationTimeoutMs, + anthropicApiKey: req.body?.anthropicApiKey, + anthropicApiUrl: req.body?.anthropicApiUrl, + anthropicModel: req.body?.anthropicModel, }); return res.json(validation); @@ -4905,6 +5351,9 @@ router.post('/api/setup/ocr/test', express.json(), async (req, res) => { * type: string * nullable: true * description: Recommended default; absent for providers without model classification + * live: + * type: boolean + * description: anthropic only — false when the live list failed and the suggested Claude models are returned instead * resolvedApiUrl: * type: string * message: @@ -4923,7 +5372,14 @@ router.post('/api/setup/ocr/models', express.json(), async (req, res) => { apiUrl: req.body?.apiUrl, // The wizard used to take the field at face value, so an operator whose // key already sat in the environment was told to supply one. - apiKey: resolveOcrApiKey(req.body?.apiKey), + apiKey: resolveOcrApiKeyForProvider( + req.body?.provider, + req.body?.apiKey, + req.body?.apiUrl || req.body?.anthropicApiUrl + ), + anthropicApiKey: req.body?.anthropicApiKey, + anthropicApiUrl: req.body?.anthropicApiUrl, + anthropicModel: req.body?.anthropicModel, // Same key mismatch as in /api/setup/ocr/test above: the discovery // helper takes setupOcrValidationTimeoutMs. It matters here because // classifying an Ollama catalogue costs one /api/show per model and @@ -5019,13 +5475,16 @@ router.post( * properties: * aiProvider: * type: string - * enum: [openai, ollama, custom, azure] + * enum: [openai, ollama, custom, azure, anthropic] * apiUrl: * type: string + * description: Not needed for anthropic unless a proxy sits in front of api.anthropic.com * token: * type: string + * description: Omit or leave empty to use the stored key of the selected provider * model: * type: string + * description: For anthropic, checked with a models lookup that spends no tokens * setupValidationTimeoutMs: * type: integer * responses: @@ -5043,7 +5502,11 @@ router.post( const validation = await validateAiConnectionForSetup({ aiProvider: req.body?.aiProvider, apiUrl: req.body?.apiUrl, - token: resolveSettingsAiToken(req.body?.aiProvider, req.body?.token), + token: resolveSettingsAiToken( + req.body?.aiProvider, + req.body?.token, + req.body?.apiUrl + ), model: req.body?.model, azureApiVersion: req.body?.azureApiVersion, setupValidationTimeoutMs: req.body?.setupValidationTimeoutMs, @@ -5060,6 +5523,60 @@ router.post( } ); +/** + * @swagger + * /api/settings/ocr/test: + * post: + * summary: Test OCR provider connectivity from the settings page + * description: | + * Runs one OCR request on a generated test image. For `anthropic` the + * key is the OCR key when one is given or stored for Claude OCR, and + * otherwise the Anthropic AI provider's key (`anthropicApiKey`, then + * the stored ANTHROPIC_API_KEY). + * tags: + * - Settings + * security: + * - BearerAuth: [] + * - ApiKeyAuth: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * properties: + * enabled: + * type: string + * enum: ["yes", "no"] + * provider: + * type: string + * enum: [mistral, custom, ollama, anthropic] + * apiUrl: + * type: string + * description: Ignored for mistral. For anthropic an optional proxy URL; empty uses the Anthropic AI provider's URL, then api.anthropic.com + * apiKey: + * type: string + * description: Omit or leave empty to use the stored key + * model: + * type: string + * description: For anthropic, a Claude model; anything else falls back to the AI model + * setupOcrValidationTimeoutMs: + * type: integer + * anthropicApiKey: + * type: string + * description: Anthropic AI key typed but not saved yet; Claude OCR falls back to it + * anthropicApiUrl: + * type: string + * description: Anthropic AI proxy URL typed but not saved yet + * anthropicModel: + * type: string + * description: Anthropic AI model typed but not saved yet + * responses: + * 200: + * description: OCR connectivity result returned + * 401: + * description: Unauthorized + */ router.post( '/api/settings/ocr/test', isAuthenticated, @@ -5070,11 +5587,20 @@ router.post( enabled: req.body?.enabled, provider: req.body?.provider, apiUrl: req.body?.apiUrl, - apiKey: resolveOcrApiKey(req.body?.apiKey), + apiKey: resolveOcrApiKeyForProvider( + req.body?.provider, + req.body?.apiKey, + req.body?.apiUrl || req.body?.anthropicApiUrl + ), model: req.body?.model, setupOcrValidationTimeoutMs: req.body?.setupOcrValidationTimeoutMs ?? req.body?.setupValidationTimeoutMs, + // A key typed into the Anthropic AI fields but not saved yet; the + // stored one comes from the environment. + anthropicApiKey: req.body?.anthropicApiKey, + anthropicApiUrl: req.body?.anthropicApiUrl, + anthropicModel: req.body?.anthropicModel, }); return res.json(validation); @@ -5088,6 +5614,181 @@ router.post( } ); +/** + * @swagger + * /api/settings/oidc/test: + * post: + * summary: Test a single sign-on provider from the settings page + * description: | + * Reads the provider's discovery document, picks the token endpoint auth + * method the login would use, and checks the client credentials by + * redeeming an authorization code that cannot exist: the provider + * answers invalid_client for wrong credentials and invalid_grant for + * right ones. Nobody is signed in and nothing is saved. + * + * An empty `clientSecret` uses the stored OIDC_CLIENT_SECRET, but only + * when the issuer has the same origin (scheme, host, port) as the + * configured OIDC_ISSUER_URL; for any other issuer the test runs + * without a secret. Private network addresses are allowed, since + * identity providers usually run next to the application. + * tags: + * - Settings + * security: + * - BearerAuth: [] + * - ApiKeyAuth: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: + * - issuerUrl + * - clientId + * properties: + * issuerUrl: + * type: string + * description: Issuer URL (http or https); a pasted discovery URL is accepted + * example: "https://auth.example.com/application/o/zettelrobbe/" + * clientId: + * type: string + * example: "zettelrobbe" + * clientSecret: + * type: string + * description: Omit or leave empty to use the stored secret (same issuer origin only) + * example: "" + * tokenAuthMethod: + * type: string + * description: Empty picks the method automatically + * enum: ["", "client_secret_basic", "client_secret_post", "none"] + * example: "" + * responses: + * 200: + * description: | + * Test result. success is false (still HTTP 200) when the provider + * could not be reached, sent no usable discovery document, or refused + * the client credentials; error then says which. + * content: + * application/json: + * schema: + * oneOf: + * - type: object + * properties: + * success: + * type: boolean + * example: true + * data: + * type: object + * properties: + * issuer: + * type: string + * description: The issuer the provider advertises + * example: "https://auth.example.com/application/o/zettelrobbe/" + * tokenAuthMethods: + * type: array + * description: Token endpoint auth methods the provider advertises (empty when it lists none) + * items: + * type: string + * example: ["client_secret_basic", "client_secret_post"] + * chosenAuthMethod: + * type: string + * description: The method a login would use + * example: "client_secret_basic" + * pkce: + * type: boolean + * description: Whether the provider advertises PKCE with S256 + * example: true + * message: + * type: string + * example: "The provider answered and accepted the client." + * - $ref: '#/components/schemas/Error' + * 400: + * description: Invalid input (issuer not an http(s) URL, empty client ID, unknown auth method, or a method that needs a secret without one) + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * 302: + * description: Not signed in; redirected to /login like every other route behind the authentication guard + */ +router.post( + '/api/settings/oidc/test', + isAuthenticated, + express.json(), + async (req, res) => { + try { + const issuerUrl = oidcService.normalizeIssuerUrl(req.body?.issuerUrl); + const clientId = String(req.body?.clientId || '').trim(); + const tokenAuthMethod = String(req.body?.tokenAuthMethod || '') + .trim() + .toLowerCase(); + + if (!oidcService.isAbsoluteHttpUrl(issuerUrl)) { + return res.status(400).json({ + success: false, + error: 'The issuer URL must start with https:// or http://.', + }); + } + if (!clientId) { + return res + .status(400) + .json({ success: false, error: 'The client ID is missing.' }); + } + if ( + tokenAuthMethod && + !oidcService.TOKEN_AUTH_METHODS.includes(tokenAuthMethod) + ) { + return res.status(400).json({ + success: false, + error: `Unknown token endpoint auth method. Use one of: ${oidcService.TOKEN_AUTH_METHODS.join(', ')}, or leave it empty.`, + }); + } + + const clientSecret = resolveOidcTestClientSecret( + issuerUrl, + req.body?.clientSecret + ); + if ( + !clientSecret && + ['client_secret_basic', 'client_secret_post'].includes(tokenAuthMethod) + ) { + return res.status(400).json({ + success: false, + error: `${tokenAuthMethod} needs a client secret. Enter it, or choose another auth method.`, + }); + } + + let data; + try { + data = await oidcService.testConnection({ + issuerUrl, + clientId, + clientSecret, + tokenAuthMethod, + }); + } catch (error) { + console.warn( + '[WARN] POST /api/settings/oidc/test failed:', + error.message + ); + return res.json({ success: false, error: error.message }); + } + + return res.json({ + success: true, + data, + message: 'The provider answered and accepted the client.', + }); + } catch (error) { + console.error('[ERROR] POST /api/settings/oidc/test:', error); + return res.status(500).json({ + success: false, + error: 'Could not test single sign-on.', + }); + } + } +); + router.post( '/api/settings/ai/models', isAuthenticated, @@ -5097,7 +5798,11 @@ router.post( const result = await discoverAiModelsForSetup({ aiProvider: req.body?.aiProvider, apiUrl: req.body?.apiUrl, - token: resolveSettingsAiToken(req.body?.aiProvider, req.body?.token), + token: resolveSettingsAiToken( + req.body?.aiProvider, + req.body?.token, + req.body?.apiUrl + ), setupValidationTimeoutMs: req.body?.setupValidationTimeoutMs, }); @@ -5154,14 +5859,23 @@ router.get('/api/settings/ai/presets', isAuthenticated, async (_req, res) => { * properties: * provider: * type: string - * enum: [mistral, custom, ollama] + * enum: [mistral, custom, ollama, anthropic] * apiUrl: * type: string * apiKey: * type: string - * description: Omit or leave empty to use the stored OCR_API_KEY / MISTRAL_API_KEY + * description: Omit or leave empty to use the stored OCR_API_KEY / MISTRAL_API_KEY (for anthropic, the stored Claude OCR key, then the Anthropic AI key) * setupOcrValidationTimeoutMs: * type: integer + * anthropicApiKey: + * type: string + * description: Anthropic AI key typed but not saved yet; used by anthropic when no OCR key is set + * anthropicApiUrl: + * type: string + * description: Anthropic AI proxy URL typed but not saved yet + * anthropicModel: + * type: string + * description: Anthropic AI model; anthropic returns it as suggestedModel * responses: * 200: * description: OCR model list returned @@ -5186,6 +5900,9 @@ router.get('/api/settings/ai/presets', isAuthenticated, async (_req, res) => { * type: string * nullable: true * description: Recommended default; absent for providers without model classification + * live: + * type: boolean + * description: anthropic only — false when the live list failed and the suggested Claude models are returned instead * resolvedApiUrl: * type: string * message: @@ -5204,7 +5921,14 @@ router.post( const result = await discoverOcrModelsForSetup({ provider: req.body?.provider, apiUrl: req.body?.apiUrl, - apiKey: resolveOcrApiKey(req.body?.apiKey), + apiKey: resolveOcrApiKeyForProvider( + req.body?.provider, + req.body?.apiKey, + req.body?.apiUrl || req.body?.anthropicApiUrl + ), + anthropicApiKey: req.body?.anthropicApiKey, + anthropicApiUrl: req.body?.anthropicApiUrl, + anthropicModel: req.body?.anthropicModel, setupOcrValidationTimeoutMs: req.body?.setupOcrValidationTimeoutMs ?? req.body?.setupValidationTimeoutMs, @@ -5408,15 +6132,25 @@ router.post('/api/setup/complete', express.json(), async (req, res) => { .trim() .toLowerCase(); const ocrApiUrlRaw = String(req.body?.ocrApiUrl || '').trim(); + // Mistral OCR has no URL field. For Claude OCR the field is an optional + // proxy URL: empty falls back to the Anthropic AI provider's URL, and + // that to api.anthropic.com. The wizard empties the field when Claude is + // picked, so a value here was typed for it. const ocrApiUrl = ocrProvider === 'mistral' ? '' : ocrApiUrlRaw; const ocrApiKey = String( req.body?.ocrApiKey || req.body?.mistralApiKey || '' ).trim(); + const anthropicAiSelected = aiProvider === 'anthropic'; const mistralOcrModel = - String(req.body?.mistralOcrModel || 'mistral-ocr-latest').trim() || - 'mistral-ocr-latest'; + ocrProvider === 'anthropic' + ? setupService.resolveAnthropicOcrModel( + req.body?.mistralOcrModel, + anthropicAiSelected ? aiModel : '' + ) + : String(req.body?.mistralOcrModel || 'mistral-ocr-latest').trim() || + 'mistral-ocr-latest'; - if (!['mistral', 'custom', 'ollama'].includes(ocrProvider)) { + if (!['mistral', 'custom', 'ollama', 'anthropic'].includes(ocrProvider)) { return res.status(400).json({ success: false, error: 'A valid OCR provider is required.', @@ -5431,6 +6165,20 @@ router.post('/api/setup/complete', express.json(), async (req, res) => { }); } + if ( + mistralOcrEnabled && + ocrProvider === 'anthropic' && + !ocrApiKey && + !(anthropicAiSelected && aiToken) && + !process.env.ANTHROPIC_API_KEY + ) { + return res.status(400).json({ + success: false, + error: + 'Claude OCR needs an Anthropic API key: enter one for OCR, or choose Anthropic as the AI provider.', + }); + } + if (!adminUsername || !adminPassword) { return res.status(400).json({ success: false, @@ -5462,7 +6210,7 @@ router.post('/api/setup/complete', express.json(), async (req, res) => { if ( !aiProvider || - !['openai', 'ollama', 'custom', 'azure'].includes(aiProvider) + !['openai', 'ollama', 'custom', 'azure', 'anthropic'].includes(aiProvider) ) { return res.status(400).json({ success: false, @@ -5509,6 +6257,9 @@ router.post('/api/setup/complete', express.json(), async (req, res) => { apiKey: ocrApiKey, model: mistralOcrModel, setupOcrValidationTimeoutMs, + anthropicApiKey: anthropicAiSelected ? aiToken : '', + anthropicApiUrl: anthropicAiSelected ? aiApiUrl : '', + anthropicModel: anthropicAiSelected ? aiModel : '', }); if (!ocrValidation.success) { @@ -5554,7 +6305,8 @@ router.post('/api/setup/complete', express.json(), async (req, res) => { OCR_PROVIDER: ocrProvider, OCR_API_URL: ocrApiUrl, OCR_API_KEY: ocrApiKey, - MISTRAL_API_KEY: ocrApiKey, + // An Anthropic key must not land where Mistral OCR would read it. + MISTRAL_API_KEY: ocrProvider === 'anthropic' ? '' : ocrApiKey, MISTRAL_OCR_MODEL: mistralOcrModel, SETUP_VALIDATION_TIMEOUT_MS: String(setupValidationTimeoutMs), SETUP_OCR_VALIDATION_TIMEOUT_MS: String(setupOcrValidationTimeoutMs), @@ -5572,6 +6324,11 @@ router.post('/api/setup/complete', express.json(), async (req, res) => { finalConfig.AZURE_API_KEY = aiToken; finalConfig.AZURE_DEPLOYMENT_NAME = aiModel; finalConfig.AZURE_API_VERSION = aiAzureApiVersion; + } else if (aiProvider === 'anthropic') { + finalConfig.ANTHROPIC_API_KEY = aiToken; + finalConfig.ANTHROPIC_MODEL = aiModel || ANTHROPIC_DEFAULT_MODEL; + // Only a proxy needs one; leaving it out keeps api.anthropic.com. + if (aiApiUrl) finalConfig.ANTHROPIC_API_URL = aiApiUrl; } else { finalConfig.CUSTOM_BASE_URL = aiApiUrl; finalConfig.CUSTOM_API_KEY = aiToken; @@ -6617,6 +7374,18 @@ router.get('/settings', async (req, res) => { AZURE_API_KEY: process.env.AZURE_API_KEY || '', AZURE_DEPLOYMENT_NAME: process.env.AZURE_DEPLOYMENT_NAME || '', AZURE_API_VERSION: process.env.AZURE_API_VERSION || '', + // Same defaults as config.anthropic, so a fresh instance shows the values + // the provider would actually run with. The key is blanked below and only + // reported through configuredSecrets. + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || '', + ANTHROPIC_API_URL: process.env.ANTHROPIC_API_URL || '', + ANTHROPIC_MODEL: process.env.ANTHROPIC_MODEL || ANTHROPIC_DEFAULT_MODEL, + ANTHROPIC_EFFORT: process.env.ANTHROPIC_EFFORT || '', + ANTHROPIC_MAX_TOKENS: + process.env.ANTHROPIC_MAX_TOKENS || String(ANTHROPIC_DEFAULT_MAX_TOKENS), + ANTHROPIC_STRUCTURED_OUTPUT: + process.env.ANTHROPIC_STRUCTURED_OUTPUT || 'yes', + ANTHROPIC_PROMPT_CACHING: process.env.ANTHROPIC_PROMPT_CACHING || 'yes', RESTRICT_TO_EXISTING_TAGS: process.env.RESTRICT_TO_EXISTING_TAGS || 'no', RESTRICT_TO_EXISTING_CORRESPONDENTS: process.env.RESTRICT_TO_EXISTING_CORRESPONDENTS || 'no', @@ -6699,6 +7468,23 @@ router.get('/settings', async (req, res) => { PAPERLESS_AI_PORT: process.env.PAPERLESS_AI_PORT || '3000', LOG_LEVEL: process.env.LOG_LEVEL || 'info', DATE_FORMAT: process.env.DATE_FORMAT || 'DD.MM.YYYY', + // Single sign-on. Defaults mirror config.getOidcConfig(); the settings + // page lists them so values injected by the container show as locked. + OIDC_ENABLED: process.env.OIDC_ENABLED || 'no', + OIDC_ISSUER_URL: process.env.OIDC_ISSUER_URL || '', + OIDC_CLIENT_ID: process.env.OIDC_CLIENT_ID || '', + OIDC_CLIENT_SECRET: process.env.OIDC_CLIENT_SECRET || '', + OIDC_ADMIN_EMAIL: process.env.OIDC_ADMIN_EMAIL || '', + OIDC_REDIRECT_URI: process.env.OIDC_REDIRECT_URI || '', + OIDC_SCOPES: process.env.OIDC_SCOPES || 'openid profile email', + OIDC_EMAIL_CLAIM: process.env.OIDC_EMAIL_CLAIM || 'email', + OIDC_ALLOW_UNVERIFIED_EMAIL: + process.env.OIDC_ALLOW_UNVERIFIED_EMAIL || 'no', + OIDC_TOKEN_AUTH_METHOD: process.env.OIDC_TOKEN_AUTH_METHOD || '', + OIDC_BUTTON_TEXT: process.env.OIDC_BUTTON_TEXT || '', + OIDC_BUTTON_ICON: process.env.OIDC_BUTTON_ICON || '', + OIDC_BUTTON_COLOR: process.env.OIDC_BUTTON_COLOR || '', + OIDC_BUTTON_TEXT_COLOR: process.env.OIDC_BUTTON_TEXT_COLOR || '', }; if (isConfigured) { @@ -6778,7 +7564,17 @@ router.get('/settings', async (req, res) => { lockedEnvKeys, lockedEnvDetails, aiProviderPresets, + anthropicOptions: getAnthropicFormOptions(), mfaSettings, + // Single sign-on: the callback URL as this request sees it (what to + // register at the provider when OIDC_REDIRECT_URI is empty), and what the + // running configuration lacks. enabled is the switch, not whether the + // button is offered: that also needs errors to be empty. + oidcDetectedRedirectUri: oidcService.deriveRedirectUri(req), + oidcStatus: { + enabled: oidcService.getSettings().enabled, + errors: oidcService.getConfigErrors(), + }, changelogReleases: changelog.releases, // No banner for the normal case: "already configured" was shown on every // visit to a configured instance, i.e. always, and said nothing. @@ -7150,7 +7946,7 @@ router.post( }); } - if (!verifyTotpToken(payload.secret, token)) { + if (!consumeTotpToken(username, payload.secret, token)) { return res .status(400) .json({ success: false, error: 'Invalid authentication code.' }); @@ -7327,7 +8123,7 @@ router.post( .json({ success: false, error: 'MFA is not enabled for this user.' }); } - if (!verifyTotpToken(user.mfa_secret, token)) { + if (!consumeTotpToken(username, user.mfa_secret, token)) { return res .status(400) .json({ success: false, error: 'Invalid authentication code.' }); @@ -7536,6 +8332,26 @@ router.post('/manual/analyze', express.json(), async (req, res) => { id || [] ); return res.json(analyzeDocument); + } else if (process.env.AI_PROVIDER === 'anthropic') { + // Through the factory, which owns the Claude service like every scan + // does. Its tokens are billed like OpenAI's, so they are recorded too. + const analyzeDocument = + await AIServiceFactory.getService().analyzeDocument( + content, + existingTagsList, + existingCorrespondentList, + existingDocumentTypesList, + id || [] + ); + if (analyzeDocument.metrics && id) { + await documentModel.addOpenAIMetrics( + id, + analyzeDocument.metrics.promptTokens, + analyzeDocument.metrics.completionTokens, + analyzeDocument.metrics.totalTokens + ); + } + return res.json(analyzeDocument); } else { return res.status(500).json({ error: 'AI provider not configured' }); } @@ -7752,6 +8568,12 @@ function buildScannerHealthSnapshot() { * When automatic processing is switched off via * `DISABLE_AUTOMATIC_PROCESSING=yes`, a missing scan loop is expected and * never reported as degraded. + * + * Without authentication the response carries only `status` and + * `database`, with the same HTTP status codes. `message`, `scanner` and + * `paperless` hold raw error messages with internal host names and + * ports, so they are returned only with a session (cookie or bearer + * token) or the `x-api-key` header. * tags: * - System * responses: @@ -7798,8 +8620,22 @@ router.get('/health', async (req, res) => { const snapshot = buildScannerHealthSnapshot(); const degraded = snapshot.scanner.degraded; + const status = degraded ? 'degraded' : 'healthy'; + + // /health is public for container healthchecks and uptime monitors, which + // only need the status and the HTTP code. The scanner and Paperless-ngx + // details carry raw error messages ("connect ECONNREFUSED 172.18.0.2:8000") + // and the schedule, so only an authenticated caller gets them. + if (!isAuthenticatedCaller(req)) { + const summary = { status, database: 'ok' }; + if (degraded && scanHealthService.strictHealthEnabled) { + return res.status(503).json(summary); + } + return res.json(summary); + } + const payload = { - status: degraded ? 'degraded' : 'healthy', + status, database: 'ok', ...snapshot, }; @@ -7819,7 +8655,9 @@ router.get('/health', async (req, res) => { console.error('Health check failed:', error); res.status(500).json({ status: 'error', - message: error.message, + message: isAuthenticatedCaller(req) + ? error.message + : 'Health check failed', }); } }); @@ -7863,7 +8701,7 @@ router.get('/health', async (req, res) => { * aiProvider: * type: string * description: Selected AI provider for document analysis - * enum: ["openai", "ollama", "custom", "azure"] + * enum: ["openai", "ollama", "custom", "azure", "anthropic"] * example: "openai" * openaiKey: * type: string @@ -7897,6 +8735,38 @@ router.get('/health', async (req, res) => { * type: string * description: Model name for custom LLM provider * example: "custom-model" + * anthropicApiKey: + * type: string + * description: API key for Anthropic (required when aiProvider is 'anthropic' and none is stored; leave empty to keep the configured value). Saved only with aiProvider 'anthropic'; the anthropic* fields below likewise. + * example: "sk-ant-api03-abc123" + * anthropicModel: + * type: string + * description: Claude model for analysis; empty keeps the configured one + * example: "claude-sonnet-5-5" + * anthropicEffort: + * type: string + * description: Effort level; empty leaves it to the model. Rejected with 400 when it is not one of the listed levels. + * enum: ["", "low", "medium", "high", "xhigh", "max"] + * example: "" + * anthropicMaxTokens: + * type: integer + * minimum: 1 + * description: Completion cap per request, thinking included. Rejected with 400 when it is not a positive whole number; empty keeps the configured value. + * example: 16000 + * anthropicStructuredOutput: + * type: string + * description: Constrain the analysis answer to a JSON schema (yes/no) + * enum: ["yes", "no"] + * example: "yes" + * anthropicPromptCaching: + * type: string + * description: Mark the system prompt cacheable (yes/no) + * enum: ["yes", "no"] + * example: "yes" + * anthropicApiUrl: + * type: string + * description: Only for a proxy; empty uses api.anthropic.com + * example: "" * scanInterval: * type: number * description: Interval in minutes for scanning new documents @@ -8086,6 +8956,11 @@ router.get('/health', async (req, res) => { * maximum: 100 * description: How many entries the vocabulary proposed from the tag names should aim at, types and topics together (5-100, default 25). Out-of-range values are clamped, a non-numeric value keeps the current setting. * example: 25 + * ocrProvider: + * type: string + * description: OCR fallback provider. For anthropic (Claude OCR) an empty ocrApiKey uses the Anthropic AI key, and ocrApiUrl is an optional proxy URL; a key or URL stored for another provider is cleared when switching to it. + * enum: ["mistral", "custom", "ollama", "anthropic"] + * example: "mistral" * ocrAutoProcessEnabled: * type: string * description: Process queued OCR documents automatically (yes/no) @@ -8107,6 +8982,72 @@ router.get('/health', async (req, res) => { * description: How every date in the web interface is rendered * enum: ["DD.MM.YYYY", "YYYY-MM-DD"] * example: "DD.MM.YYYY" + * oidcEnabled: + * type: string + * description: | + * Single sign-on switch (OIDC_ENABLED). Every oidc* field is optional; one the body + * leaves out keeps the stored value. When single sign-on ends up switched on, the save + * is refused with 400 unless the issuer URL is an http(s) URL, the client ID is set, + * the admin e-mail is valid, the redirect URI is empty or an absolute http(s) URL, and + * the auth method, button icon and colours are valid. Values injected by the container + * are never overwritten. + * enum: ["yes", "no"] + * example: "no" + * oidcIssuerUrl: + * type: string + * description: Issuer URL of the OpenID Connect provider (OIDC_ISSUER_URL); a pasted discovery URL is accepted + * example: "https://auth.example.com/application/o/zettelrobbe/" + * oidcClientId: + * type: string + * description: Client ID registered at the provider (OIDC_CLIENT_ID) + * example: "zettelrobbe" + * oidcClientSecret: + * type: string + * description: Client secret (OIDC_CLIENT_SECRET); leave empty to keep the stored one + * example: "" + * oidcAdminEmail: + * type: string + * description: E-mail address of the identity that signs in as the administrator (OIDC_ADMIN_EMAIL) + * example: "admin@example.com" + * oidcRedirectUri: + * type: string + * description: Callback URL registered at the provider (OIDC_REDIRECT_URI); empty derives it from the request + * example: "" + * oidcScopes: + * type: string + * description: Space-separated scopes (OIDC_SCOPES); openid is always added + * example: "openid profile email" + * oidcEmailClaim: + * type: string + * description: Claim that carries the e-mail address (OIDC_EMAIL_CLAIM) + * example: "email" + * oidcAllowUnverifiedEmail: + * type: string + * description: Accept an e-mail address the provider marks unverified (OIDC_ALLOW_UNVERIFIED_EMAIL) + * enum: ["yes", "no"] + * example: "no" + * oidcTokenAuthMethod: + * type: string + * description: Token endpoint auth method (OIDC_TOKEN_AUTH_METHOD); empty picks it automatically + * enum: ["", "client_secret_basic", "client_secret_post", "none"] + * example: "" + * oidcButtonText: + * type: string + * maxLength: 60 + * description: Label of the login button (OIDC_BUTTON_TEXT); longer text is cut to 60 characters + * example: "Sign in with Authentik" + * oidcButtonIcon: + * type: string + * description: Login button icon (OIDC_BUTTON_ICON); empty for the built-in icon, "none", an https URL or a path on this server + * example: "" + * oidcButtonColor: + * type: string + * description: Login button background (OIDC_BUTTON_COLOR); hex, colour name, rgb() or hsl() + * example: "#fd4b2d" + * oidcButtonTextColor: + * type: string + * description: Login button text colour (OIDC_BUTTON_TEXT_COLOR) + * example: "#ffffff" * responses: * 200: * description: Settings updated successfully @@ -8212,6 +9153,13 @@ router.post('/settings', express.json(), async (req, res) => { azureApiKey, azureDeploymentName, azureApiVersion, + anthropicApiKey, + anthropicModel, + anthropicEffort, + anthropicMaxTokens, + anthropicStructuredOutput, + anthropicPromptCaching, + anthropicApiUrl, tagCacheTTL, mistralOcrEnabled, ocrProvider, @@ -8318,6 +9266,16 @@ router.post('/settings', express.json(), async (req, res) => { AZURE_API_KEY: process.env.AZURE_API_KEY || '', AZURE_DEPLOYMENT_NAME: process.env.AZURE_DEPLOYMENT_NAME || '', AZURE_API_VERSION: process.env.AZURE_API_VERSION || '', + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || '', + ANTHROPIC_API_URL: process.env.ANTHROPIC_API_URL || '', + ANTHROPIC_MODEL: process.env.ANTHROPIC_MODEL || ANTHROPIC_DEFAULT_MODEL, + ANTHROPIC_EFFORT: process.env.ANTHROPIC_EFFORT || '', + ANTHROPIC_MAX_TOKENS: + process.env.ANTHROPIC_MAX_TOKENS || + String(ANTHROPIC_DEFAULT_MAX_TOKENS), + ANTHROPIC_STRUCTURED_OUTPUT: + process.env.ANTHROPIC_STRUCTURED_OUTPUT || 'yes', + ANTHROPIC_PROMPT_CACHING: process.env.ANTHROPIC_PROMPT_CACHING || 'yes', RESTRICT_TO_EXISTING_TAGS: process.env.RESTRICT_TO_EXISTING_TAGS || 'no', RESTRICT_TO_EXISTING_CORRESPONDENTS: process.env.RESTRICT_TO_EXISTING_CORRESPONDENTS || 'no', @@ -8396,18 +9354,42 @@ router.post('/settings', express.json(), async (req, res) => { const effectiveAzureApiKey = hasAzureApiKeyInput ? azureApiKey.trim() : currentConfig.AZURE_API_KEY; + const hasAnthropicApiKeyInput = hasValue(anthropicApiKey); + const effectiveAnthropicApiKey = hasAnthropicApiKeyInput + ? anthropicApiKey.trim() + : currentConfig.ANTHROPIC_API_KEY; const normalizedOcrApiKeyInput = hasValue(ocrApiKey) ? String(ocrApiKey).trim() : String(mistralApiKey || '').trim(); const hasOcrApiKeyInput = hasValue(normalizedOcrApiKeyInput); - const effectiveOcrApiKey = hasOcrApiKeyInput - ? normalizedOcrApiKeyInput - : currentConfig.OCR_API_KEY || currentConfig.MISTRAL_API_KEY || ''; const normalizedOcrProvider = String( ocrProvider || currentConfig.OCR_PROVIDER || 'mistral' ) .trim() .toLowerCase(); + // A stored OCR key belongs to the provider it was saved for. Claude OCR + // reads OCR_API_KEY before the Anthropic AI key, and Mistral OCR reads it + // before MISTRAL_API_KEY, so a key left behind when the provider crosses + // to or from Claude would be sent to the wrong vendor (a 401 at best). + // Such a key is cleared below unless a new one is typed; the same goes + // for a URL that would turn into a Claude proxy. + const ocrCrossesClaude = + (normalizedOcrProvider === 'anthropic') !== + (String(currentConfig.OCR_PROVIDER || 'mistral') + .trim() + .toLowerCase() === + 'anthropic'); + let storedOcrApiKey; + if (normalizedOcrProvider === 'anthropic') { + storedOcrApiKey = ocrCrossesClaude ? '' : currentConfig.OCR_API_KEY; + } else { + storedOcrApiKey = ocrCrossesClaude + ? currentConfig.MISTRAL_API_KEY + : currentConfig.OCR_API_KEY || currentConfig.MISTRAL_API_KEY; + } + const effectiveOcrApiKey = hasOcrApiKeyInput + ? normalizedOcrApiKeyInput + : storedOcrApiKey || ''; const effectiveOcrEnabled = hasValue(mistralOcrEnabled) ? String(mistralOcrEnabled).trim().toLowerCase() : String(currentConfig.MISTRAL_OCR_ENABLED || 'no') @@ -8421,9 +9403,14 @@ router.post('/settings', express.json(), async (req, res) => { 30000 ); - if (!['mistral', 'custom', 'ollama'].includes(normalizedOcrProvider)) { + if ( + !['mistral', 'custom', 'ollama', 'anthropic'].includes( + normalizedOcrProvider + ) + ) { return res.status(400).json({ - error: 'Invalid OCR provider. Allowed values are mistral and custom.', + error: + 'Invalid OCR provider. Allowed values are mistral, custom and anthropic.', }); } @@ -8438,6 +9425,27 @@ router.post('/settings', express.json(), async (req, res) => { }); } + // Claude OCR may borrow the Anthropic AI key, but only one that is saved: + // a key typed into the hidden Anthropic block while another AI provider is + // selected is not persisted (see the provider branch below). + const anthropicAiKeyForOcr = + String(aiProvider || currentConfig.AI_PROVIDER) + .trim() + .toLowerCase() === 'anthropic' + ? effectiveAnthropicApiKey + : currentConfig.ANTHROPIC_API_KEY; + if ( + effectiveOcrEnabled === 'yes' && + normalizedOcrProvider === 'anthropic' && + !effectiveOcrApiKey && + !anthropicAiKeyForOcr + ) { + return res.status(400).json({ + error: + 'Claude OCR needs an Anthropic API key: enter one for OCR, or configure the Anthropic AI provider.', + }); + } + // Saving deliberately runs no live OCR connection test — only the value // checks above. Use the "Test OCR Connection" button (/api/settings/ocr/test) // to verify connectivity on demand. @@ -8661,6 +9669,99 @@ router.post('/settings', express.json(), async (req, res) => { if (azureDeploymentName) updatedConfig.AZURE_DEPLOYMENT_NAME = azureDeploymentName; if (azureApiVersion) updatedConfig.AZURE_API_VERSION = azureApiVersion; + } else if (selectedAiProvider === 'anthropic') { + if (!effectiveAnthropicApiKey) { + return res.status(400).json({ + error: + 'Anthropic API key is required when the Anthropic provider is selected.', + }); + } + + // Empty is a value of its own here: it leaves the effort to the + // model. Anything else must be a level the provider knows, because + // the API rejects unknown ones on every request of a scan. + const normalizedAnthropicEffort = + anthropicEffort === undefined + ? currentConfig.ANTHROPIC_EFFORT + : String(anthropicEffort ?? '') + .trim() + .toLowerCase(); + if ( + normalizedAnthropicEffort && + !ANTHROPIC_EFFORT_LEVELS.includes(normalizedAnthropicEffort) + ) { + return res.status(400).json({ + error: `Invalid Anthropic effort. Allowed values are ${ANTHROPIC_EFFORT_LEVELS.join(', ')}, or empty for the model default.`, + }); + } + + // Same strictness as Token Limit: "16k" must not be read as 16. + const normalizedAnthropicMaxTokens = String( + anthropicMaxTokens ?? '' + ).trim(); + if ( + normalizedAnthropicMaxTokens && + (!/^\d+$/.test(normalizedAnthropicMaxTokens) || + Number(normalizedAnthropicMaxTokens) < 1) + ) { + return res.status(400).json({ + error: + 'Invalid Anthropic max tokens. Expected a positive whole number.', + }); + } + + const normalizedAnthropicApiUrl = + typeof anthropicApiUrl === 'string' ? anthropicApiUrl.trim() : null; + if (normalizedAnthropicApiUrl) { + let parsedAnthropicApiUrl = null; + try { + parsedAnthropicApiUrl = new URL(normalizedAnthropicApiUrl); + } catch { + parsedAnthropicApiUrl = null; + } + if ( + !parsedAnthropicApiUrl || + !['http:', 'https:'].includes(parsedAnthropicApiUrl.protocol) + ) { + return res.status(400).json({ + error: + 'Invalid Anthropic API URL. Expected an http(s) URL, or leave it empty for api.anthropic.com.', + }); + } + } + + // The switches arrive as the hidden yes/no inputs of the + // settings-switch partial; anything else keeps the stored value. + const anthropicSwitch = (rawValue, fallbackValue) => { + const normalizedValue = String(rawValue ?? '') + .trim() + .toLowerCase(); + return ['yes', 'no'].includes(normalizedValue) + ? normalizedValue + : String(fallbackValue); + }; + + if (hasAnthropicApiKeyInput) + updatedConfig.ANTHROPIC_API_KEY = effectiveAnthropicApiKey; + updatedConfig.ANTHROPIC_MODEL = + String(anthropicModel || '').trim() || + currentConfig.ANTHROPIC_MODEL || + ANTHROPIC_DEFAULT_MODEL; + updatedConfig.ANTHROPIC_EFFORT = normalizedAnthropicEffort; + if (normalizedAnthropicMaxTokens) + updatedConfig.ANTHROPIC_MAX_TOKENS = String( + Number(normalizedAnthropicMaxTokens) + ); + updatedConfig.ANTHROPIC_STRUCTURED_OUTPUT = anthropicSwitch( + anthropicStructuredOutput, + currentConfig.ANTHROPIC_STRUCTURED_OUTPUT + ); + updatedConfig.ANTHROPIC_PROMPT_CACHING = anthropicSwitch( + anthropicPromptCaching, + currentConfig.ANTHROPIC_PROMPT_CACHING + ); + if (normalizedAnthropicApiUrl !== null) + updatedConfig.ANTHROPIC_API_URL = normalizedAnthropicApiUrl; } } @@ -8940,16 +10041,43 @@ router.post('/settings', express.json(), async (req, res) => { updatedConfig.MISTRAL_OCR_ENABLED = mistralOcrEnabled; if (ocrProvider) updatedConfig.OCR_PROVIDER = String(ocrProvider).trim().toLowerCase(); + const submittedOcrApiUrl = + typeof ocrApiUrl === 'string' ? ocrApiUrl.trim() : null; if (normalizedOcrProvider === 'mistral') { updatedConfig.OCR_API_URL = ''; - } else if (typeof ocrApiUrl === 'string') { - updatedConfig.OCR_API_URL = ocrApiUrl.trim(); + } else if (normalizedOcrProvider === 'anthropic' && ocrCrossesClaude) { + // For Claude OCR the URL is an optional proxy. The one stored for the + // previous provider (an auto-detected Ollama URL, say) would be used as + // one, so only a URL that differs from it counts as typed for Claude; + // empty falls back to ANTHROPIC_API_URL. + updatedConfig.OCR_API_URL = + submittedOcrApiUrl && submittedOcrApiUrl !== currentConfig.OCR_API_URL + ? submittedOcrApiUrl + : ''; + } else if (submittedOcrApiUrl !== null) { + updatedConfig.OCR_API_URL = submittedOcrApiUrl; } if (hasOcrApiKeyInput) { updatedConfig.OCR_API_KEY = effectiveOcrApiKey; - updatedConfig.MISTRAL_API_KEY = effectiveOcrApiKey; + // An Anthropic key must not land where Mistral OCR would read it. + if (normalizedOcrProvider !== 'anthropic') { + updatedConfig.MISTRAL_API_KEY = effectiveOcrApiKey; + } + } else if (ocrCrossesClaude) { + updatedConfig.OCR_API_KEY = ''; + } + if (normalizedOcrProvider === 'anthropic') { + // Stored as the model Claude OCR will run with, so the page shows it + // instead of a leftover Mistral model name. + if (mistralOcrModel || ocrCrossesClaude) { + updatedConfig.MISTRAL_OCR_MODEL = setupService.resolveAnthropicOcrModel( + mistralOcrModel, + updatedConfig.ANTHROPIC_MODEL || currentConfig.ANTHROPIC_MODEL + ); + } + } else if (mistralOcrModel) { + updatedConfig.MISTRAL_OCR_MODEL = mistralOcrModel; } - if (mistralOcrModel) updatedConfig.MISTRAL_OCR_MODEL = mistralOcrModel; if (typeof ocrPdfRenderEnabled === 'string') { const normalizedPdfRenderEnabled = ocrPdfRenderEnabled .trim() @@ -9088,6 +10216,14 @@ router.post('/settings', express.json(), async (req, res) => { } } + // Single sign-on. Checked last among the fields so an incomplete provider + // is reported only when everything else on the page is fine. + const oidcUpdate = buildOidcSettingsUpdate(req.body); + if (oidcUpdate.error) { + return res.status(400).json({ success: false, error: oidcUpdate.error }); + } + Object.assign(updatedConfig, oidcUpdate.updates); + // Handle API key let apiToken = configFile.getApiKey ? configFile.getApiKey() @@ -9430,6 +10566,9 @@ router.get('/about', protectApiRoute, async (req, res) => { azureEndpoint: configFile.azure?.endpoint || 'unknown', azureDeploymentName: configFile.azure?.deploymentName || 'unknown', azureApiVersion: configFile.azure?.apiVersion || 'unknown', + anthropicModel: configFile.anthropic?.model || 'unknown', + // Empty means api.anthropic.com, which is worth saying in a bug report. + anthropicApiUrl: configFile.anthropic?.apiUrl || 'default', ocrProvider: configFile.mistralOcr?.provider || 'mistral', mistralOcrModel: configFile.mistralOcr?.model || 'unknown', scanInterval: configFile.scanInterval || 'unknown', @@ -9446,6 +10585,7 @@ router.get('/about', protectApiRoute, async (req, res) => { openAiKeySet: Boolean(configFile.openai?.apiKey), customKeySet: Boolean(configFile.custom?.apiKey), azureKeySet: Boolean(configFile.azure?.apiKey), + anthropicKeySet: Boolean(configFile.anthropic?.apiKey), mistralKeySet: Boolean(configFile.mistralOcr?.apiKey), apiKeySet: Boolean(configFile.getApiKey && configFile.getApiKey()), }; @@ -9479,6 +10619,8 @@ router.get('/about', protectApiRoute, async (req, res) => { azureEndpoint: configFile.azure?.endpoint || 'unknown', azureDeploymentName: configFile.azure?.deploymentName || 'unknown', azureApiVersion: configFile.azure?.apiVersion || 'unknown', + anthropicModel: configFile.anthropic?.model || 'unknown', + anthropicApiUrl: configFile.anthropic?.apiUrl || 'default', ocrProvider: configFile.mistralOcr?.provider || 'mistral', mistralOcrModel: configFile.mistralOcr?.model || 'unknown', scanInterval: configFile.scanInterval || 'unknown', @@ -9495,6 +10637,7 @@ router.get('/about', protectApiRoute, async (req, res) => { openAiKeySet: Boolean(configFile.openai?.apiKey), customKeySet: Boolean(configFile.custom?.apiKey), azureKeySet: Boolean(configFile.azure?.apiKey), + anthropicKeySet: Boolean(configFile.anthropic?.apiKey), mistralKeySet: Boolean(configFile.mistralOcr?.apiKey), apiKeySet: Boolean(configFile.getApiKey && configFile.getApiKey()), }, diff --git a/schemas.js b/schemas.js index a55f1ed5..5ea569bf 100644 --- a/schemas.js +++ b/schemas.js @@ -352,12 +352,16 @@ * example: ok * message: * type: string - * description: Additional status information (for non-healthy states) + * description: Additional status information (for non-healthy states). Authenticated callers only. * example: "Document scan failed 3 time(s) in a row: connect ECONNREFUSED 172.18.0.2:8000" * scanner: - * $ref: '#/components/schemas/ScannerHealth' + * description: Authenticated callers only. + * allOf: + * - $ref: '#/components/schemas/ScannerHealth' * paperless: - * $ref: '#/components/schemas/PaperlessHealth' + * description: Authenticated callers only. + * allOf: + * - $ref: '#/components/schemas/PaperlessHealth' * * ScannerHealth: * type: object diff --git a/scripts/run-tests.js b/scripts/run-tests.js index 65a4db53..aeb614eb 100644 --- a/scripts/run-tests.js +++ b/scripts/run-tests.js @@ -130,6 +130,11 @@ const TESTS = { 'token-limit-config': 'test-token-limit-config.js', 'truncate-returns-string': 'test-truncate-returns-string.js', 'session-token-type': 'test-session-token-type.js', + 'health-disclosure': 'test-health-disclosure.js', + 'totp-replay': 'test-totp-replay.js', + 'login-username-timing': 'test-login-username-timing.js', + 'oidc-sso': 'test-oidc-sso.js', + 'settings-oidc': 'test-settings-oidc.js', 'setup-complete-guard': 'test-setup-complete-guard.js', 'rescan-fetch-before-delete': 'test-rescan-fetch-before-delete.js', 'ocr-empty-result-guard': 'test-ocr-empty-result-guard.js', @@ -157,10 +162,24 @@ const TESTS = { 'review-sheet': 'test-review-sheet.js', 'review-assist': 'test-review-assist.js', 'review-voice': 'test-review-voice.js', + 'anthropic-support': 'test-anthropic-support.js', + 'anthropic-provider': 'test-anthropic-provider.js', + 'anthropic-settings': 'test-anthropic-settings.js', + 'ocr-provider-anthropic': 'test-ocr-provider-anthropic.js', + 'settings-save-session-secret': 'test-settings-save-session-secret.js', + 'session-guard': 'test-session-guard.js', }; const AREAS = { - auth: ['login-mfa-flow', 'rate-limiting', 'thumbnail-auth-guard'], + auth: [ + 'login-mfa-flow', + 'rate-limiting', + 'thumbnail-auth-guard', + 'settings-save-session-secret', + 'session-guard', + 'oidc-sso', + 'settings-oidc', + ], ocr: [ 'ocr-fallback-ai-errors', 'ocr-startup-recovery', @@ -265,6 +284,10 @@ const AREAS = { 'thumbnail-cache-path-sanitization', 'url-base-validation', 'session-token-type', + 'health-disclosure', + 'totp-replay', + 'login-username-timing', + 'oidc-sso', 'setup-complete-guard', ], duplicates: [ @@ -290,6 +313,12 @@ const AREAS = { 'simplify-assistant-ui', ], review: ['review-sheet', 'review-assist', 'review-voice'], + anthropic: [ + 'anthropic-support', + 'anthropic-provider', + 'anthropic-settings', + 'ocr-provider-anthropic', + ], }; // Tests intentionally excluded from the auto-discovery drift check below. diff --git a/server.js b/server.js index a7e1bc22..9310adc1 100644 --- a/server.js +++ b/server.js @@ -11,6 +11,8 @@ const documentModel = require('./models/document'); const setupService = require('./services/setupService'); const { runStartupMigrations } = require('./services/startupMigrations'); const setupRoutes = require('./routes/setup'); +const oidcRoutes = require('./routes/oidc'); +const oidcService = require('./services/oidcService'); const { isAuthenticated } = require('./routes/auth'); const mistralOcrService = require('./services/mistralOcrService'); const ocrAutoProcessService = require('./services/ocrAutoProcessService'); @@ -362,6 +364,9 @@ app.use((req, res, next) => { res.locals.appAzureEndpoint = config.azure?.endpoint || 'unknown'; res.locals.appAzureDeploymentName = config.azure?.deploymentName || 'unknown'; res.locals.appAzureApiVersion = config.azure?.apiVersion || 'unknown'; + res.locals.appAnthropicModel = config.anthropic?.model || 'unknown'; + // Empty means api.anthropic.com, which is worth saying in a bug report. + res.locals.appAnthropicApiUrl = config.anthropic?.apiUrl || 'default'; res.locals.appMistralOcrModel = config.mistralOcr?.model || 'unknown'; res.locals.appScanInterval = config.scanInterval || 'unknown'; res.locals.appTokenLimit = String(config.tokenLimit || 'unknown'); @@ -377,6 +382,7 @@ app.use((req, res, next) => { res.locals.appOpenAiKeySet = Boolean(config.openai?.apiKey); res.locals.appCustomKeySet = Boolean(config.custom?.apiKey); res.locals.appAzureKeySet = Boolean(config.azure?.apiKey); + res.locals.appAnthropicKeySet = Boolean(config.anthropic?.apiKey); res.locals.appMistralKeySet = Boolean(config.mistralOcr?.apiKey); res.locals.appApiKeySet = Boolean(config.getApiKey && config.getApiKey()); res.locals.loginCookieSecurityWarning = null; @@ -1075,6 +1081,9 @@ async function scanDocuments(source = 'scheduler') { } // Routes +// Single sign-on comes first: its callback must be reachable without a +// session, before the setup router's authentication guard runs. +app.use('/', oidcRoutes); app.use('/', setupRoutes); // Development-only reference page for the zr UI framework. It ships no product @@ -1492,6 +1501,7 @@ async function startServer() { app.listen(port, () => { console.log(`Server running on port ${port}`); warnIfRemoteSetupExposed(); + oidcService.logStatus(); startScanning(); // Armed separately from the scan scheduler so the dashboard also warns // when automatic processing is switched off. diff --git a/services/aiGenerateOptions.js b/services/aiGenerateOptions.js index b8f05a78..40f1cce7 100644 --- a/services/aiGenerateOptions.js +++ b/services/aiGenerateOptions.js @@ -5,6 +5,10 @@ const { estimateTokenCount, jsonFromText, } = require('./serviceUtils'); +const { + assertAnthropicCompleted, + mapAnthropicUsage, +} = require('./anthropicSupport'); /** * The optional second argument of `generateText(prompt, options)`. @@ -673,6 +677,134 @@ async function runChatCompletionStream({ return text; } +/** + * Reads a Claude Messages stream into a collector. + * + * The Messages API streams typed events instead of chat chunks: a text block's + * deltas are the answer, a thinking block's deltas are reasoning (empty under + * the default `display: "omitted"`, so a thinking model is often only heard + * once it answers), and the stop reason and usage arrive with the final + * message, which the caller reads off the stream afterwards. + * + * Returns as soon as the signal is aborted; leaving the loop early makes the + * SDK abort the request, so the server sees the connection go. + * + * @param {AsyncIterable} stream - from client.messages.stream() + * @param {object} collector - from createProgressCollector() + * @param {AbortSignal|null} [signal] + * @returns {Promise} + */ +async function consumeMessagesStream(stream, collector, signal = null) { + for await (const event of stream) { + if (signal?.aborted) break; + if (event?.type === 'content_block_delta') { + const delta = event.delta; + if (delta?.type === 'text_delta') { + collector.pushContent(delta.text); + } else if (delta?.type === 'thinking_delta') { + collector.pushReasoning(delta.thinking); + } + } + collector.report(); + } +} + +/** + * Runs one streamed request against the Claude Messages API. + * + * The same afterwards as runChatCompletionStream(): report as the answer + * grows, say `done` exactly once — on a clean end, on a cut-off and on an + * abort alike — and then return the answer or throw an error that carries + * what did arrive. What differs is the dialect: the events are read by + * consumeMessagesStream(), and the stop reason and usage come from the final + * message rather than from the last chunk. Usage is mapped with + * mapAnthropicUsage(), so promptTokens includes the cached input and the + * cache counts ride along; the stop reason goes through + * assertAnthropicCompleted(), so `max_tokens` is the shared truncation error + * and `refusal` is 'ai_refused', both with the partial text. + * + * Works without a handler too: the collector then reports to nobody, and the + * caller still gets the answer and the usage. + * + * @param {object} params + * @param {object} params.client - an Anthropic SDK client + * @param {object} params.request - the Messages body, without `stream` + * @param {object|null} params.requestOptions - the SDK's second argument + * (the signal), or null + * @param {AbortSignal|null} params.signal + * @param {((update: GenerateTextProgress) => void)|null} params.onProgress + * @param {string} params.provider - name for the error messages + * @param {string} params.remedy - sentence naming the limit to raise + * @param {(usage: object) => void} params.onUsage - told before a throw for + * truncation or refusal, so such a request still reports what it spent + * @returns {Promise} the answer, reasoning stripped + */ +async function runMessagesStream({ + client, + request, + requestOptions, + signal, + onProgress, + provider, + remedy, + onUsage, +}) { + const collector = createProgressCollector({ onProgress }); + let message; + + try { + const stream = requestOptions + ? client.messages.stream(request, requestOptions) + : client.messages.stream(request); + await consumeMessagesStream(stream, collector, signal); + if (signal?.aborted) { + // Same as the chat runner: the loop left early, the SDK closed the + // connection, and the caller is told with what arrived until then. + collector.finish(); + throw abortedGenerationError(null, collector.text()); + } + message = await stream.finalMessage(); + } catch (error) { + collector.finish(); + if (error?.name === 'AbortError') throw error; + if (isAbortError(error, signal)) { + throw abortedGenerationError(error, collector.text()); + } + throw error; + } + + // The reported counts decide the last report, as a usage chunk does for + // the OpenAI-compatible providers. + const reported = mapAnthropicUsage(message?.usage); + if (reported) { + collector.setUsage({ + prompt_tokens: reported.promptTokens, + completion_tokens: reported.completionTokens, + total_tokens: reported.totalTokens, + }); + } + collector.setFinishReason(message?.stop_reason); + collector.finish(); + + const summary = collector.usageSummary(); + if (typeof onUsage === 'function') { + onUsage( + reported + ? { + ...summary, + cacheReadTokens: reported.cacheReadTokens, + cacheWriteTokens: reported.cacheWriteTokens, + } + : summary + ); + } + + assertAnthropicCompleted(message, provider, remedy, { + partialText: collector.text(), + }); + return collector.text(); +} + module.exports = { reasoningEnabled, reasoningEffortForOpenAi, @@ -690,4 +822,6 @@ module.exports = { runChatCompletionStream, isAbortError, abortedGenerationError, + consumeMessagesStream, + runMessagesStream, }; diff --git a/services/aiServiceFactory.js b/services/aiServiceFactory.js index 94902888..5275772c 100644 --- a/services/aiServiceFactory.js +++ b/services/aiServiceFactory.js @@ -3,6 +3,7 @@ const openaiService = require('./openaiService'); const ollamaService = require('./ollamaService'); const customService = require('./customService'); const azureService = require('./azureService'); +const anthropicService = require('./anthropicService'); class AIServiceFactory { static getService() { @@ -16,8 +17,10 @@ class AIServiceFactory { return customService; case 'azure': return azureService; + case 'anthropic': + return anthropicService; } } } -module.exports = AIServiceFactory; \ No newline at end of file +module.exports = AIServiceFactory; diff --git a/services/analysisPrompt.js b/services/analysisPrompt.js new file mode 100644 index 00000000..e55a9ced --- /dev/null +++ b/services/analysisPrompt.js @@ -0,0 +1,429 @@ +'use strict'; + +/** + * The document-analysis prompt, assembled in one place. + * + * Each provider service builds this prompt itself, and the four copies agree + * only because they were copied from each other. This module is the copy of + * customService's assembly, kept byte for byte — the odd indentation of the + * pre-existing data block, a SYSTEM_PROMPT that is not set turning into the + * word "undefined", PROMPT_TAGS that only count toward the token budget and + * never reach the model, a webhook prompt that replaces everything including + * the restrictions and the external data — because operators tuned their + * prompts against exactly that text. Only anthropicService uses it so far; the + * other providers can switch to it later without changing what their models + * read. + * + * Beside the prompt it knows the answer: the JSON schema that describes the + * object the prompt asks for (for providers that can constrain an answer to a + * schema) and the tolerant parser that digs that object out of an answer that + * was not constrained. + * + * Pure: no environment, no config, no network, no file system, no logging. The + * caller hands in what it read and logs what comes back. + */ + +const RestrictionPromptService = require('./restrictionPromptService'); +const { toNameList } = require('./serviceUtils'); + +/** + * Which JSON object the prompt asks for. + * + * FULL is config.mustHavePrompt with the custom-fields template. + * PREDEFINED_TAGS is config.specialPromptPreDefinedTags (USE_PROMPT_TAGS=yes), + * which asks for neither a document type nor custom fields. + */ +const ANALYSIS_RESPONSE_SHAPES = Object.freeze({ + FULL: 'full', + PREDEFINED_TAGS: 'predefined_tags', +}); + +/** + * The custom fields CUSTOM_FIELDS configures, or none with the reason. + * + * customService parses the variable per document and falls back to no fields + * when it does not parse. A value that parses but holds no `custom_fields` + * list made customService fail every analysis with a TypeError; here it is + * the same fallback as an unparsable value, reported the same way. + * + * @param {unknown} customFieldsJson - the CUSTOM_FIELDS value as read + * @returns {{fields: Array, error: Error|null}} + */ +function parseCustomFields(customFieldsJson) { + let parsed; + try { + parsed = JSON.parse(customFieldsJson); + } catch (error) { + return { fields: [], error }; + } + if (!Array.isArray(parsed?.custom_fields)) { + return { + fields: [], + error: new Error('CUSTOM_FIELDS holds no custom_fields list'), + }; + } + return { fields: parsed.custom_fields, error: null }; +} + +/** + * The hint the template puts where the model writes a field's value. + * + * @param {object} field - one CUSTOM_FIELDS entry + * @returns {string} + */ +function customFieldValueHint(field) { + if (field?.data_type === 'date') { + return 'Fill in the date in ISO 8601 format (YYYY-MM-DD) based on your analysis'; + } + if (field?.data_type === 'boolean') { + return "Fill in 'true' or 'false' based on your analysis"; + } + return 'Fill in the value based on your analysis'; +} + +/** + * The system prompt of one document analysis, and what goes with it. + * + * The inputs are what customService reads from the environment, from config + * and from its arguments, under the same names. The order of the steps is + * customService's, and it matters: the restriction placeholders are replaced + * before the external data is appended, USE_PROMPT_TAGS then replaces all of + * it, and a webhook prompt replaces that again. + * + * Nothing per document goes into the system prompt — no timestamp, no id, no + * document text — so it stays byte-identical across the documents of a scan + * and a provider can cache it. What does change it: the existing tags, + * correspondents and document types (with USE_EXISTING_DATA or a restriction + * placeholder), external API data and a webhook prompt. + * + * @param {object} inputs + * @param {string|undefined} inputs.systemPrompt - SYSTEM_PROMPT + * @param {string} inputs.mustHavePrompt - config.mustHavePrompt + * @param {string} inputs.specialPromptPreDefinedTags - config.specialPromptPreDefinedTags + * @param {string|undefined} inputs.customFieldsJson - CUSTOM_FIELDS + * @param {string} inputs.useExistingData - config.useExistingData ('yes'/'no') + * @param {string} inputs.restrictToExistingTags - config.restrictToExistingTags + * @param {string} inputs.restrictToExistingCorrespondents - config.restrictToExistingCorrespondents + * @param {string|undefined} inputs.usePromptTags - USE_PROMPT_TAGS + * @param {string|undefined} inputs.promptTags - PROMPT_TAGS + * @param {Array|string} [inputs.existingTags] - entity objects or names + * @param {Array|string} [inputs.existingCorrespondentList] + * @param {Array|string} [inputs.existingDocumentTypesList] + * @param {string|null} [inputs.externalApiData] - already validated and truncated + * @param {string|null} [inputs.customPrompt] - the webhook's prompt + * @returns {{ + * systemPrompt: string, + * promptTags: string, + * customFieldsTemplate: object, + * customFieldsStr: string, + * customFields: Array, + * customFieldsError: Error|null, + * responseShape: string + * }} promptTags is counted toward the token budget and sent nowhere, as in + * customService; customFieldsError is for the caller to log + */ +function buildAnalysisPrompt({ + systemPrompt: configuredSystemPrompt, + mustHavePrompt, + specialPromptPreDefinedTags, + customFieldsJson, + useExistingData, + restrictToExistingTags, + restrictToExistingCorrespondents, + usePromptTags, + promptTags: configuredPromptTags, + existingTags = [], + existingCorrespondentList = [], + existingDocumentTypesList = [], + externalApiData = null, + customPrompt = null, +} = {}) { + const existingTagNames = toNameList(existingTags).join(', '); + const existingCorrespondentNames = toNameList(existingCorrespondentList).join( + ', ' + ); + const existingDocumentTypeNames = toNameList(existingDocumentTypesList).join( + ', ' + ); + + const { fields: customFields, error: customFieldsError } = + parseCustomFields(customFieldsJson); + + // An index map, not an array: {"0": {...}, "1": {...}} is what the models + // have always been shown, and documentProcessingService reads both. + const customFieldsTemplate = {}; + customFields.forEach((field, index) => { + customFieldsTemplate[index] = { + field_name: field?.value, + value: customFieldValueHint(field), + }; + }); + + const customFieldsStr = + '"custom_fields": ' + + JSON.stringify(customFieldsTemplate, null, 2) + .split('\n') + .map((line) => ' ' + line) + .join('\n'); + const mustHave = String(mustHavePrompt).replace( + '%CUSTOMFIELDS%', + customFieldsStr + ); + + let systemPrompt; + let promptTags = ''; + let responseShape = ANALYSIS_RESPONSE_SHAPES.FULL; + + if ( + useExistingData === 'yes' && + restrictToExistingTags === 'no' && + restrictToExistingCorrespondents === 'no' + ) { + // customService writes this block as an indented template literal; these + // are its bytes, newline and eight spaces of indentation per line. + systemPrompt = + '\n' + + ` Pre-existing tags: ${existingTagNames}\n\n\n` + + ` Pre-existing correspondents: ${existingCorrespondentNames}\n\n\n` + + ` Pre-existing document types: ${existingDocumentTypeNames}\n\n\n` + + ' ' + + configuredSystemPrompt + + '\n\n' + + mustHave; + } else { + systemPrompt = configuredSystemPrompt + '\n\n' + mustHave; + } + + systemPrompt = RestrictionPromptService.processRestrictionsInPrompt( + systemPrompt, + existingTags, + existingCorrespondentList, + existingDocumentTypesList + ); + + if (externalApiData) { + systemPrompt += `\n\nAdditional context from external API:\n${externalApiData}`; + } + + if (usePromptTags === 'yes') { + promptTags = configuredPromptTags; + systemPrompt = + '\n' + + ' Take these tags and try to match one or more to the document content.\n\n\n' + + ' ' + + specialPromptPreDefinedTags; + responseShape = ANALYSIS_RESPONSE_SHAPES.PREDEFINED_TAGS; + } + + if (customPrompt) { + systemPrompt = customPrompt + '\n\n' + mustHave; + responseShape = ANALYSIS_RESPONSE_SHAPES.FULL; + } + + return { + systemPrompt, + promptTags, + customFieldsTemplate, + customFieldsStr, + customFields, + customFieldsError, + responseShape, + }; +} + +/** + * The JSON schema of the object the prompt asks for, in the dialect the + * Claude API's structured outputs accept: every object closed with + * `additionalProperties: false`, no numeric or length constraints. + * + * Every key is required and absence is spelled null, so the model never has + * to decide between leaving a key out and sending an empty value: + * + * - `correspondent` is null when the document names no sender; the analysis + * has always accepted that. + * - `document_type` and `document_date` are null when the document says + * nothing about them. A forced date would be written to Paperless-ngx as + * the created date; null keeps the date the document already has. + * - a custom field's `value` is null when the document holds nothing for it + * ("the custom_fields are optional and can be left out"); its `field_name` + * is fixed to the configured name ("do not change the value of + * field_name"). The values stay strings, as the template shows them — + * the write-back validates and converts them per data type. + * + * The prompt still asks for the same object in words; the schema only makes + * sure the answer parses. + * + * @param {{responseShape?: string, customFields?: Array}} prompt - + * what buildAnalysisPrompt() returned + * @returns {object} the schema, for `output_config.format.schema` + */ +function analysisResponseSchema({ + responseShape = ANALYSIS_RESPONSE_SHAPES.FULL, + customFields = [], +} = {}) { + const text = () => ({ type: 'string' }); + const nullableText = () => ({ type: ['string', 'null'] }); + const full = responseShape !== ANALYSIS_RESPONSE_SHAPES.PREDEFINED_TAGS; + + // In the order the prompt lists them. + const properties = { + title: text(), + correspondent: nullableText(), + tags: { type: 'array', items: text() }, + }; + if (full) properties.document_type = nullableText(); + properties.document_date = nullableText(); + properties.language = text(); + + if (full) { + const fieldProperties = {}; + (Array.isArray(customFields) ? customFields : []).forEach( + (field, index) => { + const name = field?.value; + fieldProperties[String(index)] = { + type: 'object', + properties: { + field_name: + typeof name === 'string' + ? { type: 'string', const: name } + : text(), + value: nullableText(), + }, + required: ['field_name', 'value'], + additionalProperties: false, + }; + } + ); + const fieldKeys = Object.keys(fieldProperties); + properties.custom_fields = { + type: 'object', + properties: fieldProperties, + ...(fieldKeys.length > 0 ? { required: fieldKeys } : {}), + additionalProperties: false, + }; + } + + return { + type: 'object', + properties, + required: Object.keys(properties), + additionalProperties: false, + }; +} + +/** + * The first complete JSON object or array in a text, or null. + * + * Bracket counting that skips over strings, so a brace inside a title does + * not end the object early. customService's _extractFirstJsonValue. + * + * @param {string} text + * @returns {string|null} + */ +function extractFirstJsonValue(text) { + if (!text) { + return null; + } + + const starts = []; + const firstObject = text.indexOf('{'); + const firstArray = text.indexOf('['); + if (firstObject !== -1) starts.push(firstObject); + if (firstArray !== -1) starts.push(firstArray); + if (starts.length === 0) { + return null; + } + + const startIndex = Math.min(...starts); + const opening = text[startIndex]; + const closing = opening === '{' ? '}' : ']'; + + let depth = 0; + let inString = false; + let escaping = false; + + for (let i = startIndex; i < text.length; i += 1) { + const char = text[i]; + + if (inString) { + if (escaping) { + escaping = false; + } else if (char === '\\') { + escaping = true; + } else if (char === '"') { + inString = false; + } + continue; + } + + if (char === '"') { + inString = true; + continue; + } + + if (char === opening) { + depth += 1; + } else if (char === closing) { + depth -= 1; + if (depth === 0) { + return text.slice(startIndex, i + 1); + } + } + } + + return null; +} + +/** + * The analysis object out of a model's answer. + * + * A schema-constrained answer parses directly. Everything else may come + * wrapped in a code fence, after a block or behind a sentence of + * prose, so those are stripped and the first JSON value is tried next. + * customService's _parseJsonResponse, with the same error message — the scan + * loop sends "Invalid JSON response from API" to the OCR queue. + * + * @param {unknown} rawContent + * @returns {{parsed: unknown, normalized: string}} normalized is the JSON text + * that parsed, for the response log + * @throws {Error} 'Invalid JSON response from API', with the parse error as cause + */ +function parseAnalysisAnswer(rawContent) { + const sanitizedContent = String(rawContent || '') + .replace(/[\s\S]*?<\/think>/g, '') + .replace(/```json\n?/gi, '') + .replace(/```\n?/g, '') + .trim(); + + try { + return { + parsed: JSON.parse(sanitizedContent), + normalized: sanitizedContent, + }; + } catch (directParseError) { + const extractedJson = extractFirstJsonValue(sanitizedContent); + if (!extractedJson) { + throw new Error('Invalid JSON response from API', { + cause: directParseError, + }); + } + + try { + return { + parsed: JSON.parse(extractedJson), + normalized: extractedJson, + }; + } catch (extractedParseError) { + throw new Error('Invalid JSON response from API', { + cause: extractedParseError, + }); + } + } +} + +module.exports = { + ANALYSIS_RESPONSE_SHAPES, + buildAnalysisPrompt, + analysisResponseSchema, + parseAnalysisAnswer, +}; diff --git a/services/anthropicService.js b/services/anthropicService.js new file mode 100644 index 00000000..222c0ebf --- /dev/null +++ b/services/anthropicService.js @@ -0,0 +1,597 @@ +'use strict'; + +/** + * The Claude API as an AI provider (AI_PROVIDER=anthropic). + * + * Same contract as the other four provider services — initialize(), + * analyzeDocument(), generateText(), checkStatus() with the same arguments and + * return shapes — so nothing that calls AIServiceFactory.getService() needs to + * know which provider answers. Request building and answer reading go through + * services/anthropicSupport.js; the analysis prompt comes from + * services/analysisPrompt.js. + * + * Three things only this provider does: + * + * - Structured output (ANTHROPIC_STRUCTURED_OUTPUT): the analysis answer is + * constrained to the JSON schema of the object the prompt asks for, where + * the model supports it. The answer is still parsed the tolerant way, so + * a model or setting without it works as before. + * - Prompt caching (ANTHROPIC_PROMPT_CACHING): the system prompt goes out as + * a cacheable block. It is the same for every document of a scan, so from + * the second document on it is read from the cache at a tenth of the price. + * - Every request is streamed, also when nobody watches the progress: the + * SDK refuses a non-streamed request whose max_tokens could take longer + * than ten minutes (ANTHROPIC_MAX_TOKENS above about 21000), and thinking + * makes long answers likely. The answer is the same either way. + */ + +const fs = require('fs').promises; +const path = require('path'); +const Anthropic = require('@anthropic-ai/sdk'); +const config = require('../config/config'); +const paperlessService = require('./paperlessService'); +const { + THUMBNAIL_CACHE_DIR, + getThumbnailCachePath, +} = require('./thumbnailCachePaths'); +const { + calculateTokens, + calculateTotalPromptTokens, + truncateToTokenLimit, + isTimeoutError, + buildTimeoutErrorMessage, +} = require('./serviceUtils'); +const { + abortSignal, + abortedGenerationError, + hasSystemPrompt, + isAbortError, + modelOverride, + progressHandler, + reasoningEnabled, + runMessagesStream, +} = require('./aiGenerateOptions'); +const { + anthropicModelTraits, + anthropicThinkingParams, + anthropicMaxTokens, + createAnthropicClient, + anthropicMessageText, + mapAnthropicUsage, + assertAnthropicCompleted, +} = require('./anthropicSupport'); +const { + buildAnalysisPrompt, + analysisResponseSchema, + parseAnalysisAnswer, +} = require('./analysisPrompt'); + +const PROVIDER = 'Anthropic'; +const responseLogPath = path.join( + process.cwd(), + 'data', + 'logs', + 'response.txt' +); +/** + * What an operator can do about an answer that hit the completion cap. + * Thinking counts toward the cap, so less effort is the other way out. + */ +const TRUNCATION_REMEDY = + 'Raise ANTHROPIC_MAX_TOKENS, or lower ANTHROPIC_EFFORT so less of it goes to thinking.'; + +class AnthropicService { + constructor() { + this.client = null; + this.lastGenerateTextUsage = null; + /** + * Schemas the API refused for a model, as `model\nschema`. The analysis + * then runs without structured output for the rest of the process + * instead of paying for a refused request per document. + */ + this.rejectedSchemas = new Set(); + } + + initialize() { + if ( + !this.client && + config.aiProvider === 'anthropic' && + config.anthropic.apiKey + ) { + this.client = createAnthropicClient({ + apiKey: config.anthropic.apiKey, + apiUrl: config.anthropic.apiUrl, + }); + } + } + + /** + * The `system` field: a plain string, or with ANTHROPIC_PROMPT_CACHING one + * text block marked cacheable. The marker sits on the only system block, + * so the cached prefix is exactly the system prompt; a prompt shorter than + * the model's minimum is simply not cached, at no extra cost. + * + * @param {string} text + * @returns {string|Array} + */ + _systemField(text) { + if (config.anthropic.promptCaching !== 'yes') return text; + return [{ type: 'text', text, cache_control: { type: 'ephemeral' } }]; + } + + /** + * One debug line per request with what the cache did, so an operator can + * see whether the system prompt is actually being reused. + * + * @param {{cacheReadTokens?: number, cacheWriteTokens?: number}|null} usage + */ + _logCacheUsage(usage) { + if (config.anthropic.promptCaching !== 'yes' || !usage) return; + console.debug( + `[DEBUG] Anthropic cache: read ${usage.cacheReadTokens || 0}, write ${usage.cacheWriteTokens || 0}` + ); + } + + /** + * Caches the document's thumbnail for the history page, as every provider + * does before it analyses. A thumbnail Paperless-ngx cannot deliver is + * only a missing picture: the analysis goes on (the other providers return + * nothing at all then, which the scan loop cannot read). + * + * @param {unknown} id - the Paperless-ngx document id + */ + async _cacheThumbnail(id) { + let cachePath; + try { + cachePath = getThumbnailCachePath(id); + } catch (error) { + console.warn(`Thumbnail not cached: ${error.message}`); + return; + } + + try { + await fs.access(cachePath); + console.log('[DEBUG] Thumbnail already cached'); + return; + } catch { + console.log('Thumbnail not cached, fetching from Paperless'); + } + + const thumbnailData = await paperlessService.getThumbnailImage(id); + if (!thumbnailData) { + console.warn('Thumbnail not found'); + return; + } + await fs.mkdir(THUMBNAIL_CACHE_DIR, { recursive: true }); + await fs.writeFile(cachePath, thumbnailData); + } + + /** + * Validate and truncate external API data to prevent token overflow + * @param {any} apiData - The external API data to validate + * @param {number} maxTokens - Maximum tokens allowed for external data (default: 500) + * @returns {Promise} - Validated and potentially truncated data string + */ + async _validateAndTruncateExternalApiData(apiData, maxTokens = 500) { + if (!apiData) { + return null; + } + + const dataString = + typeof apiData === 'object' + ? JSON.stringify(apiData, null, 2) + : String(apiData); + + const model = config.anthropic.model; + const dataTokens = await calculateTokens(dataString, model); + + if (dataTokens > maxTokens) { + console.warn( + `[WARNING] External API data (${dataTokens} tokens) exceeds limit (${maxTokens}), truncating` + ); + return await truncateToTokenLimit(dataString, maxTokens, model); + } + + console.log(`[DEBUG] External API data validated: ${dataTokens} tokens`); + return dataString; + } + + /** + * The Messages body of one document analysis. + * + * No temperature: the current models reject sampling parameters, and the + * analysis does not depend on it. Thinking follows ANTHROPIC_EFFORT only; + * the analysis has no wish of its own about it. + * + * @param {string} model + * @param {object} prompt - from buildAnalysisPrompt() + * @param {string} documentText - already truncated + * @returns {object} + */ + _analysisRequest(model, prompt, documentText) { + const thinking = anthropicThinkingParams(model, { + effort: config.anthropic.effort, + }); + const request = { + model, + max_tokens: anthropicMaxTokens(null, config.anthropic.maxTokens), + system: this._systemField(prompt.systemPrompt), + messages: [{ role: 'user', content: documentText }], + }; + if (thinking.thinking) request.thinking = thinking.thinking; + + const outputConfig = { ...(thinking.output_config || {}) }; + if ( + config.anthropic.structuredOutput === 'yes' && + anthropicModelTraits(model).structuredOutput + ) { + const schema = analysisResponseSchema(prompt); + if (!this.rejectedSchemas.has(this._schemaKey(model, schema))) { + outputConfig.format = { type: 'json_schema', schema }; + } + } + if (Object.keys(outputConfig).length > 0) { + request.output_config = outputConfig; + } + return request; + } + + /** What rejectedSchemas remembers a refused schema by. */ + _schemaKey(model, schema) { + return `${model}\n${JSON.stringify(schema)}`; + } + + /** + * Sends the analysis and returns the final message. + * + * A schema the API refuses (400) would fail every document the same way, + * so the request is sent once more without it; when that one goes through, + * the schema is remembered as refused and the following documents skip it. + * Any other failure, and a 400 that has nothing to do with the schema, + * surfaces as it is. + * + * @param {object} request + * @returns {Promise} the Messages response + */ + async _sendAnalysis(request) { + try { + return await this.client.messages.stream(request).finalMessage(); + } catch (error) { + const format = request.output_config?.format; + if (!format || !(error instanceof Anthropic.BadRequestError)) { + throw error; + } + // The same request without the schema; an effort stays. + const retry = { ...request, output_config: { ...request.output_config } }; + delete retry.output_config.format; + if (Object.keys(retry.output_config).length === 0) { + delete retry.output_config; + } + const message = await this.client.messages.stream(retry).finalMessage(); + this.rejectedSchemas.add(this._schemaKey(request.model, format.schema)); + console.warn( + `[WARN] Anthropic refused the analysis schema for ${request.model} (${error.message}); analysing without structured output from now on.` + ); + return message; + } + } + + /** + * Analyse one document. Same arguments and return shapes as every other + * provider: `{ document, metrics, truncated }` on success and + * `{ document: { tags: [], correspondent: null }, metrics: null, error, + * errorCode }` on failure — it never throws. + * + * errorCode is 'ai_response_truncated' for an answer that hit + * ANTHROPIC_MAX_TOKENS and 'ai_refused' for a request the model declined. + * Neither message is one the scan loop sends to OCR: new text would be cut + * off or declined the same way. + * + * @param {string} content - the document text + * @param {Array|string} [existingTags] + * @param {Array|string} [existingCorrespondentList] + * @param {Array|string} [existingDocumentTypesList] + * @param {number|string} id - the Paperless-ngx document id + * @param {string|null} [customPrompt] - replaces SYSTEM_PROMPT (webhook) + * @param {{externalApiData?: unknown}} [options] + * @returns {Promise} + */ + async analyzeDocument( + content, + existingTags = [], + existingCorrespondentList = [], + existingDocumentTypesList = [], + id, + customPrompt = null, + options = {} + ) { + try { + this.initialize(); + + if (!this.client) { + throw new Error('Anthropic client not initialized - missing API key'); + } + + await this._cacheThumbnail(id); + + // Get external API data if available and validate it + let validatedExternalApiData = null; + if (options?.externalApiData) { + try { + validatedExternalApiData = + await this._validateAndTruncateExternalApiData( + options.externalApiData + ); + console.log('[DEBUG] External API data validated and included'); + } catch (error) { + console.warn( + '[WARNING] External API data validation failed:', + error.message + ); + validatedExternalApiData = null; + } + } + + const model = config.anthropic.model; + const usePromptTags = process.env.USE_PROMPT_TAGS; + const prompt = buildAnalysisPrompt({ + systemPrompt: process.env.SYSTEM_PROMPT, + mustHavePrompt: config.mustHavePrompt, + specialPromptPreDefinedTags: config.specialPromptPreDefinedTags, + customFieldsJson: process.env.CUSTOM_FIELDS, + useExistingData: config.useExistingData, + restrictToExistingTags: config.restrictToExistingTags, + restrictToExistingCorrespondents: + config.restrictToExistingCorrespondents, + usePromptTags, + promptTags: process.env.PROMPT_TAGS, + existingTags, + existingCorrespondentList, + existingDocumentTypesList, + externalApiData: validatedExternalApiData, + customPrompt, + }); + if (prompt.customFieldsError) { + console.error( + `Failed to parse CUSTOM_FIELDS: ${prompt.customFieldsError.message}` + ); + console.debug(prompt.customFieldsError); + } + if (customPrompt) { + console.log('[DEBUG] Replace system prompt with custom prompt'); + } + + // Calculate tokens AFTER all prompt modifications are complete + const totalPromptTokens = await calculateTotalPromptTokens( + prompt.systemPrompt, + usePromptTags === 'yes' ? [prompt.promptTags] : [], + model + ); + + const maxTokens = Number(config.tokenLimit); + const reservedTokens = totalPromptTokens + Number(config.responseTokens); + const availableTokens = maxTokens - reservedTokens; + + // A non-finite budget is the same failure as a negative one: it slips + // past a bare `<= 0` and truncates the document down to nothing. + if (!Number.isFinite(availableTokens) || availableTokens <= 0) { + console.warn( + `[WARNING] No available tokens for content. Reserved: ${reservedTokens}, Max: ${maxTokens}` + ); + throw new Error( + 'Token limit exceeded: prompt too large for available token limit' + ); + } + + console.log( + `[DEBUG] Token calculation - Prompt: ${totalPromptTokens}, Reserved: ${reservedTokens}, Available: ${availableTokens}` + ); + console.log( + `[DEBUG] Use existing data: ${config.useExistingData}, Restrictions applied based on useExistingData setting` + ); + console.log( + `[DEBUG] External API data: ${validatedExternalApiData ? 'included' : 'none'}` + ); + + const truncatedContent = await truncateToTokenLimit( + content, + availableTokens, + model + ); + + const request = this._analysisRequest(model, prompt, truncatedContent); + const message = await this._sendAnalysis(request); + + // A cut-off or declined answer is a failure of its own, reported before + // its text is looked at. + assertAnthropicCompleted(message, PROVIDER, TRUNCATION_REMEDY); + + const metrics = mapAnthropicUsage(message.usage); + console.log( + `[DEBUG] Anthropic request sent, total tokens: ${metrics ? metrics.totalTokens : 'unknown'}` + ); + this._logCacheUsage(metrics); + + let jsonContent = anthropicMessageText(message); + if (!jsonContent) { + throw new Error('Invalid API response structure'); + } + + let parsedResponse; + try { + const parsedResult = parseAnalysisAnswer(jsonContent); + parsedResponse = parsedResult.parsed; + jsonContent = parsedResult.normalized; + } catch (error) { + console.error(`Failed to parse JSON response: ${error.message}`); + console.debug(error); + throw error; + } + + try { + await fs.mkdir(path.dirname(responseLogPath), { recursive: true }); + await fs.appendFile(responseLogPath, `${jsonContent}\n`); + } catch (logError) { + console.warn('Failed to write AI response log:', logError.message); + } + + // Validate response structure + if ( + !parsedResponse || + !Array.isArray(parsedResponse.tags) || + (typeof parsedResponse.correspondent !== 'string' && + parsedResponse.correspondent !== null) + ) { + throw new Error( + 'AI could not determine assignable metadata: no tags or correspondent found' + ); + } + + return { + document: parsedResponse, + metrics, + truncated: truncatedContent.length < content.length, + }; + } catch (error) { + const timedOut = + error instanceof Anthropic.APIConnectionTimeoutError || + isTimeoutError(error); + const normalizedMessage = timedOut + ? buildTimeoutErrorMessage('AI') + : error.message; + + if (timedOut) { + console.error( + `[TIMEOUT][AI] Anthropic request timed out: ${error.message}` + ); + } + + console.error(`Failed to analyze document: ${normalizedMessage}`); + console.debug(error); + return { + document: { tags: [], correspondent: null }, + metrics: null, + error: normalizedMessage, + // Undefined for everything that is not one of ours; the scan loop + // falls back to its generic reason then. + errorCode: error.code, + }; + } + } + + /** + * Generate text based on a prompt + * @param {string} prompt - The prompt to generate text from + * @param {object} [options] - Optional overrides for this one call + * @param {string} [options.systemPrompt] - Sent as the top-level system + * field, cacheable with ANTHROPIC_PROMPT_CACHING + * @param {string} [options.model] - Overrides ANTHROPIC_MODEL + * @param {number} [options.temperature] - Accepted and ignored: the current + * Claude models answer 400 to any sampling parameter, and the callers' + * temperatures (0 for a judge) only ask for what the model does anyway + * @param {number} [options.maxTokens] - Overrides ANTHROPIC_MAX_TOKENS; + * thinking counts toward it + * @param {AbortSignal} [options.signal] - Cancels the request in flight + * @param {Function} [options.onProgress] - Reports the answer as it grows; + * see GenerateTextProgress in aiGenerateOptions. The request is streamed + * either way (see the top of this file), so without a handler nothing + * changes but the reports + * @param {boolean} [options.reasoning] - false asks for as little thinking + * as the model allows, true for thinking where the model only thinks when + * asked; see anthropicThinkingParams() + * @returns {Promise} - The generated text + */ + async generateText(prompt, options = {}) { + // Read before the try so the catch can tell a stopped request from a + // failed one. + const signal = abortSignal(options); + const onProgress = progressHandler(options); + + try { + this.initialize(); + + if (!this.client) { + throw new Error('Anthropic client not initialized - missing API key'); + } + + const model = modelOverride(options) || config.anthropic.model; + const thinking = anthropicThinkingParams(model, { + reasoning: reasoningEnabled(options), + effort: config.anthropic.effort, + }); + + const request = { + model, + max_tokens: anthropicMaxTokens( + options?.maxTokens, + config.anthropic.maxTokens + ), + }; + if (hasSystemPrompt(options)) { + request.system = this._systemField(options.systemPrompt); + } + request.messages = [{ role: 'user', content: prompt }]; + Object.assign(request, thinking); + + this.lastGenerateTextUsage = null; + return await runMessagesStream({ + client: this.client, + request, + // The SDK's request option bag; a caller that brought no signal + // gets the call without one. + requestOptions: signal ? { signal } : null, + signal, + onProgress, + provider: PROVIDER, + remedy: TRUNCATION_REMEDY, + onUsage: (usage) => { + this.lastGenerateTextUsage = usage; + this._logCacheUsage(usage); + }, + }); + } catch (error) { + // One name for a stopped request, whatever the client called it; an + // abort the runner raised already carries its partial text. + const thrown = + isAbortError(error, signal) && error?.name !== 'AbortError' + ? abortedGenerationError(error, '') + : error; + console.error(`Error generating text with Anthropic: ${thrown.message}`); + console.debug(thrown); + throw thrown; + } + } + + /** + * Checks the key and the model without spending a token: the models + * endpoint knows the configured model only when the key is valid and the + * model exists for it. + * + * @returns {Promise<{status: 'ok', model: string}|{status: 'error', error: string}>} + */ + async checkStatus() { + const model = config.anthropic.model; + try { + this.initialize(); + + if (!this.client) { + throw new Error('Anthropic client not initialized - missing API key'); + } + + await this.client.models.retrieve(model); + return { status: 'ok', model }; + } catch (error) { + let message = error.message; + if (error instanceof Anthropic.AuthenticationError) { + message = 'The Anthropic API key was rejected.'; + } else if (error instanceof Anthropic.NotFoundError) { + message = `The Anthropic model "${model}" was not found.`; + } + console.error(`Error checking Anthropic status: ${error.message}`); + console.debug(error); + return { status: 'error', error: message }; + } + } +} + +module.exports = new AnthropicService(); diff --git a/services/anthropicSupport.js b/services/anthropicSupport.js new file mode 100644 index 00000000..d4f1ddc1 --- /dev/null +++ b/services/anthropicSupport.js @@ -0,0 +1,486 @@ +'use strict'; + +/** + * Everything the app needs to know about the Anthropic Messages API before it + * sends a request and after the answer arrives, kept in one place so the AI + * provider (anthropicService) and the OCR provider (mistralOcrService) build + * their requests and read their answers the same way. + * + * The Messages API is not an OpenAI dialect: the system prompt is its own + * field, `max_tokens` is required, the answer is a list of typed blocks, usage + * is called input/output tokens, and a model can decline with + * `stop_reason: "refusal"`. The current models also reject sampling + * parameters and differ in how — and whether — thinking can be switched off. + * Those per-model rules live in `anthropicModelTraits()`; every request reads + * them from there instead of matching model names on its own. + * + * Nothing here talks to the network except the client `createAnthropicClient` + * returns. + */ + +const Anthropic = require('@anthropic-ai/sdk'); +const { + assertCompletionNotTruncated, + stripTrailingSlashes, +} = require('./serviceUtils'); + +/** The model a fresh installation starts with. */ +const ANTHROPIC_DEFAULT_MODEL = 'claude-haiku-5-5'; + +/** + * The models setup and settings offer before the live list from + * `client.models.list()` arrives, cheapest first. + */ +const ANTHROPIC_SUGGESTED_MODELS = Object.freeze([ + Object.freeze({ id: 'claude-haiku-5-5', label: 'Claude Haiku 5.5' }), + Object.freeze({ id: 'claude-sonnet-5-5', label: 'Claude Sonnet 5.5' }), + Object.freeze({ id: 'claude-opus-5-5', label: 'Claude Opus 5.5' }), +]); + +/** Every effort level the API knows, lowest first. */ +const ANTHROPIC_EFFORT_LEVELS = Object.freeze([ + 'low', + 'medium', + 'high', + 'xhigh', + 'max', +]); + +/** The completion cap used when nothing else is configured. */ +const ANTHROPIC_DEFAULT_MAX_TOKENS = 16000; + +/** + * @typedef {object} AnthropicModelTraits + * @property {'disabled'|'between_tools'|'omit'|null} thinkingOff + * How this model is asked not to think: + * - 'disabled': send `thinking: { type: 'disabled' }` (only at effort high + * or below, which is why a no-thinking request also sends effort 'low') + * - 'between_tools': send `thinking: { type: 'between_tools' }`, the lowest + * setting Claude Sonnet 5.5 has; it takes no other field + * - 'omit': the model only thinks when asked, so sending nothing is off + * - null: thinking is always on; a low effort is the only lever + * @property {boolean} adaptive - accepts `thinking: { type: 'adaptive' }` + * @property {ReadonlyArray} effortLevels - the `output_config.effort` + * values this model accepts; empty when it rejects the field + * @property {boolean} structuredOutput - accepts `output_config.format` + * @property {number} minCacheTokens - the shortest prefix a `cache_control` + * marker caches; shorter ones silently do not + */ + +const ALL_EFFORTS = ANTHROPIC_EFFORT_LEVELS; +const NO_XHIGH = Object.freeze(['low', 'medium', 'high', 'max']); +const BASIC_EFFORTS = Object.freeze(['low', 'medium', 'high']); +const NO_EFFORTS = Object.freeze([]); + +/** + * Model families, most specific first. The first pattern that matches wins. + * A model newer than this table matches nothing and gets FUTURE_MODEL, which + * is what every model released since Claude Opus 5.5 does. + */ +const MODEL_FAMILIES = [ + { + pattern: /^claude-(?:fable|mythos)-/, + traits: { + thinkingOff: null, + adaptive: true, + effortLevels: ALL_EFFORTS, + structuredOutput: true, + minCacheTokens: 512, + }, + }, + { + pattern: /^claude-opus-5-5/, + traits: { + thinkingOff: null, + adaptive: true, + effortLevels: ALL_EFFORTS, + structuredOutput: true, + minCacheTokens: 512, + }, + }, + { + pattern: /^claude-sonnet-5-5/, + traits: { + thinkingOff: 'between_tools', + adaptive: true, + effortLevels: ALL_EFFORTS, + structuredOutput: true, + minCacheTokens: 512, + }, + }, + { + pattern: /^claude-haiku-5-5/, + traits: { + thinkingOff: 'disabled', + adaptive: true, + effortLevels: ALL_EFFORTS, + structuredOutput: true, + minCacheTokens: 512, + }, + }, + { + pattern: /^claude-opus-5(?:$|[^-.\d])/, + traits: { + thinkingOff: 'disabled', + adaptive: true, + effortLevels: ALL_EFFORTS, + structuredOutput: true, + minCacheTokens: 512, + }, + }, + { + pattern: /^claude-sonnet-5(?:$|[^-.\d])/, + traits: { + thinkingOff: 'disabled', + adaptive: true, + effortLevels: ALL_EFFORTS, + structuredOutput: true, + minCacheTokens: 1024, + }, + }, + { + pattern: /^claude-opus-4-8/, + traits: { + thinkingOff: 'omit', + adaptive: true, + effortLevels: ALL_EFFORTS, + structuredOutput: true, + minCacheTokens: 1024, + }, + }, + { + pattern: /^claude-opus-4-7/, + traits: { + thinkingOff: 'omit', + adaptive: true, + effortLevels: ALL_EFFORTS, + structuredOutput: false, + minCacheTokens: 2048, + }, + }, + { + pattern: /^claude-opus-4-6/, + traits: { + thinkingOff: 'omit', + adaptive: true, + effortLevels: NO_XHIGH, + structuredOutput: false, + minCacheTokens: 4096, + }, + }, + { + pattern: /^claude-sonnet-4-6/, + traits: { + thinkingOff: 'omit', + adaptive: true, + effortLevels: NO_XHIGH, + structuredOutput: false, + minCacheTokens: 1024, + }, + }, + { + pattern: /^claude-opus-4-5/, + traits: { + thinkingOff: 'omit', + adaptive: false, + effortLevels: BASIC_EFFORTS, + structuredOutput: true, + minCacheTokens: 4096, + }, + }, + { + pattern: /^claude-haiku-4-5/, + traits: { + thinkingOff: 'omit', + adaptive: false, + effortLevels: NO_EFFORTS, + structuredOutput: true, + minCacheTokens: 4096, + }, + }, + { + // Sonnet 4.5, Opus 4.1 and everything older: no adaptive thinking, no + // effort, and thinking only with a budget this app never sends. + pattern: /^claude-(?:sonnet-4|opus-4|3)/, + traits: { + thinkingOff: 'omit', + adaptive: false, + effortLevels: NO_EFFORTS, + structuredOutput: false, + minCacheTokens: 1024, + }, + }, +]; + +const FUTURE_MODEL = Object.freeze({ + thinkingOff: null, + adaptive: true, + effortLevels: ALL_EFFORTS, + structuredOutput: true, + minCacheTokens: 1024, +}); + +/** + * The model name as the table reads it: trimmed and lower-cased. + * + * @param {unknown} model + * @returns {string} + */ +function normalizeAnthropicModel(model) { + return String(model ?? '') + .trim() + .toLowerCase(); +} + +/** + * What a request to this model may and must contain. See AnthropicModelTraits. + * + * @param {unknown} model + * @returns {AnthropicModelTraits} + */ +function anthropicModelTraits(model) { + const name = normalizeAnthropicModel(model); + const family = MODEL_FAMILIES.find((entry) => entry.pattern.test(name)); + return Object.freeze({ ...(family ? family.traits : FUTURE_MODEL) }); +} + +/** + * A configured effort level, or null when it names none the API knows. + * + * @param {unknown} value + * @returns {string|null} + */ +function normalizeAnthropicEffort(value) { + const level = String(value ?? '') + .trim() + .toLowerCase(); + return ANTHROPIC_EFFORT_LEVELS.includes(level) ? level : null; +} + +/** + * The thinking-related fields of a Messages request, to spread into it. + * + * `reasoning` is the wish `generateText` callers express (see + * reasoningEnabled() in aiGenerateOptions): false asks for as little thinking + * as the model allows, true for thinking where the model only thinks when + * asked, null for whatever the model does by default. `effort` is the + * operator's ANTHROPIC_EFFORT; a no-thinking wish overrides it with 'low'. + * + * Never returns `temperature`: the current models reject it, and the + * analysis prompt does not depend on it. + * + * @param {unknown} model + * @param {{reasoning?: boolean|null, effort?: unknown}} [wish] + * @returns {{thinking?: object, output_config?: {effort: string}}} + */ +function anthropicThinkingParams(model, wish = {}) { + const traits = anthropicModelTraits(model); + const reasoning = typeof wish.reasoning === 'boolean' ? wish.reasoning : null; + const params = {}; + + let effort = normalizeAnthropicEffort(wish.effort); + if (reasoning === false) effort = 'low'; + + if (reasoning === false) { + if (traits.thinkingOff === 'disabled') { + params.thinking = { type: 'disabled' }; + } else if (traits.thinkingOff === 'between_tools') { + params.thinking = { type: 'between_tools' }; + } + } else if (reasoning === true && traits.thinkingOff === 'omit') { + if (traits.adaptive) params.thinking = { type: 'adaptive' }; + } + + if (effort && traits.effortLevels.includes(effort)) { + params.output_config = { effort }; + } + return params; +} + +/** + * The completion cap for one request: the caller's, else the configured one, + * else ANTHROPIC_DEFAULT_MAX_TOKENS. Thinking counts toward it, so it is + * never derived from RESPONSE_TOKENS alone, which is sized for a JSON answer + * without any. + * + * @param {unknown} requested - a caller's cap, ignored unless a positive number + * @param {unknown} configured - ANTHROPIC_MAX_TOKENS + * @returns {number} + */ +function anthropicMaxTokens(requested, configured) { + const pick = (value) => + typeof value === 'number' && Number.isFinite(value) && value > 0 + ? Math.floor(value) + : null; + return ( + pick(requested) ?? pick(Number(configured)) ?? ANTHROPIC_DEFAULT_MAX_TOKENS + ); +} + +/** + * A client for the Claude API. `apiUrl` is only for a proxy in front of it; + * empty means api.anthropic.com. + * + * @param {{apiKey: string, apiUrl?: string, timeoutMs?: number}} options + * @returns {Anthropic} + */ +function createAnthropicClient({ apiKey, apiUrl = '', timeoutMs } = {}) { + const clientOptions = { apiKey: String(apiKey || '') }; + const baseURL = stripTrailingSlashes(String(apiUrl || '').trim()); + if (baseURL) clientOptions.baseURL = baseURL; + if (typeof timeoutMs === 'number' && timeoutMs > 0) { + clientOptions.timeout = timeoutMs; + } + return new Anthropic(clientOptions); +} + +/** Where a client goes when no proxy URL is set. */ +const ANTHROPIC_DEFAULT_ORIGIN = 'https://api.anthropic.com'; + +/** + * Whether a request targets a Claude endpoint this installation already uses: + * ANTHROPIC_API_URL (empty means api.anthropic.com), or OCR_API_URL while + * Claude is the OCR provider. A stored key is only substituted for an empty + * key field when this holds; otherwise any signed-in caller could name their + * own host and receive the saved key in the x-api-key header. Typed keys are + * not affected. An empty URL passes: the request then goes to the configured + * proxy or to api.anthropic.com, both of which already hold the key. + * + * @param {string} requestedUrl - the URL the request names, empty for none + * @param {NodeJS.ProcessEnv} [env] + * @returns {boolean} + */ +function isConfiguredAnthropicHost(requestedUrl, env = process.env) { + const origin = (value) => { + const trimmed = String(value || '').trim(); + if (!trimmed) return ANTHROPIC_DEFAULT_ORIGIN; + try { + const parsed = new URL(trimmed); + return `${parsed.protocol}//${parsed.host}`.toLowerCase(); + } catch { + return null; + } + }; + + if (!String(requestedUrl || '').trim()) return true; + const requested = origin(requestedUrl); + if (!requested) return false; + + const configured = [origin(env.ANTHROPIC_API_URL)]; + const ocrIsClaude = + String(env.OCR_PROVIDER || '') + .trim() + .toLowerCase() === 'anthropic'; + if (ocrIsClaude && String(env.OCR_API_URL || '').trim()) { + configured.push(origin(env.OCR_API_URL)); + } + return configured.includes(requested); +} + +/** + * The visible answer of a Messages response: every text block, in order. + * Thinking blocks are no part of it, and a response can begin with them, so + * the content is read by type and never by position. + * + * @param {{content?: Array<{type: string, text?: string}>}} message + * @returns {string} + */ +function anthropicMessageText(message) { + const blocks = Array.isArray(message?.content) ? message.content : []; + return blocks + .filter((block) => block?.type === 'text' && typeof block.text === 'string') + .map((block) => block.text) + .join('') + .trim(); +} + +/** + * Messages usage in the shape the rest of the app records: the same fields + * readCompletionUsage() returns for the OpenAI-compatible providers, plus the + * cache counts. + * + * promptTokens is everything the model read — uncached input, cache writes + * and cache reads — so the dashboard's token totals mean the same thing for + * every provider; the cache fields say how much of it was cached. + * + * @param {object|null|undefined} usage - `message.usage` + * @returns {{promptTokens:number, completionTokens:number, totalTokens:number, reasoningTokens:null, cacheReadTokens:number, cacheWriteTokens:number}|null} + */ +function mapAnthropicUsage(usage) { + if (!usage || typeof usage !== 'object') return null; + const count = (value) => { + const number = Number(value); + return Number.isFinite(number) && number > 0 ? number : 0; + }; + const cacheReadTokens = count(usage.cache_read_input_tokens); + const cacheWriteTokens = count(usage.cache_creation_input_tokens); + const promptTokens = + count(usage.input_tokens) + cacheReadTokens + cacheWriteTokens; + const completionTokens = count(usage.output_tokens); + return { + promptTokens, + completionTokens, + totalTokens: promptTokens + completionTokens, + reasoningTokens: null, + cacheReadTokens, + cacheWriteTokens, + }; +} + +/** + * Throws when a Messages response did not end with a usable answer. + * + * - `max_tokens`: the same error every provider throws for a cut-off answer + * (code 'ai_response_truncated', partial text on `error.partialText`), so + * the scan loop and the Duplicates review treat it alike. + * - `refusal`: the model declined. Code 'ai_refused', the classifier's + * category in the message. Re-sending the same request would be declined + * again, so it must not look like a JSON problem that OCR could fix. + * + * @param {object} message - the Messages response + * @param {string} provider - name for the message, e.g. 'Anthropic' + * @param {string} remedy - sentence naming the limit to raise + * @param {{partialText?: string}} [extra] + */ +function assertAnthropicCompleted(message, provider, remedy, extra) { + const stopReason = message?.stop_reason; + if (stopReason === 'max_tokens') { + assertCompletionNotTruncated( + { + choices: [{ finish_reason: 'length' }], + usage: { completion_tokens: message?.usage?.output_tokens }, + }, + provider, + remedy, + extra + ); + } + if (stopReason === 'refusal') { + const category = message?.stop_details?.category; + const error = new Error( + `${provider} declined to answer${category ? ` (category: ${category})` : ''}.` + ); + error.code = 'ai_refused'; + if (typeof extra?.partialText === 'string') { + error.partialText = extra.partialText; + } + throw error; + } +} + +module.exports = { + ANTHROPIC_DEFAULT_MODEL, + ANTHROPIC_SUGGESTED_MODELS, + ANTHROPIC_EFFORT_LEVELS, + ANTHROPIC_DEFAULT_MAX_TOKENS, + normalizeAnthropicModel, + normalizeAnthropicEffort, + anthropicModelTraits, + anthropicThinkingParams, + anthropicMaxTokens, + createAnthropicClient, + isConfiguredAnthropicHost, + anthropicMessageText, + mapAnthropicUsage, + assertAnthropicCompleted, +}; diff --git a/services/bootstrapConfigSanitizer.js b/services/bootstrapConfigSanitizer.js index 1b5d1b47..a8de6231 100644 --- a/services/bootstrapConfigSanitizer.js +++ b/services/bootstrapConfigSanitizer.js @@ -8,10 +8,12 @@ const BOOTSTRAP_SECRET_FIELDS = [ 'OLLAMA_API_KEY', 'CUSTOM_API_KEY', 'AZURE_API_KEY', + 'ANTHROPIC_API_KEY', 'OCR_API_KEY', 'MISTRAL_API_KEY', 'API_KEY', 'JWT_SECRET', + 'OIDC_CLIENT_SECRET', ]; // Returns a shallow copy of config with every secret field removed. diff --git a/services/entityMatchAiService.js b/services/entityMatchAiService.js index 1993720f..aafc8f6c 100644 --- a/services/entityMatchAiService.js +++ b/services/entityMatchAiService.js @@ -1088,6 +1088,10 @@ class EntityMatchAiService { runtimeConfig.azure?.deploymentName || null ); + case 'anthropic': + return ( + process.env.ANTHROPIC_MODEL || runtimeConfig.anthropic?.model || null + ); default: return process.env.OPENAI_MODEL || runtimeConfig.openai?.model || null; } diff --git a/services/loggerService.js b/services/loggerService.js index 7d31b73c..422241c2 100644 --- a/services/loggerService.js +++ b/services/loggerService.js @@ -36,6 +36,7 @@ const SECRET_ENV_KEYS = [ 'OPENAI_API_KEY', 'CUSTOM_API_KEY', 'AZURE_API_KEY', + 'ANTHROPIC_API_KEY', 'OLLAMA_API_KEY', 'MISTRAL_API_KEY', 'OCR_API_KEY', @@ -43,6 +44,7 @@ const SECRET_ENV_KEYS = [ 'API_KEY', 'PAPERLESS_AI_API_KEY', 'EXTERNAL_API_KEY', + 'OIDC_CLIENT_SECRET', ]; /** diff --git a/services/mistralOcrService.js b/services/mistralOcrService.js index 26e3ed19..401c79e4 100644 --- a/services/mistralOcrService.js +++ b/services/mistralOcrService.js @@ -5,15 +5,111 @@ // write the extracted markdown text back to Paperless-ngx via PATCH. // Falls back to storing the OCR text locally in the ocr_queue table when // the Paperless PATCH endpoint does not allow writing the content field. +// +// OCR_PROVIDER picks the engine: `mistral` (default), `ollama`/`custom` (a +// local vision model) or `anthropic` (a Claude model via the Messages API). const axios = require('axios'); +const { APIError } = require('@anthropic-ai/sdk'); const config = require('../config/config'); const PaperlessService = require('./paperlessService'); const popplerService = require('./popplerService'); const documentModel = require('../models/document'); const AIServiceFactory = require('./aiServiceFactory'); const documentProcessingService = require('./documentProcessingService'); -const { isTimeoutError, buildTimeoutErrorMessage } = require('./serviceUtils'); +const { + isTimeoutError, + buildTimeoutErrorMessage, + stripTrailingSlashes, +} = require('./serviceUtils'); +const { + ANTHROPIC_DEFAULT_MODEL, + anthropicThinkingParams, + anthropicMaxTokens, + createAnthropicClient, + anthropicMessageText, + mapAnthropicUsage, + assertAnthropicCompleted, +} = require('./anthropicSupport'); + +// ── Claude OCR limits ────────────────────────────────────────────────────── +// +// The Messages API rejects a request body over 32 MB and an image over 10 MB +// (both measured as base64, which is what travels). Checking before sending +// turns a bare 413 into an error that says which limit was hit and what to do +// about it. Decimal megabytes on purpose: if the API means MiB, this errs on +// the safe side by a megabyte and a half. +const CLAUDE_OCR_MAX_REQUEST_BYTES = 32 * 1000 * 1000; +const CLAUDE_OCR_MAX_IMAGE_BYTES = 10 * 1000 * 1000; +// Room for everything in the request besides the document: the JSON around +// it and the instruction text. +const CLAUDE_OCR_REQUEST_RESERVE_BYTES = 64 * 1024; +const CLAUDE_OCR_IMAGE_TYPES = new Set([ + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', +]); + +// A transcript is as long as the document, so the completion cap has to be +// sized for the document rather than for a JSON answer. Every current model +// writes up to 128K tokens; 64000 also fits Claude Haiku 4.5, which stops at +// 64K, so one cap works whichever model the operator picks. It holds roughly +// a hundred dense pages; past it the answer is cut off, which fails the OCR +// run instead of writing half a document. Operators with longer documents +// raise ANTHROPIC_MAX_TOKENS, which OCR honours up to the 128K ceiling. Caps +// this large require streaming. +const CLAUDE_OCR_MAX_TOKENS = 64000; +const CLAUDE_OCR_MAX_TOKENS_CEILING = 128000; + +// What the model answers for a page without any text. The empty-result guard +// in processQueueItem() then refuses to overwrite the document's content, +// instead of a sentence like "This page is blank." becoming its text. +const CLAUDE_OCR_NO_TEXT_MARKER = '[no text]'; + +const CLAUDE_OCR_INSTRUCTION = [ + 'Transcribe all text in this document exactly as it is written, in reading order, page by page.', + '- Keep the original language, spelling, numbers and punctuation. Do not translate, correct, summarize or explain anything.', + '- Reproduce tables as Markdown tables. Keep headings, lists and paragraph breaks.', + '- Include handwriting, stamps and filled-in form values where they are legible.', + '- Write [illegible] where text cannot be read. Do not guess.', + '- Answer with the transcript only: no introduction, no comments, no code fences.', + `- If the document contains no readable text at all, answer with exactly ${CLAUDE_OCR_NO_TEXT_MARKER}`, +].join('\n'); + +/** + * The media type Claude would be told, from a Content-Type value: parameters + * dropped, lower-cased, the common `image/jpg` misspelling fixed. + * + * @param {unknown} mimeType + * @returns {string} + */ +function normalizeClaudeOcrMediaType(mimeType) { + const mediaType = String(mimeType || 'application/pdf') + .split(';')[0] + .trim() + .toLowerCase(); + return mediaType === 'image/jpg' ? 'image/jpeg' : mediaType; +} + +/** A byte count as decimal megabytes with one decimal, for messages. */ +function formatMegabytes(bytes) { + return (bytes / 1000 / 1000).toFixed(1); +} + +/** Whether a base64 PDF leaves no room for the rest of a Messages request. */ +function exceedsClaudeOcrRequestLimit(base64Length) { + return ( + base64Length + CLAUDE_OCR_REQUEST_RESERVE_BYTES > + CLAUDE_OCR_MAX_REQUEST_BYTES + ); +} + +function claudeOcrInputError(message, code) { + const error = new Error(message); + error.code = code; + return error; +} class MistralOcrService { constructor() { @@ -27,6 +123,18 @@ class MistralOcrService { // ── Helpers ────────────────────────────────────────────────────────────── get apiKey() { + if (this.provider === 'anthropic') { + // Not config.mistralOcr.apiKey: it falls back to MISTRAL_API_KEY, and a + // Mistral key left over from an earlier setup must never be sent to the + // Claude API. A dedicated OCR key wins, else the AI provider's key. + return ( + String(process.env.OCR_API_KEY || '').trim() || + config.anthropic?.apiKey || + process.env.ANTHROPIC_API_KEY || + '' + ); + } + const configuredKey = config.mistralOcr?.apiKey || process.env.MISTRAL_API_KEY || ''; if (configuredKey || this.provider !== 'ollama') { @@ -39,6 +147,18 @@ class MistralOcrService { } get model() { + if (this.provider === 'anthropic') { + // MISTRAL_OCR_MODEL is the OCR model field for every provider, and it + // defaults to mistral-ocr-latest, which the Claude API would answer + // with a 404. Only a Claude model name in it counts; otherwise OCR uses + // the model the AI provider is configured with. Proxies sometimes + // prefix the name (anthropic.claude-…, anthropic/claude-…). + const ocrModel = String(config.mistralOcr?.model || '').trim(); + return /(?:^|[./])claude-/i.test(ocrModel) + ? ocrModel + : config.anthropic?.model || ANTHROPIC_DEFAULT_MODEL; + } + return config.mistralOcr?.model || 'mistral-ocr-latest'; } @@ -46,12 +166,46 @@ class MistralOcrService { const normalizedProvider = String(config.mistralOcr?.provider || 'mistral') .trim() .toLowerCase(); + if (normalizedProvider === 'anthropic') { + return 'anthropic'; + } return normalizedProvider === 'ollama' || normalizedProvider === 'custom' ? 'ollama' : 'mistral'; } + /** The provider's name in progress messages and errors. */ + get providerLabel() { + if (this.provider === 'ollama') return 'Local OCR'; + if (this.provider === 'anthropic') return 'Claude OCR'; + return 'Mistral OCR'; + } + + /** + * The completion cap for one Claude OCR request: CLAUDE_OCR_MAX_TOKENS, or + * ANTHROPIC_MAX_TOKENS when the operator raised it above that, never more + * than the 128K output ceiling of the current models. + */ + get anthropicOcrMaxTokens() { + const configured = Number(config.anthropic?.maxTokens) || 0; + return Math.min( + Math.max(CLAUDE_OCR_MAX_TOKENS, configured), + CLAUDE_OCR_MAX_TOKENS_CEILING + ); + } + get apiBase() { + if (this.provider === 'anthropic') { + // OCR_API_URL only when the operator set one (a proxy used for OCR + // alone); otherwise the AI provider's ANTHROPIC_API_URL. Empty means + // api.anthropic.com. + return stripTrailingSlashes( + String( + config.mistralOcr?.apiUrl || config.anthropic?.apiUrl || '' + ).trim() + ); + } + if (this.provider === 'ollama') { const ollamaDefault = config.ollama?.apiUrl || @@ -168,7 +322,8 @@ class MistralOcrService { * @param {string} mimeType - MIME type of the document * @param {number|null} documentId - Paperless document ID (local provider only) * @param {{onPageProgress?: (pageNumber: number, pageCount: number) => void}} opts - * - progress hooks; only used by the local provider's PDF render path + * - progress hooks; only used by the PDF render paths (local provider, and + * Claude for a PDF over its request limit) * @returns {Promise} - Extracted text */ async performOcr( @@ -181,6 +336,10 @@ class MistralOcrService { return this.performOcrWithOllama(base64, mimeType, documentId, opts); } + if (this.provider === 'anthropic') { + return this.performOcrWithAnthropic(base64, mimeType, documentId, opts); + } + return this.performOcrWithMistral(base64, mimeType); } @@ -244,6 +403,311 @@ class MistralOcrService { .trim(); } + /** + * OCR with the configured Claude model (OCR_PROVIDER=anthropic). + * + * A PDF goes to Claude whole, as one document; Claude reads its text layer + * and its page images together. Only a PDF over the 32 MB request limit + * takes another route: the poppler rendering the local provider uses, one + * page image per request, because a page image is far smaller than the + * PDF around it. Without poppler that PDF fails with the limit named. + * + * @param {string} base64 + * @param {string} mimeType + * @param {number|null} documentId - log context for the render path + * @param {{onPageProgress?: (pageNumber: number, pageCount: number) => void}} opts + * @returns {Promise} + */ + async performOcrWithAnthropic( + base64, + mimeType = 'application/pdf', + documentId = null, + opts = {} + ) { + const apiKey = this.apiKey; + if (!apiKey) { + throw new Error( + 'Claude OCR needs an API key: set ANTHROPIC_API_KEY, or OCR_API_KEY for a key used only for OCR' + ); + } + + const request = { + apiKey, + apiUrl: this.apiBase, + model: this.model, + timeoutMs: this.ocrTimeoutMs, + maxTokens: this.anthropicOcrMaxTokens, + }; + + if ( + normalizeClaudeOcrMediaType(mimeType) === 'application/pdf' && + exceedsClaudeOcrRequestLimit(String(base64 || '').length) + ) { + return this._performAnthropicOcrOnRenderedPdf( + base64, + documentId, + opts, + request + ); + } + + return this.runAnthropicOcr({ ...request, base64, mimeType }); + } + + /** + * The fallback for a PDF too large for one Claude request: render its pages + * with poppler (same OCR_PDF_RENDER_* settings as local OCR) and transcribe + * each page image on its own. + */ + async _performAnthropicOcrOnRenderedPdf(base64, documentId, opts, request) { + const limit = `The PDF is ${formatMegabytes(String(base64).length)} MB as base64; Claude accepts at most ${formatMegabytes(CLAUDE_OCR_MAX_REQUEST_BYTES)} MB per request`; + + if (!this.pdfRenderEnabled) { + throw claudeOcrInputError( + `${limit}. Enable OCR_PDF_RENDER_ENABLED to send it page by page instead.`, + 'claude_ocr_too_large' + ); + } + if (!(await popplerService.isAvailable())) { + throw claudeOcrInputError( + `${limit}. Install poppler-utils (pdftoppm) to send it page by page instead.`, + 'claude_ocr_too_large' + ); + } + + console.warn( + `[OCR] Document ${documentId}: ${limit} — sending up to ${this.pdfRenderMaxPages} rendered page(s) as images instead` + ); + const rendered = await popplerService.renderPdfToImages( + Buffer.from(base64, 'base64'), + { maxPages: this.pdfRenderMaxPages, dpi: this.pdfRenderDpi } + ); + + return this._performOcrOnRenderedPages(rendered, documentId, opts, { + label: 'Claude OCR', + ocrImage: (pageBase64, pageMimeType) => + this.runAnthropicOcr({ + ...request, + base64: pageBase64, + mimeType: pageMimeType, + }), + }); + } + + /** + * OCR one document with a Claude model (OCR_PROVIDER=anthropic). + * + * Takes every connection detail as an argument so the setup validation can + * try a key and model before they are saved; performOcr() passes the + * configured ones. Nothing here reads config. + * + * One Messages request: the document block (PDF) or image block first, the + * transcription instruction after it. Thinking is turned down as far as the + * model allows — copying text needs none, and thinking tokens count toward + * the cap. The answer is streamed because a transcript can run far past the + * size a non-streaming request may wait for. + * + * A cut-off or declined answer throws (assertAnthropicCompleted), so the + * caller fails the OCR run exactly as it does for a truncated local OCR + * page instead of writing part of a document back to Paperless-ngx. + * + * @param {object} params + * @param {string} params.apiKey + * @param {string} [params.apiUrl] - proxy only; empty means api.anthropic.com + * @param {string} params.model + * @param {string} params.base64 - the document, base64-encoded (a data URL + * is accepted too) + * @param {string} params.mimeType - application/pdf or an image type + * @param {number} [params.timeoutMs] - how long the request may go without + * progress: until the answer starts, and between two streamed events + * @param {number} [params.maxTokens] - completion cap; default 64000 + * @returns {Promise} the transcribed text; empty for a document + * without readable text + */ + async runAnthropicOcr({ + apiKey, + apiUrl = '', + model, + base64, + mimeType, + timeoutMs, + maxTokens, + } = {}) { + const key = String(apiKey || '').trim(); + if (!key) { + throw new Error('Claude OCR needs an API key'); + } + + const source = this._prepareClaudeOcrSource(base64, mimeType); + const modelName = String(model || '').trim() || ANTHROPIC_DEFAULT_MODEL; + const cap = anthropicMaxTokens(maxTokens, CLAUDE_OCR_MAX_TOKENS); + + const params = { + model: modelName, + max_tokens: cap, + ...anthropicThinkingParams(modelName, { reasoning: false }), + messages: [ + { + role: 'user', + content: [ + { + type: source.blockType, + source: { + type: 'base64', + media_type: source.mediaType, + data: source.data, + }, + }, + { type: 'text', text: CLAUDE_OCR_INSTRUCTION }, + ], + }, + ], + }; + + const client = createAnthropicClient({ apiKey: key, apiUrl, timeoutMs }); + const message = await this._streamClaudeOcr(client, params, timeoutMs); + + const usage = mapAnthropicUsage(message?.usage); + if (usage) { + console.log( + `[OCR] Claude OCR (${modelName}): ${usage.promptTokens} input / ${usage.completionTokens} output tokens` + ); + } + + assertAnthropicCompleted( + message, + 'Claude OCR', + cap < CLAUDE_OCR_MAX_TOKENS_CEILING + ? `The transcript needs more room: raise ANTHROPIC_MAX_TOKENS (up to ${CLAUDE_OCR_MAX_TOKENS_CEILING}) or split the document.` + : 'The transcript is longer than one answer can be; split the document into smaller parts.' + ); + + const text = anthropicMessageText(message); + return text.toLowerCase() === CLAUDE_OCR_NO_TEXT_MARKER ? '' : text; + } + + /** + * The document as a Messages content source, or an error naming the limit + * it breaks. Checked here rather than left to the API so the setup + * validation and the OCR queue both get a message that says what to do. + * + * @param {string} base64 + * @param {string} mimeType + * @returns {{blockType: 'document'|'image', mediaType: string, data: string}} + */ + _prepareClaudeOcrSource(base64, mimeType) { + let data = String(base64 || ''); + let declaredType = mimeType; + const dataUrl = /^data:([^;,]+);base64,/i.exec(data); + if (dataUrl) { + data = data.slice(dataUrl[0].length); + declaredType = declaredType || dataUrl[1]; + } + // The API wants base64 without line breaks; MIME-style wrapped input + // would be rejected. + if (/\s/.test(data)) { + data = data.replace(/\s+/g, ''); + } + if (!data) { + throw claudeOcrInputError( + 'Claude OCR received an empty document', + 'claude_ocr_empty_input' + ); + } + + const mediaType = normalizeClaudeOcrMediaType(declaredType); + + if (mediaType === 'application/pdf') { + if (exceedsClaudeOcrRequestLimit(data.length)) { + throw claudeOcrInputError( + `The PDF is ${formatMegabytes(data.length)} MB as base64; Claude accepts at most ${formatMegabytes(CLAUDE_OCR_MAX_REQUEST_BYTES)} MB per request.`, + 'claude_ocr_too_large' + ); + } + return { blockType: 'document', mediaType, data }; + } + + if (CLAUDE_OCR_IMAGE_TYPES.has(mediaType)) { + if (data.length > CLAUDE_OCR_MAX_IMAGE_BYTES) { + throw claudeOcrInputError( + `The image is ${formatMegabytes(data.length)} MB as base64; Claude accepts at most ${formatMegabytes(CLAUDE_OCR_MAX_IMAGE_BYTES)} MB per image.`, + 'claude_ocr_too_large' + ); + } + return { blockType: 'image', mediaType, data }; + } + + // Paperless-ngx serves its archived PDF when it has one, so this is an + // original it could not convert (TIFF, office formats, …). + throw claudeOcrInputError( + `Claude OCR cannot read ${mediaType} files; it accepts PDF, PNG, JPEG, GIF and WebP. ` + + 'Paperless-ngx has no archived PDF of this document to send instead.', + 'claude_ocr_unsupported_type' + ); + } + + /** + * Streams one Messages request and returns the final message. + * + * The SDK's own timeout only covers the wait for the response headers; a + * stream that stalls after them would hold the queue item in 'processing' + * for good. So a watchdog aborts the request once it has gone `timeoutMs` + * without progress — before it starts, or between two events — and a long + * transcript that keeps arriving is never cut short. + */ + async _streamClaudeOcr(client, params, timeoutMs) { + const stream = client.messages.stream(params); + let stalled = false; + const watchdog = + typeof timeoutMs === 'number' && timeoutMs > 0 + ? setTimeout(() => { + stalled = true; + stream.abort(); + }, timeoutMs) + : null; + if (watchdog) { + const touch = () => watchdog.refresh(); + stream.on('connect', touch).on('streamEvent', touch); + } + + try { + return await stream.finalMessage(); + } catch (error) { + if (stalled || isTimeoutError(error)) { + console.error( + `[TIMEOUT][OCR] Claude OCR request timed out: ${error.message}` + ); + throw new Error(buildTimeoutErrorMessage('OCR', timeoutMs), { + cause: error, + }); + } + throw this._describeClaudeApiError(error); + } finally { + if (watchdog) clearTimeout(watchdog); + } + } + + /** + * An API error as one readable line. The SDK puts the whole JSON error body + * into its message; the queue and the setup wizard show this text as is. + */ + _describeClaudeApiError(error) { + if (!(error instanceof APIError) || !error.status) { + return error; + } + const body = error.error?.error; + const type = body?.type ? ` (${body.type})` : ''; + const detail = body?.message || error.message; + const hint = + error.status === 413 + ? ` Claude accepts at most ${formatMegabytes(CLAUDE_OCR_MAX_REQUEST_BYTES)} MB per request.` + : ''; + return new Error( + `Claude API answered HTTP ${error.status}${type}: ${detail}${hint}`, + { cause: error } + ); + } + /** * OCR a document with a local vision model. Images are sent as-is; PDFs are * rendered to per-page images via poppler (pdftoppm) when enabled and @@ -296,15 +760,27 @@ class MistralOcrService { } /** - * OCR every rendered PDF page through the local vision model and join the - * page texts. A failed page is skipped so one bad page cannot discard the - * rest; if no page succeeds, the last error is thrown. + * OCR every rendered PDF page through the local vision model (or the + * engine `pageOcr` names) and join the page texts. A failed page is skipped + * so one bad page cannot discard the rest; if no page succeeds, the last + * error is thrown. * @param {{pages: Array<{base64: string, mimeType: string, pageNumber: number}>, totalPages: number|null, truncated: boolean}} rendered * @param {number|null} documentId - log context only * @param {{onPageProgress?: (pageNumber: number, pageCount: number) => void}} opts + * @param {{label?: string, ocrImage?: (base64: string, mimeType: string) => Promise}} [pageOcr] + * - the engine for one page image and its name in errors; local OCR by default * @returns {Promise} */ - async _performOcrOnRenderedPages(rendered, documentId, opts = {}) { + async _performOcrOnRenderedPages( + rendered, + documentId, + opts = {}, + { + label = 'Local OCR', + ocrImage = (imageBase64, imageMimeType) => + this._performLocalOcrOnImage(imageBase64, imageMimeType), + } = {} + ) { const { pages, totalPages, truncated } = rendered; if (truncated) { @@ -322,10 +798,7 @@ class MistralOcrService { } try { - const pageText = await this._performLocalOcrOnImage( - page.base64, - page.mimeType - ); + const pageText = await ocrImage(page.base64, page.mimeType); if (pageText) { pageTexts.push(pageText); } @@ -342,7 +815,7 @@ class MistralOcrService { if (pageTexts.length === 0) { throw ( lastError || - new Error('Local OCR returned empty output for all rendered PDF pages') + new Error(`${label} returned empty output for all rendered PDF pages`) ); } @@ -811,8 +1284,7 @@ class MistralOcrService { emit('download', `Download complete (${mimeType}).`); // Step 2: OCR - const providerLabel = - this.provider === 'ollama' ? 'Local OCR' : 'Mistral OCR'; + const providerLabel = this.providerLabel; emit('ocr', `Sending document to ${providerLabel}…`); let ocrText; try { diff --git a/services/oidcService.js b/services/oidcService.js new file mode 100644 index 00000000..3eba124b --- /dev/null +++ b/services/oidcService.js @@ -0,0 +1,734 @@ +/** + * OidcService + * + * Single sign-on through any OpenID Connect provider (Authentik, Authelia, + * Keycloak, Kanidm, Pocket ID, Zitadel, Entra ID, ...). It implements the + * standard authorization code flow with PKCE, state and nonce, and nothing + * provider-specific: endpoints and capabilities come from the provider's + * discovery document. + * + * Zettelrobbe has exactly one account, the administrator created by the setup + * wizard. SSO therefore does not provision or manage users: an identity whose + * e-mail address equals OIDC_ADMIN_EMAIL signs in as that account, every + * other identity is refused. Who may reach the application at all is decided + * at the identity provider. + * + * The protocol work is delegated to openid-client, which validates the ID + * token (signature, issuer, audience, expiry, nonce) and the PKCE exchange. + */ + +const config = require('../config/config'); +const { stripTrailingSlashes } = require('./serviceUtils'); + +// Loaded on first use, not at startup: an install that never turns SSO on +// keeps running even on an image whose dependencies predate openid-client. +let openidClient = null; +function getOpenidClient() { + if (!openidClient) { + openidClient = require('openid-client'); + } + return openidClient; +} + +const CALLBACK_PATH = '/auth/oidc/callback'; +const DISCOVERY_SUFFIX = '/.well-known/openid-configuration'; +// Discovery and token requests should not leave a login hanging. +const REQUEST_TIMEOUT_SECONDS = 10; +const DEFAULT_BUTTON_TEXT = 'Sign in with SSO'; +const MAX_BUTTON_TEXT_LENGTH = 60; + +const TOKEN_AUTH_METHODS = [ + 'client_secret_basic', + 'client_secret_post', + 'none', +]; + +function stripTrailingSlash(value) { + return stripTrailingSlashes(String(value)); +} + +/** + * Light syntax check for an e-mail address: one "@", no whitespace, and a dot + * in the domain. The address only has to be comparable with what the + * provider sends. + * + * @param {*} value + * @returns {boolean} + */ +function isValidEmail(value) { + const text = String(value ?? '').trim(); + return text.length <= 254 && /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(text); +} + +/** + * True for an absolute http(s) URL. A redirect URI without its scheme + * ("app.example.com/auth/oidc/callback") is a common slip that every provider + * rejects with a redirect URI error. + * + * @param {string} value + * @returns {boolean} + */ +function isAbsoluteHttpUrl(value) { + try { + const url = new URL(String(value || '').trim()); + return url.protocol === 'https:' || url.protocol === 'http:'; + } catch { + return false; + } +} + +/** + * The issuer URL to discover. Operators often paste the full discovery URL + * instead of the issuer; its well-known suffix is removed. + * + * @param {string} value + * @returns {string} + */ +function normalizeIssuerUrl(value) { + const text = String(value || '').trim(); + return text.endsWith(DISCOVERY_SUFFIX) + ? text.slice(0, -DISCOVERY_SUFFIX.length) + : text; +} + +/** + * A colour for the login button, accepted only in plain CSS colour syntax: + * #rgb / #rrggbb (with optional alpha), a named colour, or an rgb()/hsl() + * function. Anything else is rejected because the value lands in a style + * attribute. + * + * @param {string} value + * @returns {boolean} + */ +function isSafeCssColor(value) { + const text = String(value || '').trim(); + return ( + /^#(?:[0-9a-f]{3,4}|[0-9a-f]{6}|[0-9a-f]{8})$/i.test(text) || + /^[a-z]{3,30}$/i.test(text) || + /^(?:rgb|rgba|hsl|hsla)\([0-9a-z.,%/\s]{1,60}\)$/i.test(text) + ); +} + +/** + * The icon shown on the login button. Empty means the built-in key icon, + * "none" hides it, and an http(s) URL or a same-origin path ("/logo.png") + * shows that image. Anything else falls back to the built-in icon. + * + * @param {string} value + * @returns {{ type: 'default'|'none'|'image', src?: string }} + */ +function resolveButtonIcon(value) { + const text = String(value || '').trim(); + if (!text) return { type: 'default' }; + if (text.toLowerCase() === 'none') return { type: 'none' }; + if (/^\/(?!\/)\S*$/.test(text)) return { type: 'image', src: text }; + try { + const url = new URL(text); + if (url.protocol === 'https:' || url.protocol === 'http:') { + return { type: 'image', src: url.href }; + } + } catch { + // Not a URL; fall through to the default icon. + } + return { type: 'default' }; +} + +/** + * Picks how the client authenticates at the token endpoint. + * + * OIDC_TOKEN_AUTH_METHOD wins when set. Without a client secret the client is + * public ("none"). Otherwise client_secret_basic, the default of the OAuth 2.0 + * and OpenID Connect specifications, unless the provider's discovery document + * lists only client_secret_post. + * + * @param {{ tokenAuthMethod: string, clientSecret: string }} settings + * @param {{ token_endpoint_auth_methods_supported?: string[] }} serverMetadata + * @returns {string} + */ +function selectTokenAuthMethod(settings, serverMetadata) { + const configured = String(settings.tokenAuthMethod || '') + .trim() + .toLowerCase(); + if (configured) { + if (!TOKEN_AUTH_METHODS.includes(configured)) { + throw new Error( + `Unsupported OIDC_TOKEN_AUTH_METHOD "${configured}". Use one of: ${TOKEN_AUTH_METHODS.join(', ')}.` + ); + } + return configured; + } + + if (!settings.clientSecret) { + return 'none'; + } + + const supported = serverMetadata?.token_endpoint_auth_methods_supported; + if ( + Array.isArray(supported) && + !supported.includes('client_secret_basic') && + supported.includes('client_secret_post') + ) { + return 'client_secret_post'; + } + return 'client_secret_basic'; +} + +/** + * The scopes to request, always including "openid". + * + * @param {{ scopes: string }} settings + * @returns {string} + */ +function resolveScopes(settings) { + const scopes = String(settings.scopes || '') + .split(/\s+/) + .filter(Boolean); + if (!scopes.includes('openid')) { + scopes.unshift('openid'); + } + return scopes.join(' '); +} + +/** + * A short, readable reason for a failed request. openid-client wraps network + * errors, so the useful part (ECONNREFUSED, a timeout, a certificate problem) + * often sits in the cause. + * + * @param {Error} error + * @returns {string} + */ +function describeError(error) { + const cause = error?.cause; + const causeText = + cause && typeof cause === 'object' && (cause.code || cause.message) + ? ` (${cause.code || cause.message})` + : ''; + return `${error?.message || String(error)}${causeText}`; +} + +function buildClientAuthentication(client, method, clientSecret) { + if (method === 'none') return client.None(); + if (!clientSecret) { + throw new Error( + `OIDC_TOKEN_AUTH_METHOD=${method} needs OIDC_CLIENT_SECRET to be set.` + ); + } + return method === 'client_secret_post' + ? client.ClientSecretPost(clientSecret) + : client.ClientSecretBasic(clientSecret); +} + +class OidcService { + constructor() { + this._clientConfigPromise = null; + this._clientConfigKey = null; + } + + /** @returns {ReturnType} */ + getSettings() { + return config.getOidcConfig(); + } + + /** + * What is missing or invalid in the configuration. Empty when SSO is off or + * fully configured. Without an argument it checks the running + * configuration; the settings page passes the values it is about to save. + * + * @param {{ enabled: boolean, issuerUrl: string, clientId: string, redirectUri: string, adminEmail: string }} [settings] + * @returns {string[]} + */ + getConfigErrors(settings = this.getSettings()) { + if (!settings.enabled) return []; + + const errors = []; + try { + const issuer = new URL(normalizeIssuerUrl(settings.issuerUrl)); + if (issuer.protocol !== 'https:' && issuer.protocol !== 'http:') { + errors.push('OIDC_ISSUER_URL must be an http(s) URL'); + } + } catch { + errors.push('OIDC_ISSUER_URL is missing or not a URL'); + } + if (!settings.clientId) { + errors.push('OIDC_CLIENT_ID is missing'); + } + if (settings.redirectUri && !isAbsoluteHttpUrl(settings.redirectUri)) { + errors.push( + 'OIDC_REDIRECT_URI must be an absolute URL starting with https:// or http://' + ); + } + if (!isValidEmail(settings.adminEmail)) { + errors.push('OIDC_ADMIN_EMAIL is missing or not an e-mail address'); + } + return errors; + } + + /** + * True when SSO is switched on and completely configured. A half-configured + * provider is treated as off rather than shown as a button that can only + * fail; logStatus() tells the operator why. + * + * @returns {boolean} + */ + isEnabled() { + return this.getSettings().enabled && this.getConfigErrors().length === 0; + } + + /** + * Logs once at startup whether SSO is active and, when it is switched on + * but incomplete, what is missing. + */ + logStatus() { + const settings = this.getSettings(); + if (!settings.enabled) return; + const errors = this.getConfigErrors(); + if (errors.length > 0) { + console.warn( + `[WARN] OIDC_ENABLED=yes but single sign-on stays off: ${errors.join('; ')}.` + ); + return; + } + console.log( + `[INFO] Single sign-on enabled with issuer ${normalizeIssuerUrl(settings.issuerUrl)}` + ); + } + + /** + * What the login page needs to render the SSO button. + * + * @returns {{ enabled: boolean, text: string, icon: { type: string, src?: string }, style: string }} + */ + getLoginButton() { + const settings = this.getSettings(); + const text = + settings.buttonText.slice(0, MAX_BUTTON_TEXT_LENGTH) || + DEFAULT_BUTTON_TEXT; + const style = []; + if (isSafeCssColor(settings.buttonColor)) { + style.push(`--login-sso-bg: ${settings.buttonColor}`); + } + if (isSafeCssColor(settings.buttonTextColor)) { + style.push(`--login-sso-fg: ${settings.buttonTextColor}`); + } + return { + enabled: this.isEnabled(), + text, + icon: resolveButtonIcon(settings.buttonIcon), + style: style.join('; '), + }; + } + + /** + * Discovers the provider once and reuses the result. The cache is keyed on + * the settings it was built from, so changed settings take effect, and a + * failed discovery is not cached, so a provider that was down at the first + * attempt is retried on the next login. + * + * @returns {Promise} + */ + async getClientConfig() { + const settings = this.getSettings(); + const key = JSON.stringify([ + settings.issuerUrl, + settings.clientId, + settings.clientSecret, + settings.tokenAuthMethod, + ]); + + if (this._clientConfigPromise && this._clientConfigKey === key) { + return this._clientConfigPromise; + } + + this._clientConfigKey = key; + this._clientConfigPromise = this._buildClientConfig(settings).catch( + (error) => { + if (this._clientConfigKey === key) { + this._clientConfigPromise = null; + this._clientConfigKey = null; + } + throw error; + } + ); + + return this._clientConfigPromise; + } + + /** + * Reads the provider's discovery document and builds the client from it. + * + * @param {ReturnType} settings + * @returns {Promise} + */ + async _buildClientConfig(settings) { + const { client, serverMetadata, insecure } = + await this._discoverProvider(settings); + const authMethod = selectTokenAuthMethod(settings, serverMetadata); + return this._createClientConfig( + client, + serverMetadata, + settings, + authMethod, + insecure + ); + } + + /** + * Reads the provider's discovery document. + * + * @param {{ issuerUrl: string, clientId: string }} settings + * @returns {Promise<{ client: object, serverMetadata: object, insecure: boolean }>} + */ + async _discoverProvider(settings) { + const client = getOpenidClient(); + const issuer = new URL(normalizeIssuerUrl(settings.issuerUrl)); + const insecure = issuer.protocol === 'http:'; + const options = { timeout: REQUEST_TIMEOUT_SECONDS }; + if (insecure) { + // Plain HTTP is only acceptable on a trusted network, but homelab + // providers sometimes run that way behind the same reverse proxy. + console.warn( + '[WARN] OIDC_ISSUER_URL uses plain HTTP. Use HTTPS unless the provider is only reachable on a trusted network.' + ); + options.execute = [client.allowInsecureRequests]; + } + + const serverMetadata = await this._discover( + client, + issuer, + settings, + options + ); + return { client, serverMetadata, insecure }; + } + + /** + * The openid-client configuration for a discovered provider and a chosen + * token endpoint auth method. + * + * @returns {import('openid-client').Configuration} + */ + _createClientConfig(client, serverMetadata, settings, authMethod, insecure) { + const clientConfig = new client.Configuration( + serverMetadata, + settings.clientId, + undefined, + buildClientAuthentication(client, authMethod, settings.clientSecret) + ); + clientConfig.timeout = REQUEST_TIMEOUT_SECONDS; + if (insecure) { + client.allowInsecureRequests(clientConfig); + } + return clientConfig; + } + + /** + * Checks a provider from the settings page without signing anybody in: + * discovery, the token endpoint auth method that would be used, and the + * client credentials. The credentials are checked by redeeming an + * authorization code that cannot exist. The token endpoint authenticates + * the client before it looks at the code (RFC 6749, section 5.2), so + * "invalid_client" means the client ID, secret or auth method is wrong, + * while "invalid_grant" means the client was accepted. + * + * Throws an Error whose message is meant for the settings page when the + * provider cannot be used. + * + * @param {{ issuerUrl: string, clientId: string, clientSecret: string, tokenAuthMethod: string }} settings + * @returns {Promise<{ issuer: string, tokenAuthMethods: string[], chosenAuthMethod: string, pkce: boolean }>} + */ + async testConnection(settings) { + const discoveryUrl = `${stripTrailingSlash(normalizeIssuerUrl(settings.issuerUrl))}${DISCOVERY_SUFFIX}`; + let discovered; + try { + discovered = await this._discoverProvider(settings); + } catch (error) { + const advertised = error?.cause?.body?.issuer; + if ( + error?.code === 'OAUTH_JSON_ATTRIBUTE_COMPARISON_FAILED' && + typeof advertised === 'string' + ) { + throw new Error( + `The provider calls itself "${advertised}". Use exactly that as the issuer URL.`, + { cause: error } + ); + } + throw new Error( + `No usable discovery document at ${discoveryUrl}: ${describeError(error)}`, + { cause: error } + ); + } + + const { client, serverMetadata, insecure } = discovered; + const chosenAuthMethod = selectTokenAuthMethod(settings, serverMetadata); + const clientConfig = this._createClientConfig( + client, + serverMetadata, + settings, + chosenAuthMethod, + insecure + ); + + try { + await client.genericGrantRequest(clientConfig, 'authorization_code', { + code: client.randomState(), + redirect_uri: `${new URL(serverMetadata.issuer).origin}${CALLBACK_PATH}`, + code_verifier: client.randomPKCECodeVerifier(), + }); + } catch (error) { + const code = error?.error; + if (code === 'invalid_client' || error?.status === 401) { + throw new Error( + `The provider refused the client (${code || `HTTP ${error.status}`}). Check the client ID, the client secret and the token endpoint auth method (${chosenAuthMethod}).`, + { cause: error } + ); + } + if (!(error instanceof client.ResponseBodyError)) { + throw new Error( + `The token endpoint could not be reached: ${describeError(error)}`, + { cause: error } + ); + } + // Any other OAuth error (invalid_grant, as expected) means the client + // was authenticated and only the made-up code was refused. + } + + const advertisedMethods = + serverMetadata.token_endpoint_auth_methods_supported; + const challengeMethods = serverMetadata.code_challenge_methods_supported; + return { + issuer: serverMetadata.issuer, + tokenAuthMethods: Array.isArray(advertisedMethods) + ? advertisedMethods + : [], + chosenAuthMethod, + pkce: + Array.isArray(challengeMethods) && challengeMethods.includes('S256'), + }; + } + + /** + * Runs discovery and returns the provider metadata. OpenID Connect demands + * that the configured issuer equals the advertised one character for + * character, and providers disagree on the trailing slash (Authentik has + * one, Keycloak and Authelia do not). A mismatch that is only that slash is + * retried with the provider's spelling instead of failing the login. + * + * @returns {Promise} + */ + async _discover(client, issuer, settings, options) { + try { + const discovered = await client.discovery( + issuer, + settings.clientId, + undefined, + undefined, + options + ); + return discovered.serverMetadata(); + } catch (error) { + const advertised = error?.cause?.body?.issuer; + const differsOnlyBySlash = + error?.code === 'OAUTH_JSON_ATTRIBUTE_COMPARISON_FAILED' && + typeof advertised === 'string' && + stripTrailingSlash(advertised) === stripTrailingSlash(issuer.href); + if (!differsOnlyBySlash) { + throw error; + } + const discovered = await client.discovery( + new URL(advertised), + settings.clientId, + undefined, + undefined, + options + ); + return discovered.serverMetadata(); + } + } + + /** + * The redirect URI registered with the provider. OIDC_REDIRECT_URI wins; + * otherwise it is derived from the request, which honours TRUST_PROXY for + * the protocol and host when the app runs behind a reverse proxy. + * + * @param {import('express').Request} req + * @returns {string} + */ + resolveRedirectUri(req) { + const settings = this.getSettings(); + if (settings.redirectUri) { + return settings.redirectUri; + } + return this.deriveRedirectUri(req); + } + + /** + * The callback URL as seen from the request, ignoring OIDC_REDIRECT_URI. + * The settings page shows it as the value to register with the provider. + * + * @param {import('express').Request} req + * @returns {string} + */ + deriveRedirectUri(req) { + return `${req.protocol}://${req.get('host')}${CALLBACK_PATH}`; + } + + /** + * Builds the provider's authorization URL and the values the callback must + * check it against. + * + * @param {import('express').Request} req + * @returns {Promise<{ url: string, transaction: { state: string, nonce: string, codeVerifier: string, redirectUri: string } }>} + */ + async startLogin(req) { + const client = getOpenidClient(); + const settings = this.getSettings(); + const clientConfig = await this.getClientConfig(); + const redirectUri = this.resolveRedirectUri(req); + + const codeVerifier = client.randomPKCECodeVerifier(); + const codeChallenge = await client.calculatePKCECodeChallenge(codeVerifier); + const state = client.randomState(); + const nonce = client.randomNonce(); + + const url = client.buildAuthorizationUrl(clientConfig, { + redirect_uri: redirectUri, + scope: resolveScopes(settings), + code_challenge: codeChallenge, + code_challenge_method: 'S256', + state, + nonce, + }); + + return { + url: url.href, + transaction: { state, nonce, codeVerifier, redirectUri }, + }; + } + + /** + * Exchanges the authorization code and returns the user's claims: the + * validated ID token claims, completed from the userinfo endpoint where the + * provider offers one (some providers keep the e-mail out of the ID token). + * + * @param {URLSearchParams|string} callbackQuery - the query string the provider redirected back with + * @param {{ state: string, nonce: string, codeVerifier: string, redirectUri: string }} transaction + * @returns {Promise} + */ + async completeLogin(callbackQuery, transaction) { + const client = getOpenidClient(); + const clientConfig = await this.getClientConfig(); + + // Rebuild the callback URL from the redirect URI that was sent to the + // provider rather than from the incoming request: openid-client sends + // this URL (minus the query) as redirect_uri in the token request, and it + // must match the authorization request exactly, whatever a reverse proxy + // did to the Host header in between. + const currentUrl = new URL(transaction.redirectUri); + currentUrl.search = new URLSearchParams(callbackQuery).toString(); + + const tokens = await client.authorizationCodeGrant( + clientConfig, + currentUrl, + { + pkceCodeVerifier: transaction.codeVerifier, + expectedState: transaction.state, + expectedNonce: transaction.nonce, + idTokenExpected: true, + } + ); + + const idClaims = tokens.claims() || {}; + let userInfo = {}; + if (clientConfig.serverMetadata().userinfo_endpoint) { + try { + userInfo = await client.fetchUserInfo( + clientConfig, + tokens.access_token, + idClaims.sub + ); + } catch (error) { + // The ID token alone may be enough; userinfo only adds claims. + console.warn( + '[WARN] OIDC userinfo request failed, continuing with ID token claims:', + error.message + ); + } + } + + // The ID token is authoritative for anything it carries. + return { ...userInfo, ...idClaims }; + } + + /** + * Decides whether an identity may sign in. Pure: it only looks at the + * claims and the given list of local users. + * + * The e-mail claim (OIDC_EMAIL_CLAIM, default "email") must equal + * OIDC_ADMIN_EMAIL, ignoring case. The identity then signs in as the one + * local account, whatever its username. + * + * @param {object} claims - the user's claims from completeLogin() + * @param {Array<{ id: number, username: string }>} users - local accounts + * @returns {{ user: object|null, email: string|null, error: string|null }} + */ + authorize(claims, users) { + const settings = this.getSettings(); + const value = claims ? claims[settings.emailClaim] : undefined; + const email = typeof value === 'string' && value.trim() ? value.trim() : ''; + + if (!email) { + return { + user: null, + email: null, + error: `The identity provider did not send the "${settings.emailClaim}" claim. Request the "email" scope (OIDC_SCOPES) or set OIDC_EMAIL_CLAIM.`, + }; + } + + // A provider that marks the address unverified may have let somebody + // register or edit it freely, so it cannot vouch for the admin. A claim + // the provider leaves out is not a "no". + const unverified = + claims.email_verified === false || claims.email_verified === 'false'; + if (unverified && !settings.allowUnverifiedEmail) { + return { + user: null, + email, + error: + 'The identity provider reports this e-mail address as unverified. Verify it there, or set OIDC_ALLOW_UNVERIFIED_EMAIL=yes if the provider never confirms addresses.', + }; + } + + if (email.toLowerCase() !== settings.adminEmail.toLowerCase()) { + return { + user: null, + email, + error: + 'This account may not sign in to Zettelrobbe. Its e-mail address does not match OIDC_ADMIN_EMAIL.', + }; + } + + const user = Array.isArray(users) && users.length > 0 ? users[0] : null; + if (!user) { + return { + user: null, + email, + error: + 'Zettelrobbe has no administrator account yet. Finish the setup first.', + }; + } + + return { user, email, error: null }; + } +} + +module.exports = new OidcService(); +module.exports.CALLBACK_PATH = CALLBACK_PATH; +module.exports.TOKEN_AUTH_METHODS = TOKEN_AUTH_METHODS; +module.exports.MAX_BUTTON_TEXT_LENGTH = MAX_BUTTON_TEXT_LENGTH; +module.exports.isAbsoluteHttpUrl = isAbsoluteHttpUrl; +module.exports.isValidEmail = isValidEmail; +module.exports.isSafeCssColor = isSafeCssColor; +module.exports.normalizeIssuerUrl = normalizeIssuerUrl; +module.exports.resolveButtonIcon = resolveButtonIcon; +module.exports.resolveScopes = resolveScopes; +module.exports.selectTokenAuthMethod = selectTokenAuthMethod; diff --git a/services/serviceUtils.js b/services/serviceUtils.js index 1745cf17..a25bb5d9 100644 --- a/services/serviceUtils.js +++ b/services/serviceUtils.js @@ -77,6 +77,11 @@ async function calculateTokens( text, model = process.env.OPENAI_MODEL || 'gpt-4o-mini' ) { + if (isClaudeModel(model)) { + console.log(`[DEBUG] Using Claude token estimation for model: ${model}`); + return estimateTokensForClaude(text); + } + try { const compatibleModel = getCompatibleModel(model); @@ -163,6 +168,13 @@ async function truncateToTokenLimit( ); } + if (isClaudeModel(model)) { + console.log( + `[DEBUG] Using Claude character-based truncation for model: ${model}` + ); + return truncateForClaude(text, maxTokens); + } + try { const compatibleModel = getCompatibleModel(model); @@ -1025,6 +1037,63 @@ function estimateTokenCount(textOrLength) { return Math.ceil(length / 4); } +/** + * Claude models, by the name the Claude API knows them under. + * + * Their tokenizers are not tiktoken's, and the newer ones count noticeably + * more tokens per character than the ÷4 rule assumes (the current generation + * counts the same text as roughly 30% more tokens than the one before). An + * estimate that comes out low lets through a document the budget meant to + * cut, so Claude gets its own, more conservative rate. Every other model keeps + * the estimate it always had. + */ +const CLAUDE_MODEL_PATTERN = /^claude-/i; +const CLAUDE_CHARS_PER_TOKEN = 3; + +/** + * @param {unknown} model + * @returns {boolean} + */ +function isClaudeModel(model) { + return CLAUDE_MODEL_PATTERN.test(String(model ?? '').trim()); +} + +/** + * Tokens of a text for a Claude model: about three characters each. + * + * @param {unknown} text + * @returns {number} + */ +function estimateTokensForClaude(text) { + return Math.ceil(String(text ?? '').length / CLAUDE_CHARS_PER_TOKEN); +} + +/** + * truncateToTokenLimit() for a Claude model: the same character cut and word + * boundary as the other estimated models, at Claude's rate. + * + * @param {unknown} text + * @param {number} maxTokens - a positive finite number, checked by the caller + * @returns {string} + */ +function truncateForClaude(text, maxTokens) { + const source = typeof text === 'string' ? text : String(text ?? ''); + if (estimateTokensForClaude(source) <= maxTokens) { + return source; + } + + const maxChars = Math.floor(maxTokens * CLAUDE_CHARS_PER_TOKEN); + const truncatedText = source.substring(0, maxChars); + + // Break at a word boundary when that costs little of the budget. + const lastSpaceIndex = truncatedText.lastIndexOf(' '); + if (lastSpaceIndex > maxChars * 0.8) { + return truncatedText.substring(0, lastSpaceIndex); + } + + return truncatedText; +} + module.exports = { calculateTokens, calculateTotalPromptTokens, diff --git a/services/setupService.js b/services/setupService.js index f95e55f9..5327c21a 100644 --- a/services/setupService.js +++ b/services/setupService.js @@ -5,6 +5,12 @@ const { OpenAI } = require('openai'); const config = require('../config/config'); const AzureOpenAI = require('openai').AzureOpenAI; const { validateApiUrl, stripTrailingSlashes } = require('./serviceUtils'); +const { + ANTHROPIC_DEFAULT_MODEL, + ANTHROPIC_SUGGESTED_MODELS, + createAnthropicClient, + isConfiguredAnthropicHost, +} = require('./anthropicSupport'); const CUSTOM_PROVIDER_FALLBACK_API_KEY = 'no-auth-required'; @@ -92,6 +98,11 @@ class SetupService { AZURE_DEPLOYMENT_NAME: process.env.AZURE_DEPLOYMENT_NAME || '', CUSTOM_BASE_URL: process.env.CUSTOM_BASE_URL || '', CUSTOM_MODEL: process.env.CUSTOM_MODEL || '', + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || '', + // config.anthropic runs with the default model when none is set, so an + // operator who only injects the key has a complete configuration. + ANTHROPIC_MODEL: process.env.ANTHROPIC_MODEL || ANTHROPIC_DEFAULT_MODEL, + ANTHROPIC_API_URL: process.env.ANTHROPIC_API_URL || '', }; } @@ -529,6 +540,172 @@ class SetupService { } } + /** + * Proves an Anthropic key and model without spending tokens: the Models API + * answers for a model the key may use and refuses everything else, which + * is all a test before saving needs to know. A completion request would + * bill the operator for every click on "Test". + * + * @param {string} apiKey + * @param {string} model + * @param {string} [apiUrl] - proxy only; empty means api.anthropic.com + * @returns {Promise} + */ + async validateAnthropicConfig(apiKey, model, apiUrl = '') { + const normalizedApiKey = String(apiKey || '').trim(); + const normalizedModel = String(model || '').trim(); + const normalizedApiUrl = String(apiUrl || '').trim(); + + if (!normalizedApiKey || !normalizedModel) { + console.error( + 'Anthropic validation error: API key and model are required' + ); + return false; + } + + // A proxy URL is operator input like the custom provider's base URL, so it + // gets the same SSRF rules. The default endpoint needs no check. + if (normalizedApiUrl) { + const urlValidation = await validateApiUrl( + normalizedApiUrl, + this.getSetupUrlValidationOptions() + ); + if (!urlValidation.valid) { + console.error('Anthropic URL validation error:', urlValidation.error); + return false; + } + } + + try { + const client = createAnthropicClient({ + apiKey: normalizedApiKey, + apiUrl: normalizedApiUrl, + timeoutMs: this.getValidationTimeoutMs(), + }); + const modelInfo = await this.withValidationTimeout( + client.models.retrieve(normalizedModel), + 'Anthropic validation' + ); + return Boolean(modelInfo && modelInfo.id); + } catch (error) { + console.error('Anthropic validation error:', error.message); + return false; + } + } + + /** + * The Claude models a key can use, for the model pickers of setup and + * settings. Falls back to ANTHROPIC_SUGGESTED_MODELS whenever the live list + * cannot be had — no key yet, a refused key, a blocked proxy URL, a network + * error — because the picker is still useful with the three current models + * in it, and an empty list is not. + * + * @param {{apiKey?: string, apiUrl?: string}} options + * @returns {Promise<{models: string[], live: boolean, error: string|null}>} + */ + async discoverAnthropicModels(options = {}) { + const apiKey = String(options.apiKey || '').trim(); + const apiUrl = String(options.apiUrl || '').trim(); + const suggested = (error) => ({ + models: ANTHROPIC_SUGGESTED_MODELS.map((entry) => entry.id), + live: false, + error, + }); + + if (!apiKey) { + return suggested('No Anthropic API key is configured yet.'); + } + + if (apiUrl) { + const urlValidation = await validateApiUrl( + apiUrl, + this.getSetupUrlValidationOptions() + ); + if (!urlValidation.valid) { + return suggested(`Model URL validation failed: ${urlValidation.error}`); + } + } + + try { + const client = createAnthropicClient({ + apiKey, + apiUrl, + timeoutMs: this.getValidationTimeoutMs(), + }); + const collect = async () => { + const ids = []; + // The page object paginates by itself when iterated. + for await (const modelInfo of client.models.list()) { + ids.push(String(modelInfo?.id || '').trim()); + } + return ids; + }; + const models = this.dedupeModelIds( + await this.withValidationTimeout(collect(), 'Anthropic model discovery') + ); + if (models.length === 0) { + return suggested('The Models API returned no models for this key.'); + } + return { models, live: true, error: null }; + } catch (error) { + console.error('Anthropic model discovery error:', error.message); + return suggested(error.message); + } + } + + /** + * Key and URL for Claude OCR: the OCR fields when they are set, else the + * Anthropic AI provider's. The same order mistralOcrService follows at + * runtime, so a green OCR test means the saved configuration reads too. + * + * `anthropicApiKey` / `anthropicApiUrl` carry AI provider values that are + * not saved yet (the setup wizard collects both before anything is + * written); the environment covers the ones that are. The stored + * ANTHROPIC_API_KEY is only used when the URL is one already configured + * (see isConfiguredAnthropicHost). + * + * @param {object} options + * @returns {{apiKey: string, apiUrl: string}} + */ + resolveAnthropicOcrConnection(options = {}) { + const firstSet = (...values) => + values.map((value) => String(value || '').trim()).find(Boolean) || ''; + const apiUrl = firstSet( + options.apiUrl, + options.anthropicApiUrl, + process.env.ANTHROPIC_API_URL + ); + const storedKey = isConfiguredAnthropicHost(apiUrl) + ? process.env.ANTHROPIC_API_KEY + : ''; + return { + apiKey: firstSet(options.apiKey, options.anthropicApiKey, storedKey), + apiUrl, + }; + } + + /** + * The model Claude OCR runs with. MISTRAL_OCR_MODEL is shared by every OCR + * provider and defaults to a Mistral model, so it only counts when it names + * a Claude model; otherwise the AI provider's model (or the default) is + * used, exactly as mistralOcrService decides at runtime. + * + * @param {string} model - the OCR model field + * @param {string} [aiModel] - ANTHROPIC_MODEL, saved or about to be + * @returns {string} + */ + resolveAnthropicOcrModel(model, aiModel = '') { + const ocrModel = String(model || '').trim(); + if (/^claude-/i.test(ocrModel)) { + return ocrModel; + } + return ( + String(aiModel || '').trim() || + String(process.env.ANTHROPIC_MODEL || '').trim() || + ANTHROPIC_DEFAULT_MODEL + ); + } + extractOpenAiModelIds(payload) { const candidates = Array.isArray(payload?.data) ? payload.data : []; return candidates @@ -667,7 +844,7 @@ class SetupService { if ( !provider || - !['openai', 'ollama', 'custom', 'azure'].includes(provider) + !['openai', 'ollama', 'custom', 'azure', 'anthropic'].includes(provider) ) { throw new Error('A valid AI provider is required for URL detection'); } @@ -679,6 +856,15 @@ class SetupService { }; } + // Nothing to probe: empty means api.anthropic.com, anything else is a + // proxy the operator typed and the SDK appends the paths itself. + if (provider === 'anthropic') { + return { + resolvedApiUrl: apiUrl, + mode: 'anthropic', + }; + } + if (provider === 'openai') { const candidates = this.buildVersionedApiUrlCandidates( apiUrl, @@ -756,10 +942,16 @@ class SetupService { const apiUrl = String(options.apiUrl || '').trim(); const apiKey = String(options.apiKey || '').trim(); - if (!['mistral', 'ollama'].includes(provider)) { + if (!['mistral', 'ollama', 'anthropic'].includes(provider)) { throw new Error('A valid OCR provider is required for URL detection'); } + // Claude OCR has no endpoint of its own to find: an empty URL falls back + // to the AI provider's, which falls back to api.anthropic.com. + if (provider === 'anthropic') { + return { resolvedApiUrl: apiUrl, mode: 'anthropic' }; + } + const cacheKey = `${provider}|${apiUrl}|${apiKey ? 'with-key' : 'without-key'}`; if (this.setupOcrDetectionCache.has(cacheKey)) { return this.setupOcrDetectionCache.get(cacheKey); @@ -840,7 +1032,7 @@ class SetupService { if ( !provider || - !['openai', 'ollama', 'custom', 'azure'].includes(provider) + !['openai', 'ollama', 'custom', 'azure', 'anthropic'].includes(provider) ) { throw new Error('A valid AI provider is required for model discovery'); } @@ -849,6 +1041,11 @@ class SetupService { return []; } + if (provider === 'anthropic') { + const discovery = await this.discoverAnthropicModels({ apiKey, apiUrl }); + return discovery.models; + } + if (provider === 'openai') { const targetUrl = (apiUrl || 'https://api.openai.com/v1').replace( /\/+$/, @@ -921,10 +1118,17 @@ class SetupService { const apiUrl = String(options.apiUrl || '').trim(); const apiKey = String(options.apiKey || '').trim(); - if (!['mistral', 'ollama'].includes(provider)) { + if (!['mistral', 'ollama', 'anthropic'].includes(provider)) { throw new Error('A valid OCR provider is required for model discovery'); } + if (provider === 'anthropic') { + const discovery = await this.discoverAnthropicModels( + this.resolveAnthropicOcrConnection(options) + ); + return discovery.models; + } + if (provider === 'mistral') { if (!apiKey) { // Reached only when the field, the saved configuration and the @@ -1229,6 +1433,73 @@ class SetupService { return ''; } + /** + * Claude OCR, tested the way it will run: one real transcription of the + * validation image through mistralOcrService.runAnthropicOcr(), with the + * key, URL and model the saved configuration would resolve to. A Models API + * check alone would pass for a model that cannot read images. + * + * @param {object} options - see validateOcrConfig(); also anthropicApiKey, + * anthropicApiUrl and anthropicModel for AI provider values not saved yet + * @returns {Promise} + */ + async validateAnthropicOcrConfig(options = {}) { + const { apiKey, apiUrl } = this.resolveAnthropicOcrConnection(options); + const model = this.resolveAnthropicOcrModel( + options.model, + options.anthropicModel + ); + + if (!apiKey) { + console.error( + 'Claude OCR validation error: no API key — enter one for OCR or configure the Anthropic AI provider' + ); + return false; + } + + if (apiUrl) { + const urlValidation = await validateApiUrl( + apiUrl, + this.getSetupUrlValidationOptions() + ); + if (!urlValidation.valid) { + console.error('Claude OCR URL validation error:', urlValidation.error); + return false; + } + } + + try { + const token = this.getOcrValidationToken(); + const imageDataUrl = this.buildOcrValidationImageDataUrl(token); + // Required here rather than at the top: mistralOcrService opens the + // database when it loads, and nothing else in this service needs it. + const mistralOcrService = require('./mistralOcrService'); + const ocrOutput = await this.withValidationTimeout( + mistralOcrService.runAnthropicOcr({ + apiKey, + apiUrl, + model, + base64: imageDataUrl.replace(/^data:image\/png;base64,/, ''), + mimeType: 'image/png', + timeoutMs: this.getValidationTimeoutMs(), + }), + 'Claude OCR validation' + ); + + if (!this.isExpectedOcrTokenPresent(ocrOutput, token)) { + console.error( + 'Claude OCR validation error: OCR output did not match expected token' + ); + return false; + } + + return true; + } catch (error) { + console.error('Claude OCR validation error:', error.message); + return false; + } + } + async validateOcrConfig(options = {}) { const enabled = String(options.enabled || 'no') @@ -1246,11 +1517,15 @@ class SetupService { const apiKey = String(options.apiKey || '').trim(); const configuredApiUrl = String(options.apiUrl || '').trim(); - if (!['mistral', 'ollama'].includes(provider)) { + if (!['mistral', 'ollama', 'anthropic'].includes(provider)) { console.error('OCR validation error: unsupported provider', provider); return false; } + if (provider === 'anthropic') { + return this.validateAnthropicOcrConfig(options); + } + if (!model) { console.error('OCR validation error: missing OCR model'); return false; @@ -1455,13 +1730,55 @@ class SetupService { if (!azureValid) { throw new Error('Invalid Azure configuration'); } + } else if (aiProvider === 'anthropic') { + const anthropicValid = await this.validateAnthropicConfig( + config.ANTHROPIC_API_KEY, + config.ANTHROPIC_MODEL || ANTHROPIC_DEFAULT_MODEL, + config.ANTHROPIC_API_URL + ); + if (!anthropicValid) { + throw new Error('Invalid Anthropic configuration'); + } } return true; } - async saveConfig(configValues, options = {}) { + /** + * Adds the running JWT_SECRET and OIDC_* settings to a configuration that + * does not carry them. + * + * The secret is minted once at first start (server.js) and appears in no + * form, while saveConfig rewrites both stores whole. The settings page saved + * without it, so the restart after every save minted a new secret and every + * open session — the one that had just saved included — stopped verifying. + * Single sign-on is configured outside the forms as well, so a save dropped + * OIDC_* lines kept in data/.env or the runtime overrides in the same way. + * + * @param {object} configValues + * @returns {object} + */ + withSessionSecret(configValues) { + const carried = {}; + const secret = String(process.env.JWT_SECRET || '').trim(); + if (secret && !String(configValues?.JWT_SECRET || '').trim()) { + carried.JWT_SECRET = secret; + } + Object.keys(process.env) + .filter((key) => key.startsWith('OIDC_')) + .filter((key) => !(configValues && key in configValues)) + .forEach((key) => { + carried[key] = process.env[key]; + }); + return Object.keys(carried).length > 0 + ? { ...configValues, ...carried } + : configValues; + } + + async saveConfig(rawConfigValues, options = {}) { try { + const configValues = this.withSessionSecret(rawConfigValues); + // Validate the new configuration before saving unless explicitly skipped if (!options.skipValidation) { await this.validateConfig(configValues); @@ -1537,6 +1854,14 @@ class SetupService { ); } + // No URL: empty means api.anthropic.com. + if (aiProvider === 'anthropic') { + return ( + Boolean(String(config.ANTHROPIC_API_KEY || '').trim()) && + Boolean(String(config.ANTHROPIC_MODEL || '').trim()) + ); + } + return false; } diff --git a/tests/helpers/fake-anthropic.js b/tests/helpers/fake-anthropic.js new file mode 100644 index 00000000..769af07f --- /dev/null +++ b/tests/helpers/fake-anthropic.js @@ -0,0 +1,380 @@ +'use strict'; + +/** + * A real HTTP server that answers like the Claude Messages API. + * + * anthropicService talks to the Claude API through the Anthropic SDK, and the + * SDK is the part a stub could not be trusted to imitate: it parses the + * server-sent events off a socket, accumulates them into the final message, + * maps error statuses onto its error classes and aborts the socket when the + * caller stops caring. So the tests point ANTHROPIC_API_URL at this server + * on an ephemeral port, and what it answers is a script the test writes: + * + * text and thinking chunks (consecutive chunks of one kind form one content + * block, so a thinking block followed by a text block is two entries), a + * stop reason (end_turn, max_tokens, refusal) with stop details, the usage + * with cache counts, a delay before each chunk so a test can stop a request + * in the middle, or an error status with an Anthropic error body. + * + * `POST /v1/messages` answers JSON or, for `stream: true`, the SSE event + * sequence message_start, content_block_start/delta/stop per block, + * message_delta (stop reason, output tokens), message_stop. + * `GET /v1/models/:id` knows the models in `models` and answers 404 for any + * other. With `apiKey` set, a request with any other x-api-key gets 401. + * + * Every request is recorded with its body and headers, and every record knows + * whether the client hung up before the answer was finished. + * + * Usage: + * const fake = await createFakeAnthropic({ chunks: [{ text: 'hi' }] }); + * process.env.ANTHROPIC_API_URL = fake.url; + * ... await fake.close(); + */ + +const http = require('http'); + +/** The models GET /v1/models/:id knows when the test names none. */ +const DEFAULT_MODELS = [ + 'claude-haiku-5-5', + 'claude-sonnet-5-5', + 'claude-opus-5-5', +]; + +/** + * @typedef {object} FakeAnthropicChunk + * @property {string} [text] a piece of the answer + * @property {string} [thinking] a piece of a thinking block ('' for a block + * whose text is omitted, as the API sends it by default) + * @property {number} [delayMs] wait this long before sending it + */ + +/** + * @typedef {object} FakeAnthropicScript + * @property {FakeAnthropicChunk[]} [chunks] + * @property {'end_turn'|'max_tokens'|'refusal'} [stopReason] default end_turn + * @property {object|null} [stopDetails] e.g. { category: 'cyber' } + * @property {object} [usage] input_tokens, output_tokens, + * cache_read_input_tokens, cache_creation_input_tokens + * @property {number} [chunkDelayMs] default delay before every chunk + * @property {number} [responseDelayMs] wait this long before answering at + * all, headers included, so a client timeout fires + * @property {number} [status] answer this status instead, with `error` + * @property {object} [error] the error body for a non-200 status + */ + +const sleep = (ms) => + ms > 0 + ? new Promise((resolve) => setTimeout(resolve, ms)) + : Promise.resolve(); + +const ERROR_TYPES = { + 400: 'invalid_request_error', + 401: 'authentication_error', + 404: 'not_found_error', + 429: 'rate_limit_error', + 500: 'api_error', +}; + +function errorBody(status, message) { + return { + type: 'error', + error: { type: ERROR_TYPES[status] || 'api_error', message }, + }; +} + +function normalizeScript(script) { + const status = Number(script?.status) || 200; + return { + chunks: Array.isArray(script?.chunks) ? script.chunks : [], + stopReason: script?.stopReason || 'end_turn', + stopDetails: script?.stopDetails ?? null, + usage: { + input_tokens: 10, + output_tokens: 5, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + ...(script?.usage || {}), + }, + chunkDelayMs: Number(script?.chunkDelayMs) || 0, + responseDelayMs: Number(script?.responseDelayMs) || 0, + status, + error: script?.error || errorBody(status, 'fake failure'), + }; +} + +/** + * The script's chunks as content blocks: a change between text and thinking + * starts a new block. + * + * @returns {Array<{type: string, pieces: FakeAnthropicChunk[]}>} + */ +function groupBlocks(chunks) { + const blocks = []; + for (const chunk of chunks) { + const type = typeof chunk?.thinking === 'string' ? 'thinking' : 'text'; + const last = blocks[blocks.length - 1]; + if (last && last.type === type) { + last.pieces.push(chunk); + } else { + blocks.push({ type, pieces: [chunk] }); + } + } + return blocks; +} + +/** A content block as the non-streaming answer carries it. */ +function wholeBlock(block) { + if (block.type === 'thinking') { + return { + type: 'thinking', + thinking: block.pieces.map((piece) => piece.thinking).join(''), + signature: 'fake-signature', + }; + } + return { + type: 'text', + text: block.pieces.map((piece) => piece.text || '').join(''), + }; +} + +function readBody(request) { + return new Promise((resolve) => { + const pieces = []; + request.on('data', (piece) => pieces.push(piece)); + request.on('end', () => { + const raw = Buffer.concat(pieces).toString('utf8'); + try { + resolve(raw === '' ? {} : JSON.parse(raw)); + } catch { + resolve({ unparsable: raw }); + } + }); + }); +} + +/** + * Starts the fake and listens on an ephemeral port. + * + * @param {FakeAnthropicScript|FakeAnthropicScript[]} [script] - one script + * for every request, or a list answering one request each (the last one + * repeats) + * @param {{models?: string[], apiKey?: string}} [options] + * @returns {Promise} the handle; close() it in a finally + */ +async function createFakeAnthropic(script = {}, options = {}) { + let scripts = Array.isArray(script) ? script.slice() : [script]; + let served = 0; + const models = new Set(options.models || DEFAULT_MODELS); + let expectedKey = options.apiKey || null; + const requests = []; + const sockets = new Set(); + + const nextScript = () => { + const chosen = scripts[Math.min(served, scripts.length - 1)] || {}; + served += 1; + return normalizeScript(chosen); + }; + + const sendJson = (response, status, payload) => { + response.writeHead(status, { 'Content-Type': 'application/json' }); + response.end(JSON.stringify(payload)); + }; + + const server = http.createServer(async (request, response) => { + const path = String(request.url || '').split('?')[0]; + const body = await readBody(request); + + const record = { + method: request.method, + path, + body, + headers: request.headers, + /** The client hung up before the answer was finished. */ + aborted: false, + /** Resolves when the connection is gone, either way. */ + whenClosed: null, + }; + record.whenClosed = new Promise((resolve) => { + response.on('close', () => { + record.aborted = !response.writableFinished; + resolve(record); + }); + }); + requests.push(record); + + if (expectedKey && request.headers['x-api-key'] !== expectedKey) { + sendJson(response, 401, errorBody(401, 'invalid x-api-key')); + return; + } + + const modelMatch = /^\/v1\/models\/([^/]+)$/.exec(path); + if (request.method === 'GET' && modelMatch) { + const id = decodeURIComponent(modelMatch[1]); + if (!models.has(id)) { + sendJson(response, 404, errorBody(404, `model: ${id}`)); + return; + } + sendJson(response, 200, { + type: 'model', + id, + display_name: id, + created_at: '2026-01-01T00:00:00Z', + }); + return; + } + + if (request.method !== 'POST' || path !== '/v1/messages') { + sendJson(response, 404, errorBody(404, `no route ${path}`)); + return; + } + + const plan = nextScript(); + await sleep(plan.responseDelayMs); + if (response.destroyed) return; + if (plan.status !== 200) { + sendJson(response, plan.status, plan.error); + return; + } + + if (body?.stream === true) { + await writeStream(response, plan, body); + return; + } + + sendJson(response, 200, { + id: 'msg_fake', + type: 'message', + role: 'assistant', + model: body?.model || 'claude-haiku-5-5', + content: groupBlocks(plan.chunks).map(wholeBlock), + stop_reason: plan.stopReason, + stop_sequence: null, + stop_details: plan.stopDetails, + usage: plan.usage, + }); + }); + + server.on('connection', (socket) => { + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + }); + + /** Writes the script as the Messages API's server-sent events. */ + async function writeStream(response, plan, body) { + response.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); + + const send = (type, payload) => { + if (response.writableEnded || response.destroyed) return false; + response.write( + `event: ${type}\ndata: ${JSON.stringify({ type, ...payload })}\n\n` + ); + return true; + }; + + const started = send('message_start', { + message: { + id: 'msg_fake', + type: 'message', + role: 'assistant', + model: body?.model || 'claude-haiku-5-5', + content: [], + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: plan.usage.input_tokens, + output_tokens: 1, + cache_creation_input_tokens: plan.usage.cache_creation_input_tokens, + cache_read_input_tokens: plan.usage.cache_read_input_tokens, + }, + }, + }); + if (!started) return; + + const blocks = groupBlocks(plan.chunks); + for (let index = 0; index < blocks.length; index += 1) { + const block = blocks[index]; + const opening = + block.type === 'thinking' + ? { type: 'thinking', thinking: '', signature: '' } + : { type: 'text', text: '' }; + if (!send('content_block_start', { index, content_block: opening })) { + return; + } + for (const piece of block.pieces) { + const delay = Number.isFinite(Number(piece?.delayMs)) + ? Number(piece.delayMs) + : plan.chunkDelayMs; + await sleep(delay); + let delta; + if (block.type === 'thinking') { + if (piece.thinking === '') continue; + delta = { type: 'thinking_delta', thinking: piece.thinking }; + } else { + delta = { type: 'text_delta', text: piece.text || '' }; + } + if (!send('content_block_delta', { index, delta })) return; + } + if (block.type === 'thinking') { + const signature = { + type: 'signature_delta', + signature: 'fake-signature', + }; + if (!send('content_block_delta', { index, delta: signature })) return; + } + if (!send('content_block_stop', { index })) return; + } + + const delta = { + stop_reason: plan.stopReason, + stop_sequence: null, + stop_details: plan.stopDetails, + }; + if ( + !send('message_delta', { + delta, + usage: { output_tokens: plan.usage.output_tokens }, + }) + ) { + return; + } + if (!send('message_stop', {})) return; + if (!response.writableEnded && !response.destroyed) response.end(); + } + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address(); + + return { + url: `http://127.0.0.1:${port}`, + requests, + /** The request that came in last, or undefined before the first one. */ + last: () => requests[requests.length - 1], + /** The message requests only, oldest first. */ + messageRequests: () => + requests.filter((record) => record.path === '/v1/messages'), + /** Replaces the script; a list answers one request each. */ + setScript(next) { + scripts = Array.isArray(next) ? next.slice() : [next]; + served = 0; + }, + /** Requires this x-api-key from now on; null accepts any. */ + setApiKey(key) { + expectedKey = key || null; + }, + /** Forgets every recorded request. */ + reset() { + requests.length = 0; + served = 0; + }, + async close() { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(resolve)); + }, + }; +} + +module.exports = { createFakeAnthropic, DEFAULT_MODELS }; diff --git a/tests/test-anthropic-provider.js b/tests/test-anthropic-provider.js new file mode 100644 index 00000000..7b6b5bc6 --- /dev/null +++ b/tests/test-anthropic-provider.js @@ -0,0 +1,1254 @@ +'use strict'; + +/** + * The Claude API as an AI provider (services/anthropicService.js). + * + * The service talks to tests/helpers/fake-anthropic.js, a real HTTP server + * that answers like the Messages API, through the real SDK: the parts worth + * testing are the request the SDK puts on the wire, the server-sent events it + * reads back and the connection it drops on an abort. ANTHROPIC_API_URL points + * at the fake before config is loaded, so the client is the one initialize() + * builds from config. + * + * Cases: + * 1. the analysis request: top-level system, one user message with the + * document, max_tokens from ANTHROPIC_MAX_TOKENS, no sampling parameters + * 2. thinking and effort per model (Haiku 5.5, Sonnet 5.5, Opus 5.5) + * 3. structured output: the schema goes out when it is on and the model + * takes it, and not when it is off or the model does not + * 4. prompt caching: a cacheable system block when on, a string when off + * 5. the system prompt is byte-identical between two documents of a scan + * 6. an answer that starts with a thinking block is read by type + * 7. usage becomes metrics, with the cache counts and one debug line + * 8. an answer cut off at max_tokens is ai_response_truncated + * 9. a refusal is ai_refused and is not sent to the OCR queue + * 10. a schema the API refuses is dropped once, and the analysis goes on + * 11. USE_PROMPT_TAGS asks for the predefined-tags object, the schema follows + * 12. the schema for a configuration with custom fields + * 13. the prompt builder reproduces customService's system prompt byte for + * byte, in every branch it has + * 14. generateText: plain and streamed give the same answer, done once, + * thinking reported as thinking and kept out of the text + * 15. generateText: the usage the caller reads off the service + * 16. generateText: model, maxTokens and systemPrompt are honoured, + * temperature is not sent + * 17. generateText: a cut-off stream throws with what it managed + * 18. generateText: a refusal is ai_refused with what arrived + * 19. an abort mid-stream is an AbortError, and the server sees it + * 20. an abort before the request is an AbortError too + * 21. checkStatus: ok without spending tokens, clear errors otherwise + * 22. the Claude token estimate, every other model unchanged + * 23. the Duplicates review names the Anthropic model + * 24. an analysis that times out reports the AI timeout + * 25. without a key: the analysis fails softly, generateText throws + */ + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const OpenAI = require('openai'); + +const { createFakeAnthropic } = require('./helpers/fake-anthropic'); +const { createFakeOpenAI } = require('./helpers/fake-openai'); + +let passed = 0; +let failed = 0; + +async function test(name, fn) { + try { + await fn(); + console.log(`✅ ${name}`); + passed += 1; + } catch (error) { + console.error(`❌ ${name}`); + console.error(` ${error.stack || error.message}`); + failed += 1; + } +} + +/** + * Runs a body with the services' logging captured instead of printed: they + * log every token calculation, and half of these cases fail on purpose. + * + * @returns {Promise<{result: unknown, lines: Array<{level: string, text: string}>}>} + */ +async function captured(fn) { + const levels = ['log', 'info', 'warn', 'error', 'debug']; + const real = {}; + const lines = []; + for (const level of levels) { + real[level] = console[level]; + console[level] = (...args) => + lines.push({ + level, + text: args + .map((arg) => (typeof arg === 'string' ? arg : String(arg))) + .join(' '), + }); + } + try { + const result = await fn(); + return { result, lines }; + } finally { + for (const level of levels) console[level] = real[level]; + } +} + +/** The result of fn, with its logging swallowed. */ +async function quiet(fn) { + return (await captured(fn)).result; +} + +/** Collects every progress update a call reports. */ +function recorder() { + const updates = []; + return { + updates, + onProgress: (update) => updates.push(update), + last: () => updates[updates.length - 1], + }; +} + +/** The text of a request's system field, whichever form it took. */ +function systemText(body) { + if (typeof body.system === 'string') return body.system; + if (Array.isArray(body.system)) { + return body.system.map((block) => block.text).join(''); + } + return undefined; +} + +/** Walks a JSON schema and calls visit on every object schema in it. */ +function eachObjectSchema(schema, visit) { + if (!schema || typeof schema !== 'object') return; + if (schema.type === 'object') visit(schema); + for (const value of Object.values(schema)) { + if (value && typeof value === 'object') eachObjectSchema(value, visit); + } +} + +const ANSWER = { + title: 'Telekom invoice July', + correspondent: 'Telekom Deutschland GmbH', + tags: ['Invoice', 'Telephone'], + document_type: 'Invoice', + document_date: '2026-07-03', + language: 'de', + custom_fields: {}, +}; +const ANSWER_JSON = JSON.stringify(ANSWER); + +const CUSTOM_FIELDS = JSON.stringify({ + custom_fields: [ + { value: 'Invoice number', data_type: 'string' }, + { value: 'Due date', data_type: 'date' }, + { value: 'Paid', data_type: 'boolean' }, + ], +}); + +const DOCUMENT_IDS = [7, 8, 9]; + +async function main() { + const originalCwd = process.cwd(); + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'zr-anthropic-')); + process.chdir(tempRoot); + // The analysis caches a thumbnail first; with one in place it never asks + // Paperless-ngx for it. + fs.mkdirSync(path.join(tempRoot, 'data', 'thumb-cache'), { + recursive: true, + }); + for (const id of DOCUMENT_IDS) { + fs.writeFileSync( + path.join(tempRoot, 'data', 'thumb-cache', `${id}.png`), + 'png' + ); + } + + const fake = await createFakeAnthropic({}); + const fakeOpenAI = await createFakeOpenAI({ + chunks: [{ content: ANSWER_JSON }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + + process.env.NODE_ENV = 'test'; + process.env.LOG_LEVEL = 'error'; + process.env.PAPERLESS_API_URL = 'http://127.0.0.1:9'; + process.env.PAPERLESS_API_TOKEN = 'test-token'; + process.env.AI_PROVIDER = 'anthropic'; + process.env.ANTHROPIC_API_KEY = 'sk-ant-test'; + process.env.ANTHROPIC_API_URL = fake.url; + delete process.env.ANTHROPIC_MODEL; + delete process.env.ANTHROPIC_EFFORT; + delete process.env.ANTHROPIC_MAX_TOKENS; + delete process.env.DUPLICATES_AI_MODEL; + process.env.SYSTEM_PROMPT = 'Analyse the document and answer in JSON.'; + process.env.CUSTOM_FIELDS = JSON.stringify({ custom_fields: [] }); + process.env.USE_PROMPT_TAGS = 'no'; + + const config = require('../config/config'); + const anthropicService = require('../services/anthropicService'); + const customService = require('../services/customService'); + const { + calculateTokens, + truncateToTokenLimit, + shouldQueueForOcrOnAiError, + classifyOcrQueueReasonFromAiError, + buildTimeoutErrorMessage, + } = require('../services/serviceUtils'); + const { createAnthropicClient } = require('../services/anthropicSupport'); + const { + buildAnalysisPrompt, + analysisResponseSchema, + } = require('../services/analysisPrompt'); + + customService.client = new OpenAI({ + apiKey: 'test', + baseURL: fakeOpenAI.baseURL, + maxRetries: 0, + }); + + const defaults = { + anthropic: { ...config.anthropic }, + useExistingData: config.useExistingData, + restrictToExistingTags: config.restrictToExistingTags, + restrictToExistingCorrespondents: config.restrictToExistingCorrespondents, + systemPrompt: process.env.SYSTEM_PROMPT, + customFields: process.env.CUSTOM_FIELDS, + }; + + /** Back to the configuration every case starts from. */ + function restore() { + Object.assign(config.anthropic, defaults.anthropic); + config.useExistingData = defaults.useExistingData; + config.restrictToExistingTags = defaults.restrictToExistingTags; + config.restrictToExistingCorrespondents = + defaults.restrictToExistingCorrespondents; + process.env.SYSTEM_PROMPT = defaults.systemPrompt; + process.env.CUSTOM_FIELDS = defaults.customFields; + process.env.USE_PROMPT_TAGS = 'no'; + anthropicService.rejectedSchemas.clear(); + fake.setApiKey(null); + } + + /** Points the fake at a fresh script and forgets the old requests. */ + function useScript(script) { + fake.reset(); + fake.setScript(script); + } + + const analyze = (content = 'Invoice 4711 from Telekom', id = 7, ...rest) => + quiet(() => + anthropicService.analyzeDocument( + content, + ['Invoice', 'Telephone'], + ['Telekom'], + ['Invoice'], + id, + ...rest + ) + ); + + try { + // ------------------------------------------------------------ the request + + await test('The analysis sends top-level system, one user message, max_tokens and no sampling parameters', async () => { + restore(); + useScript({ chunks: [{ text: ANSWER_JSON }] }); + + const result = await analyze('Invoice 4711 from Telekom'); + + assert.strictEqual(result.error, undefined, result.error); + const record = fake.last(); + const body = record.body; + assert.strictEqual(record.path, '/v1/messages'); + assert.strictEqual(record.headers['x-api-key'], 'sk-ant-test'); + assert.ok(record.headers['anthropic-version'], 'the SDK sends a version'); + assert.strictEqual(body.model, 'claude-haiku-5-5', 'the default model'); + assert.strictEqual(body.max_tokens, 16000, 'ANTHROPIC_MAX_TOKENS'); + assert.deepStrictEqual(body.messages, [ + { role: 'user', content: 'Invoice 4711 from Telekom' }, + ]); + assert.match(systemText(body), /^Analyse the document/); + assert.ok( + !body.messages.some((message) => message.role === 'system'), + 'the system prompt is no message' + ); + for (const key of ['temperature', 'top_p', 'top_k']) { + assert.strictEqual(key in body, false, `no ${key}`); + } + assert.strictEqual( + 'thinking' in body, + false, + 'no effort configured: the model decides' + ); + + config.anthropic.maxTokens = 32000; + useScript({ chunks: [{ text: ANSWER_JSON }] }); + await analyze(); + assert.strictEqual(fake.last().body.max_tokens, 32000); + }); + + await test('Thinking and effort follow the model and ANTHROPIC_EFFORT', async () => { + restore(); + useScript({ chunks: [{ text: ANSWER_JSON }] }); + + config.anthropic.effort = 'high'; + await analyze(); + assert.deepStrictEqual( + fake.last().body.output_config.effort, + 'high', + 'the analysis sends the configured effort' + ); + assert.ok( + fake.last().body.output_config.format, + 'merged with the schema, not replaced by it' + ); + + // reasoning: false asks each model for as little thinking as it allows. + const expected = { + 'claude-haiku-5-5': { + thinking: { type: 'disabled' }, + output_config: { effort: 'low' }, + }, + 'claude-sonnet-5-5': { + thinking: { type: 'between_tools' }, + output_config: { effort: 'low' }, + }, + // Always thinks; a low effort is the only lever. + 'claude-opus-5-5': { output_config: { effort: 'low' } }, + }; + for (const [model, fields] of Object.entries(expected)) { + useScript({ chunks: [{ text: 'ok' }] }); + await quiet(() => + anthropicService.generateText('judge this', { + model, + reasoning: false, + }) + ); + const body = fake.last().body; + assert.strictEqual(body.model, model); + assert.deepStrictEqual(body.thinking, fields.thinking, model); + assert.deepStrictEqual(body.output_config, fields.output_config, model); + } + + // reasoning: null sends only the configured effort. + useScript({ chunks: [{ text: 'ok' }] }); + await quiet(() => + anthropicService.generateText('judge this', { + model: 'claude-sonnet-5-5', + }) + ); + assert.strictEqual('thinking' in fake.last().body, false); + assert.deepStrictEqual(fake.last().body.output_config, { + effort: 'high', + }); + + // reasoning: true asks for adaptive only where thinking is off by default. + useScript({ chunks: [{ text: 'ok' }] }); + await quiet(() => + anthropicService.generateText('judge this', { + model: 'claude-haiku-5-5', + reasoning: true, + }) + ); + assert.strictEqual('thinking' in fake.last().body, false); + }); + + await test('Structured output: the schema goes out when on and supported, not otherwise', async () => { + restore(); + useScript({ chunks: [{ text: ANSWER_JSON }] }); + + await analyze(); + const format = fake.last().body.output_config?.format; + assert.ok(format, 'the schema went out'); + assert.strictEqual(format.type, 'json_schema'); + assert.deepStrictEqual( + format.schema, + analysisResponseSchema({ responseShape: 'full', customFields: [] }) + ); + + config.anthropic.structuredOutput = 'no'; + useScript({ chunks: [{ text: ANSWER_JSON }] }); + const off = await analyze(); + assert.strictEqual(off.error, undefined, off.error); + assert.strictEqual( + 'output_config' in fake.last().body, + false, + 'switched off: no output_config at all' + ); + + // A model that does not take output_config.format would answer 400. + config.anthropic.structuredOutput = 'yes'; + config.anthropic.model = 'claude-sonnet-4-6'; + useScript({ chunks: [{ text: '```json\n' + ANSWER_JSON + '\n```' }] }); + const older = await analyze(); + assert.strictEqual(older.error, undefined, older.error); + assert.strictEqual(fake.last().body.model, 'claude-sonnet-4-6'); + assert.strictEqual( + fake.last().body.output_config?.format, + undefined, + 'no schema for a model without structured output' + ); + assert.deepStrictEqual( + older.document, + ANSWER, + 'the fenced answer is still parsed the tolerant way' + ); + }); + + await test('Prompt caching: a cacheable system block when on, a plain string when off', async () => { + restore(); + useScript({ chunks: [{ text: ANSWER_JSON }] }); + + await analyze(); + const system = fake.last().body.system; + assert.ok(Array.isArray(system), 'one block'); + assert.strictEqual(system.length, 1); + assert.strictEqual(system[0].type, 'text'); + assert.deepStrictEqual(system[0].cache_control, { type: 'ephemeral' }); + + useScript({ chunks: [{ text: 'ok' }] }); + await quiet(() => + anthropicService.generateText('hello', { systemPrompt: 'Be brief.' }) + ); + assert.deepStrictEqual(fake.last().body.system, [ + { + type: 'text', + text: 'Be brief.', + cache_control: { type: 'ephemeral' }, + }, + ]); + + useScript({ chunks: [{ text: 'ok' }] }); + await quiet(() => anthropicService.generateText('hello')); + assert.strictEqual( + 'system' in fake.last().body, + false, + 'no system prompt, no system field' + ); + + config.anthropic.promptCaching = 'no'; + useScript({ chunks: [{ text: ANSWER_JSON }] }); + await analyze(); + assert.strictEqual(typeof fake.last().body.system, 'string'); + + useScript({ chunks: [{ text: 'ok' }] }); + await quiet(() => + anthropicService.generateText('hello', { systemPrompt: 'Be brief.' }) + ); + assert.strictEqual(fake.last().body.system, 'Be brief.'); + }); + + await test('The system prompt is byte-identical between two documents of one scan', async () => { + restore(); + process.env.CUSTOM_FIELDS = CUSTOM_FIELDS; + config.useExistingData = 'yes'; + config.restrictToExistingTags = 'no'; + config.restrictToExistingCorrespondents = 'no'; + useScript({ chunks: [{ text: ANSWER_JSON }] }); + + await analyze('First document: an invoice from Telekom', 7); + await analyze('Second document: a contract with the landlord', 8); + + const [first, second] = fake.messageRequests(); + assert.ok(first && second, 'two requests'); + assert.strictEqual( + JSON.stringify(first.body.system), + JSON.stringify(second.body.system), + 'same bytes, so the second request reads the cache' + ); + assert.notStrictEqual( + first.body.messages[0].content, + second.body.messages[0].content + ); + const text = systemText(first.body); + assert.ok(!text.includes('Telekom invoice'), 'no document text in it'); + assert.ok( + !/\b[78]\b/.test(text.replace(/\d{4}-\d{2}-\d{2}/g, '')), + 'no document id in it' + ); + assert.ok(text.includes('Pre-existing tags: Invoice, Telephone')); + }); + + // -------------------------------------------------------------- the answer + + await test('An answer that starts with a thinking block is read by type', async () => { + restore(); + useScript({ + chunks: [{ thinking: '' }, { text: ANSWER_JSON }], + }); + + const result = await analyze(); + + assert.strictEqual(result.error, undefined, result.error); + assert.deepStrictEqual(result.document, ANSWER); + assert.strictEqual(result.truncated, false); + }); + + await test('Usage becomes metrics, cache counts included, with one debug line per request', async () => { + restore(); + useScript({ + chunks: [{ text: ANSWER_JSON }], + usage: { + input_tokens: 120, + output_tokens: 80, + cache_read_input_tokens: 900, + cache_creation_input_tokens: 0, + }, + }); + + const { result, lines } = await captured(() => + anthropicService.analyzeDocument('Invoice 4711', [], [], [], 7) + ); + + assert.deepStrictEqual(result.metrics, { + promptTokens: 1020, + completionTokens: 80, + totalTokens: 1100, + reasoningTokens: null, + cacheReadTokens: 900, + cacheWriteTokens: 0, + }); + const cacheLines = lines.filter((line) => + line.text.includes('Anthropic cache:') + ); + assert.deepStrictEqual( + cacheLines.map((line) => [line.level, line.text]), + [['debug', '[DEBUG] Anthropic cache: read 900, write 0']] + ); + }); + + await test('An answer cut off at max_tokens is ai_response_truncated and names ANTHROPIC_MAX_TOKENS', async () => { + restore(); + useScript({ + chunks: [{ text: '{"title":"Telekom inv' }], + stopReason: 'max_tokens', + usage: { output_tokens: 16000 }, + }); + + const result = await analyze(); + + assert.strictEqual(result.errorCode, 'ai_response_truncated'); + assert.match(result.error, /after 16000 tokens/); + assert.match(result.error, /ANTHROPIC_MAX_TOKENS/); + assert.strictEqual(result.metrics, null); + assert.deepStrictEqual(result.document, { + tags: [], + correspondent: null, + }); + assert.strictEqual( + shouldQueueForOcrOnAiError(result.error), + false, + 'OCR text would be cut off the same way' + ); + }); + + await test('A refusal is ai_refused and does not go to the OCR queue', async () => { + restore(); + useScript({ + chunks: [], + stopReason: 'refusal', + stopDetails: { category: 'cyber', explanation: 'declined' }, + }); + + const result = await analyze(); + + assert.strictEqual(result.errorCode, 'ai_refused'); + assert.match(result.error, /declined to answer/); + assert.match(result.error, /cyber/); + assert.deepStrictEqual(result.document, { + tags: [], + correspondent: null, + }); + assert.strictEqual( + shouldQueueForOcrOnAiError(result.error), + false, + 'OCR cannot fix a refusal' + ); + assert.strictEqual( + classifyOcrQueueReasonFromAiError(result.error), + 'ai_failed_unknown' + ); + + // The classifier is live: an unreadable answer from the same provider + // does go to the OCR queue. + useScript({ chunks: [{ text: 'no JSON in here' }] }); + const unreadable = await analyze(); + assert.strictEqual(unreadable.error, 'Invalid JSON response from API'); + assert.strictEqual(shouldQueueForOcrOnAiError(unreadable.error), true); + }); + + await test('A schema the API refuses is dropped once, and the analysis goes on without it', async () => { + restore(); + useScript([ + { status: 400, error: undefined }, + { chunks: [{ text: ANSWER_JSON }] }, + ]); + + const { result, lines } = await captured(() => + anthropicService.analyzeDocument('Invoice', [], [], [], 7) + ); + + assert.strictEqual(result.error, undefined, result.error); + assert.deepStrictEqual(result.document, ANSWER); + const [refused, retried] = fake.messageRequests(); + assert.ok(refused.body.output_config.format, 'first with the schema'); + assert.strictEqual( + retried.body.output_config, + undefined, + 'then without it (no effort configured, so no output_config at all)' + ); + assert.strictEqual( + lines.filter((line) => line.text.includes('refused the analysis')) + .length, + 1 + ); + + useScript({ chunks: [{ text: ANSWER_JSON }] }); + await analyze(); + assert.strictEqual(fake.messageRequests().length, 1); + assert.strictEqual( + fake.last().body.output_config, + undefined, + 'the next document does not try the refused schema again' + ); + + // The retry drops the schema and keeps the effort. + anthropicService.rejectedSchemas.clear(); + config.anthropic.effort = 'medium'; + useScript([{ status: 400 }, { chunks: [{ text: ANSWER_JSON }] }]); + await analyze(); + assert.deepStrictEqual(fake.messageRequests()[1].body.output_config, { + effort: 'medium', + }); + config.anthropic.effort = defaults.anthropic.effort; + + // A 400 that the retry does not fix surfaces as it is. + anthropicService.rejectedSchemas.clear(); + useScript({ status: 400 }); + const broken = await analyze(); + assert.match(broken.error, /400/); + assert.strictEqual(anthropicService.rejectedSchemas.size, 0); + }); + + await test('USE_PROMPT_TAGS asks for the predefined-tags object, and the schema follows', async () => { + restore(); + process.env.USE_PROMPT_TAGS = 'yes'; + process.env.CUSTOM_FIELDS = CUSTOM_FIELDS; + const answer = { + title: 'Telekom invoice', + correspondent: 'Telekom', + tags: ['Invoice'], + document_date: '2026-07-03', + language: 'de', + }; + useScript({ chunks: [{ text: JSON.stringify(answer) }] }); + + const result = await analyze(); + + assert.deepStrictEqual(result.document, answer); + const body = fake.last().body; + assert.ok( + systemText(body).startsWith( + '\n Take these tags and try to match one or more' + ) + ); + assert.deepStrictEqual( + Object.keys(body.output_config.format.schema.properties), + ['title', 'correspondent', 'tags', 'document_date', 'language'], + 'no document type and no custom fields in that prompt' + ); + }); + + await test('The schema for a configuration with custom fields', async () => { + const prompt = buildAnalysisPrompt({ + systemPrompt: 'Analyse.', + mustHavePrompt: config.mustHavePrompt, + specialPromptPreDefinedTags: config.specialPromptPreDefinedTags, + customFieldsJson: CUSTOM_FIELDS, + useExistingData: 'no', + restrictToExistingTags: 'no', + restrictToExistingCorrespondents: 'no', + usePromptTags: 'no', + }); + assert.strictEqual(prompt.customFieldsError, null); + assert.ok( + prompt.systemPrompt.includes('"field_name": "Due date"'), + 'the template names the field' + ); + + const field = (name) => ({ + type: 'object', + properties: { + field_name: { type: 'string', const: name }, + value: { type: ['string', 'null'] }, + }, + required: ['field_name', 'value'], + additionalProperties: false, + }); + assert.deepStrictEqual(analysisResponseSchema(prompt), { + type: 'object', + properties: { + title: { type: 'string' }, + correspondent: { type: ['string', 'null'] }, + tags: { type: 'array', items: { type: 'string' } }, + document_type: { type: ['string', 'null'] }, + document_date: { type: ['string', 'null'] }, + language: { type: 'string' }, + custom_fields: { + type: 'object', + properties: { + 0: field('Invoice number'), + 1: field('Due date'), + 2: field('Paid'), + }, + required: ['0', '1', '2'], + additionalProperties: false, + }, + }, + required: [ + 'title', + 'correspondent', + 'tags', + 'document_type', + 'document_date', + 'language', + 'custom_fields', + ], + additionalProperties: false, + }); + + // What the API demands of every object in it. + eachObjectSchema(analysisResponseSchema(prompt), (object) => { + assert.strictEqual(object.additionalProperties, false); + assert.deepStrictEqual( + object.required || [], + Object.keys(object.properties) + ); + }); + const serialized = JSON.stringify(analysisResponseSchema(prompt)); + for (const forbidden of [ + 'minimum', + 'maximum', + 'minLength', + 'maxLength', + ]) { + assert.ok(!serialized.includes(forbidden), `no ${forbidden}`); + } + + // An unreadable CUSTOM_FIELDS is reported, and the schema has no fields. + const broken = buildAnalysisPrompt({ + systemPrompt: 'Analyse.', + mustHavePrompt: config.mustHavePrompt, + customFieldsJson: '{"oops":', + }); + assert.ok(broken.customFieldsError instanceof Error); + assert.deepStrictEqual( + analysisResponseSchema(broken).properties.custom_fields, + { type: 'object', properties: {}, additionalProperties: false } + ); + const noList = buildAnalysisPrompt({ + systemPrompt: 'Analyse.', + mustHavePrompt: config.mustHavePrompt, + customFieldsJson: '{}', + }); + assert.ok(noList.customFieldsError instanceof Error); + assert.deepStrictEqual(noList.customFields, []); + + // The configured fields reach the request. + restore(); + process.env.CUSTOM_FIELDS = CUSTOM_FIELDS; + useScript({ chunks: [{ text: ANSWER_JSON }] }); + await analyze(); + assert.deepStrictEqual( + Object.keys( + fake.last().body.output_config.format.schema.properties.custom_fields + .properties + ), + ['0', '1', '2'] + ); + }); + + await test("The prompt builder reproduces customService's system prompt byte for byte", async () => { + const scenarios = [ + { label: 'plain' }, + { label: 'custom fields', customFields: CUSTOM_FIELDS }, + { + label: 'USE_EXISTING_DATA without restrictions', + useExistingData: 'yes', + customFields: CUSTOM_FIELDS, + }, + { + label: 'USE_EXISTING_DATA with a restriction (no pre-existing block)', + useExistingData: 'yes', + restrictToExistingTags: 'yes', + }, + { + label: 'restriction placeholders', + systemPrompt: + 'Tags: %RESTRICTED_TAGS%; senders: %RESTRICTED_CORRESPONDENTS%; types: %RESTRICTED_DOCUMENT_TYPES%', + restrictToExistingTags: 'yes', + }, + { + label: 'external API data', + options: { externalApiData: { project: 'Alpha', budget: 1200 } }, + }, + { label: 'USE_PROMPT_TAGS', usePromptTags: 'yes' }, + { + label: 'webhook prompt', + customPrompt: 'Only find the sender.', + customFields: CUSTOM_FIELDS, + useExistingData: 'yes', + }, + { label: 'SYSTEM_PROMPT not set', systemPrompt: null }, + ]; + + for (const scenario of scenarios) { + restore(); + config.useExistingData = scenario.useExistingData || 'no'; + config.restrictToExistingTags = scenario.restrictToExistingTags || 'no'; + config.restrictToExistingCorrespondents = 'no'; + if (scenario.systemPrompt === null) { + delete process.env.SYSTEM_PROMPT; + } else if (scenario.systemPrompt) { + process.env.SYSTEM_PROMPT = scenario.systemPrompt; + } + process.env.CUSTOM_FIELDS = + scenario.customFields || JSON.stringify({ custom_fields: [] }); + process.env.USE_PROMPT_TAGS = scenario.usePromptTags || 'no'; + const args = [ + 'Invoice 4711 from Telekom', + [{ id: 1, name: 'Invoice' }, 'Telephone'], + ['Telekom', 'Landlord'], + [{ id: 4, name: 'Invoice' }], + 9, + scenario.customPrompt || null, + scenario.options || {}, + ]; + + fakeOpenAI.reset(); + useScript({ chunks: [{ text: ANSWER_JSON }] }); + const reference = await quiet(() => + customService.analyzeDocument(...args) + ); + const ours = await quiet(() => + anthropicService.analyzeDocument(...args) + ); + assert.strictEqual( + reference.error, + undefined, + `${scenario.label}: ${reference.error}` + ); + assert.strictEqual( + ours.error, + undefined, + `${scenario.label}: ${ours.error}` + ); + + const expected = fakeOpenAI.last().body.messages[0]; + assert.strictEqual(expected.role, 'system'); + assert.strictEqual( + systemText(fake.last().body), + expected.content, + `${scenario.label}: the same system prompt` + ); + } + }); + + // ------------------------------------------------------------ generateText + + await test('generateText: plain and streamed give the same answer, done once, thinking kept out', async () => { + restore(); + useScript({ + chunks: [ + { thinking: 'weighing the two names', delayMs: 130 }, + { text: 'Rechnung', delayMs: 130 }, + { text: ' und', delayMs: 130 }, + { text: ' Rechnungen', delayMs: 130 }, + ], + usage: { input_tokens: 40, output_tokens: 33 }, + }); + + const plain = await quiet(() => anthropicService.generateText('hello')); + const seen = recorder(); + const streamed = await quiet(() => + anthropicService.generateText('hello', { onProgress: seen.onProgress }) + ); + + assert.strictEqual(plain, 'Rechnung und Rechnungen'); + assert.strictEqual(streamed, plain); + assert.ok( + seen.updates.length >= 3, + `expected several reports, got ${seen.updates.length}` + ); + assert.strictEqual( + seen.updates.filter((update) => update.done).length, + 1, + 'done fires exactly once' + ); + assert.strictEqual(seen.last().done, true, 'and it is the last one'); + assert.strictEqual(seen.last().text, plain); + assert.strictEqual( + seen.updates[0].thinking, + true, + 'the first report was the thinking' + ); + assert.strictEqual(seen.updates[0].text, ''); + assert.strictEqual(seen.last().thinking, false); + for (const update of seen.updates) { + assert.ok(!update.text.includes('weighing'), 'thinking never in text'); + } + for (let index = 1; index < seen.updates.length; index += 1) { + assert.ok( + seen.updates[index].text.startsWith(seen.updates[index - 1].text), + `report ${index} grew out of the one before it` + ); + } + assert.strictEqual( + seen.last().completionTokens, + 33, + 'the final message decides the count' + ); + }); + + await test('generateText: the usage the caller reads off the service', async () => { + restore(); + useScript({ + chunks: [{ text: 'twelve chars' }], + usage: { + input_tokens: 30, + output_tokens: 7, + cache_read_input_tokens: 600, + cache_creation_input_tokens: 50, + }, + }); + + const { lines } = await captured(() => + anthropicService.generateText('hello', { systemPrompt: 'Be brief.' }) + ); + + assert.deepStrictEqual(anthropicService.lastGenerateTextUsage, { + promptTokens: 680, + completionTokens: 7, + totalTokens: 687, + reasoningTokens: null, + finishReason: 'end_turn', + answerChars: 12, + reasoningChars: 0, + cacheReadTokens: 600, + cacheWriteTokens: 50, + }); + assert.ok( + lines.some( + (line) => + line.level === 'debug' && + line.text === '[DEBUG] Anthropic cache: read 600, write 50' + ) + ); + }); + + await test('generateText: model, maxTokens and systemPrompt are honoured, temperature is not sent', async () => { + restore(); + useScript({ chunks: [{ text: 'ok' }] }); + + await quiet(() => + anthropicService.generateText('judge these', { + model: 'claude-opus-5-5', + maxTokens: 300, + temperature: 0, + systemPrompt: 'You are a judge.', + }) + ); + + const body = fake.last().body; + assert.strictEqual(body.model, 'claude-opus-5-5'); + assert.strictEqual(body.max_tokens, 300); + assert.strictEqual('temperature' in body, false); + assert.strictEqual(systemText(body), 'You are a judge.'); + assert.deepStrictEqual(body.messages, [ + { role: 'user', content: 'judge these' }, + ]); + + useScript({ chunks: [{ text: 'ok' }] }); + await quiet(() => + anthropicService.generateText('judge these', { maxTokens: null }) + ); + assert.strictEqual( + fake.last().body.max_tokens, + 16000, + 'without a cap of its own the call gets ANTHROPIC_MAX_TOKENS' + ); + }); + + await test('generateText: a cut-off stream throws with what it managed', async () => { + restore(); + useScript({ + chunks: [ + { text: '[{"id":"tags:1-2","verdict":"same"},' }, + { text: '{"id":"tags:3-4","verdi' }, + ], + stopReason: 'max_tokens', + usage: { input_tokens: 3797, output_tokens: 3200 }, + }); + const seen = recorder(); + + await quiet(() => + assert.rejects( + anthropicService.generateText('hello', { + maxTokens: 3200, + onProgress: seen.onProgress, + }), + (error) => { + assert.strictEqual(error.code, 'ai_response_truncated'); + assert.match(error.message, /after 3200 tokens/); + assert.strictEqual( + error.partialText, + '[{"id":"tags:1-2","verdict":"same"},{"id":"tags:3-4","verdi' + ); + return true; + } + ) + ); + assert.strictEqual(seen.last().done, true, 'done fired before the throw'); + assert.strictEqual( + seen.updates.filter((update) => update.done).length, + 1 + ); + assert.strictEqual( + anthropicService.lastGenerateTextUsage.completionTokens, + 3200, + 'and what it spent is still readable' + ); + + // Without a handler the same error, with the same prefix. + useScript({ + chunks: [{ text: 'half an ans' }], + stopReason: 'max_tokens', + usage: { output_tokens: 64 }, + }); + await quiet(() => + assert.rejects(anthropicService.generateText('hello'), (error) => { + assert.strictEqual(error.code, 'ai_response_truncated'); + assert.strictEqual(error.partialText, 'half an ans'); + return true; + }) + ); + }); + + await test('generateText: a refusal is ai_refused with what arrived', async () => { + restore(); + useScript({ + chunks: [{ text: 'I will not' }], + stopReason: 'refusal', + stopDetails: { category: 'bio' }, + }); + + await quiet(() => + assert.rejects(anthropicService.generateText('hello'), (error) => { + assert.strictEqual(error.code, 'ai_refused'); + assert.match(error.message, /bio/); + assert.strictEqual(error.partialText, 'I will not'); + return true; + }) + ); + }); + + await test('An abort mid-stream is an AbortError with what arrived, and the server sees it', async () => { + restore(); + useScript({ + chunks: [ + { text: 'first piece', delayMs: 40 }, + { text: ' second piece', delayMs: 300 }, + { text: ' third piece', delayMs: 300 }, + ], + }); + const controller = new AbortController(); + const seen = recorder(); + + await quiet(() => + assert.rejects( + anthropicService.generateText('hello', { + signal: controller.signal, + onProgress: (update) => { + seen.onProgress(update); + if (!update.done) controller.abort(); + }, + }), + (error) => { + assert.strictEqual(error.name, 'AbortError'); + assert.strictEqual(error.code, 'ai_request_aborted'); + assert.strictEqual(error.partialText, 'first piece'); + return true; + } + ) + ); + + assert.strictEqual(seen.last().done, true, 'done fired before the throw'); + assert.strictEqual( + seen.updates.filter((update) => update.done).length, + 1 + ); + const request = await fake.last().whenClosed; + assert.strictEqual( + request.aborted, + true, + 'the fake saw the connection go before it had finished writing' + ); + }); + + await test('An abort before the request is an AbortError too', async () => { + restore(); + useScript({ chunks: [{ text: 'never sent' }] }); + const controller = new AbortController(); + controller.abort(); + + await quiet(() => + assert.rejects( + anthropicService.generateText('hello', { signal: controller.signal }), + (error) => { + assert.strictEqual(error.name, 'AbortError'); + assert.strictEqual(error.code, 'ai_request_aborted'); + assert.strictEqual(error.partialText, ''); + return true; + } + ) + ); + }); + + // ------------------------------------------------------------ checkStatus + + await test('checkStatus: ok without spending tokens, clear errors otherwise', async () => { + restore(); + fake.reset(); + + const ok = await quiet(() => anthropicService.checkStatus()); + assert.deepStrictEqual(ok, { status: 'ok', model: 'claude-haiku-5-5' }); + assert.deepStrictEqual( + fake.requests.map((record) => `${record.method} ${record.path}`), + ['GET /v1/models/claude-haiku-5-5'], + 'one models lookup, no message' + ); + + config.anthropic.model = 'claude-nonexistent-1'; + const missing = await quiet(() => anthropicService.checkStatus()); + assert.strictEqual(missing.status, 'error'); + assert.match(missing.error, /claude-nonexistent-1/); + assert.match(missing.error, /not found/); + + config.anthropic.model = defaults.anthropic.model; + fake.setApiKey('sk-ant-other'); + const rejected = await quiet(() => anthropicService.checkStatus()); + assert.strictEqual(rejected.status, 'error'); + assert.match(rejected.error, /key was rejected/); + fake.setApiKey(null); + }); + + // ------------------------------------------------------------- the rest + + await test('The Claude token estimate is three characters a token; other models keep theirs', async () => { + const text = 'x'.repeat(300); + assert.strictEqual( + await quiet(() => calculateTokens(text, 'claude-haiku-5-5')), + 100 + ); + assert.strictEqual( + await quiet(() => calculateTokens(text, 'Claude-Opus-5-5')), + 100 + ); + assert.strictEqual( + await quiet(() => calculateTokens(text, 'llama3.1:8b')), + 75, + 'every other estimated model stays at four characters a token' + ); + + const words = 'word '.repeat(200); + const forClaude = await quiet(() => + truncateToTokenLimit(words, 50, 'claude-haiku-5-5') + ); + assert.ok(forClaude.length <= 150, `cut to ${forClaude.length} chars`); + assert.ok(forClaude.length >= 120, 'at a word boundary near the limit'); + assert.ok(!forClaude.endsWith(' ')); + const forLlama = await quiet(() => + truncateToTokenLimit(words, 50, 'llama3.1:8b') + ); + assert.ok(forLlama.length > 150 && forLlama.length <= 200); + assert.strictEqual( + await quiet(() => + truncateToTokenLimit('short', 50, 'claude-haiku-5-5') + ), + 'short' + ); + }); + + await test('The Duplicates review names the Anthropic model', async () => { + restore(); + const entityMatchAiService = require('../services/entityMatchAiService'); + assert.strictEqual(config.aiProvider, 'anthropic'); + assert.strictEqual(entityMatchAiService.modelName(), 'claude-haiku-5-5'); + }); + + await test('An analysis that times out reports the AI timeout', async () => { + restore(); + useScript({ responseDelayMs: 600, chunks: [{ text: ANSWER_JSON }] }); + const client = anthropicService.client; + anthropicService.client = createAnthropicClient({ + apiKey: 'sk-ant-test', + apiUrl: fake.url, + timeoutMs: 100, + }); + // One attempt: the SDK retries a timeout twice by default. + anthropicService.client.maxRetries = 0; + try { + const result = await analyze(); + assert.strictEqual(result.error, buildTimeoutErrorMessage('AI')); + assert.deepStrictEqual(result.document, { + tags: [], + correspondent: null, + }); + } finally { + anthropicService.client = client; + } + }); + + await test('Without a key the analysis fails softly and generateText throws', async () => { + restore(); + const client = anthropicService.client; + anthropicService.client = null; + config.anthropic.apiKey = ''; + try { + const result = await analyze(); + assert.match(result.error, /not initialized/); + assert.deepStrictEqual(result.document, { + tags: [], + correspondent: null, + }); + await quiet(() => + assert.rejects( + anthropicService.generateText('hello'), + /not initialized/ + ) + ); + const status = await quiet(() => anthropicService.checkStatus()); + assert.strictEqual(status.status, 'error'); + } finally { + anthropicService.client = client; + config.anthropic.apiKey = defaults.anthropic.apiKey; + } + }); + } finally { + restore(); + await fake.close(); + await fakeOpenAI.close(); + process.chdir(originalCwd); + fs.rmSync(tempRoot, { recursive: true, force: true }); + } + + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) process.exit(1); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/tests/test-anthropic-settings.js b/tests/test-anthropic-settings.js new file mode 100644 index 00000000..3f958296 --- /dev/null +++ b/tests/test-anthropic-settings.js @@ -0,0 +1,1098 @@ +'use strict'; + +/** + * Anthropic in the setup wizard, the settings page and their validation. + * + * The provider itself is tested elsewhere (test-anthropic-provider.js); this + * file holds the configuration path together: which configuration counts as + * complete, how a key and a model are proved without spending tokens, how the + * model list falls back when the live one cannot be had, what the settings + * save stores and refuses, and what the two pages render. + * + * Claude OCR gets the most attention because its credentials are borrowed. + * Its key is the OCR key or, failing that, the Anthropic AI key, and the OCR + * fields are shared with Mistral and the local providers. A Mistral key or an + * auto-detected Ollama URL left in OCR_API_KEY / OCR_API_URL would win over + * the Anthropic settings at runtime and end in a 401 or a request to the + * wrong host, so the save route clears both when the provider crosses to or + * from Claude unless something new was typed. + * + * Nothing here reaches api.anthropic.com: the SDK talks to a small fake + * server on 127.0.0.1 (allowed through PAPERLESS_AI_SETUP_ALLOW_LOCALHOST, + * which is exactly the switch the SSRF rules read), and the OCR round trip is + * a stub of mistralOcrService.runAnthropicOcr(). + */ + +const assert = require('assert'); +const http = require('http'); +const path = require('path'); +const { mountRouter, REPO_ROOT } = require('./helpers/mount-router'); +const { + ANTHROPIC_DEFAULT_MODEL, + ANTHROPIC_SUGGESTED_MODELS, + ANTHROPIC_EFFORT_LEVELS, +} = require('../services/anthropicSupport'); +const { + sanitizeConfigForBootstrap, +} = require('../services/bootstrapConfigSanitizer'); + +const API_KEY = 'test-api-key'; +const GOOD_KEY = 'sk-ant-good-key'; +const AI_KEY = 'sk-ant-ai-provider-key'; +const STALE_MISTRAL_KEY = 'mistral-key-left-behind'; +const SUGGESTED_IDS = ANTHROPIC_SUGGESTED_MODELS.map((entry) => entry.id); +const LIVE_MODELS = ['claude-live-one', 'claude-live-two', ...SUGGESTED_IDS]; + +// server.js sets these on res.locals for every page; the harness does not run +// server.js, so the shell partials would throw on the first missing one. +const SHELL_LOCALS = { + theme: 'light', + csrfToken: 'test-csrf', + appVersion: 'test', + appCommitSha: 'test', + appPaperlessNgxVersion: 'test', + appNodeVersion: 'test', + appPlatform: 'test', + appNodeEnv: 'test', + appAiProvider: 'anthropic', + appOcrProvider: 'anthropic', + appServerTimeUtc: 'test', + appServerTimezone: 'test', + appPaperlessApiUrl: 'test', + appOllamaApiUrl: 'test', + appOllamaModel: 'test', + appCustomBaseUrl: 'test', + appCustomModel: 'test', + appAzureEndpoint: 'test', + appAzureDeploymentName: 'test', + appAzureApiVersion: 'test', + appAnthropicModel: 'test', + appAnthropicApiUrl: 'default', + appMistralOcrModel: 'test', + appScanInterval: 'test', + appTokenLimit: 'test', + appResponseTokens: 'test', + appTrustProxy: 'test', + appUseExistingData: 'no', + appRestrictTags: 'no', + appRestrictCorrespondents: 'no', + appRestrictDocumentTypes: 'no', + appDateFormat: 'DD.MM.YYYY', + appOcrEnabled: false, + appPaperlessTokenSet: false, + appOpenAiKeySet: false, + appCustomKeySet: false, + appAzureKeySet: false, + appAnthropicKeySet: true, + appMistralKeySet: false, + appApiKeySet: false, +}; + +// Every key a test below sets; reset before each scenario so one save does +// not leak into the next (the stubbed saveConfig writes onto process.env). +const SCENARIO_KEYS = [ + 'AI_PROVIDER', + 'ANTHROPIC_API_KEY', + 'ANTHROPIC_API_URL', + 'ANTHROPIC_MODEL', + 'ANTHROPIC_EFFORT', + 'ANTHROPIC_MAX_TOKENS', + 'ANTHROPIC_STRUCTURED_OUTPUT', + 'ANTHROPIC_PROMPT_CACHING', + 'MISTRAL_OCR_ENABLED', + 'OCR_PROVIDER', + 'OCR_API_KEY', + 'OCR_API_URL', + 'MISTRAL_API_KEY', + 'MISTRAL_OCR_MODEL', + 'PAPERLESS_AI_SETUP_ALLOW_LOCALHOST', +]; + +function setScenario(values) { + SCENARIO_KEYS.forEach((key) => delete process.env[key]); + Object.entries(values).forEach(([key, value]) => { + process.env[key] = String(value); + }); +} + +let passed = 0; +let failed = 0; + +// A successful POST /settings or /api/setup/complete schedules +// process.exit(0) five seconds later. +const realExit = process.exit.bind(process); +process.exit = () => {}; + +async function test(name, fn) { + try { + await fn(); + console.log(`✅ ${name}`); + passed++; + } catch (error) { + console.error(`❌ ${name}`); + console.error(` ${error.stack || error.message}`); + failed++; + } +} + +function sliceBetween(source, from, to) { + const start = source.indexOf(from); + assert.ok(start !== -1, `Could not find ${from}`); + const end = source.indexOf(to, start); + assert.ok(end > start, `Could not delimit the block starting at ${from}`); + return source.slice(start, end); +} + +/* The two Models endpoints the configuration path uses, answered the way the + Claude API answers them: 401 for an unknown key, 404 for an unknown model, + and a single page for the list. */ +function startFakeAnthropic() { + const requests = []; + const server = http.createServer((req, res) => { + const url = new URL(req.url, 'http://fake'); + requests.push({ + method: req.method, + path: url.pathname, + headers: req.headers, + }); + const send = (status, body) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); + }; + const error = (status, type, message) => + send(status, { type: 'error', error: { type, message } }); + + if (req.headers['x-api-key'] !== GOOD_KEY) { + return error(401, 'authentication_error', 'invalid x-api-key'); + } + if (req.method === 'GET' && url.pathname === '/v1/models') { + return send(200, { + data: LIVE_MODELS.map((id) => ({ + type: 'model', + id, + display_name: id, + created_at: '2026-01-01T00:00:00Z', + })), + has_more: false, + first_id: LIVE_MODELS[0], + last_id: LIVE_MODELS[LIVE_MODELS.length - 1], + }); + } + const match = url.pathname.match(/^\/v1\/models\/([^/]+)$/); + if (req.method === 'GET' && match) { + const id = decodeURIComponent(match[1]); + if (!LIVE_MODELS.includes(id)) { + return error(404, 'not_found_error', `model: ${id}`); + } + return send(200, { + type: 'model', + id, + display_name: id, + created_at: '2026-01-01T00:00:00Z', + }); + } + return error(404, 'not_found_error', 'no such route'); + }); + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => + resolve({ + server, + requests, + url: `http://127.0.0.1:${server.address().port}`, + }) + ); + }); +} + +(async () => { + const fake = await startFakeAnthropic(); + const savedConfigs = []; + const harness = await mountRouter({ + env: { PAPERLESS_AI_SETUP_ALLOW_LOCALHOST: 'yes' }, + // No account exists in the temp database, so the setup wizard is open. + stub: ({ setupService }) => { + // The real saveConfig writes the merged configuration back onto + // process.env and touches the data directory; the first half is what + // later reads see. + setupService.saveConfig = async (config) => { + savedConfigs.push(config); + Object.entries(config).forEach(([key, value]) => { + process.env[key] = Array.isArray(value) + ? value.join(',') + : String(value); + }); + }; + }, + }); + Object.assign(harness.app.locals, SHELL_LOCALS); + const { setupService } = harness; + + // The very instance setupService requires lazily for the OCR round trip. + const mistralOcrService = require( + path.join(REPO_ROOT, 'services', 'mistralOcrService.js') + ); + const ocrCalls = []; + let ocrAnswer = () => setupService.getOcrValidationToken(); + mistralOcrService.runAnthropicOcr = async (params) => { + ocrCalls.push(params); + return ocrAnswer(params); + }; + + const request = (method, route, body) => + fetch(harness.base + route, { + method, + redirect: 'manual', + headers: { 'x-api-key': API_KEY, 'Content-Type': 'application/json' }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const lastSaved = () => savedConfigs[savedConfigs.length - 1]; + + try { + // ── hasRequiredConfiguration ──────────────────────────────────────────── + await test('hasRequiredConfiguration: anthropic needs a key and a model', async () => { + const base = { + PAPERLESS_API_URL: 'http://paperless', + AI_PROVIDER: 'anthropic', + }; + assert.strictEqual( + setupService.hasRequiredConfiguration({ + ...base, + ANTHROPIC_API_KEY: GOOD_KEY, + ANTHROPIC_MODEL: ANTHROPIC_DEFAULT_MODEL, + }), + true + ); + assert.strictEqual( + setupService.hasRequiredConfiguration({ + ...base, + ANTHROPIC_MODEL: ANTHROPIC_DEFAULT_MODEL, + }), + false, + 'no key is no configuration' + ); + assert.strictEqual( + setupService.hasRequiredConfiguration({ + ...base, + ANTHROPIC_API_KEY: GOOD_KEY, + ANTHROPIC_MODEL: ' ', + }), + false, + 'a blank model is no model' + ); + }); + + await test('an injected key alone is complete: the snapshot fills in the default model', async () => { + setScenario({ AI_PROVIDER: 'anthropic', ANTHROPIC_API_KEY: GOOD_KEY }); + const snapshot = setupService.getRuntimeConfigurationSnapshot(); + assert.strictEqual(snapshot.ANTHROPIC_MODEL, ANTHROPIC_DEFAULT_MODEL); + assert.strictEqual(snapshot.ANTHROPIC_API_KEY, GOOD_KEY); + }); + + // ── validateAnthropicConfig / validateConfig ──────────────────────────── + await test('validateAnthropicConfig proves key and model with a models lookup', async () => { + setScenario({ PAPERLESS_AI_SETUP_ALLOW_LOCALHOST: 'yes' }); + fake.requests.length = 0; + assert.strictEqual( + await setupService.validateAnthropicConfig( + GOOD_KEY, + 'claude-live-one', + fake.url + ), + true + ); + assert.deepStrictEqual( + fake.requests.map((entry) => `${entry.method} ${entry.path}`), + ['GET /v1/models/claude-live-one'], + 'one GET /v1/models/{id} and no Messages request, so no tokens are spent' + ); + assert.strictEqual( + await setupService.validateAnthropicConfig( + 'sk-ant-wrong', + 'claude-live-one', + fake.url + ), + false, + 'a refused key fails' + ); + assert.strictEqual( + await setupService.validateAnthropicConfig( + GOOD_KEY, + 'claude-unknown', + fake.url + ), + false, + 'an unknown model fails' + ); + assert.strictEqual( + await setupService.validateAnthropicConfig( + '', + 'claude-live-one', + fake.url + ), + false, + 'no key fails without a request' + ); + }); + + await test('validateAnthropicConfig applies the SSRF rules to a proxy URL', async () => { + setScenario({}); + fake.requests.length = 0; + assert.strictEqual( + await setupService.validateAnthropicConfig( + GOOD_KEY, + 'claude-live-one', + fake.url + ), + false, + 'localhost is refused unless PAPERLESS_AI_SETUP_ALLOW_LOCALHOST allows it' + ); + assert.strictEqual( + fake.requests.length, + 0, + 'the refused URL is never contacted' + ); + }); + + await test('validateConfig routes anthropic to validateAnthropicConfig', async () => { + const originalPaperless = setupService.validatePaperlessConfig; + const originalAnthropic = setupService.validateAnthropicConfig; + const calls = []; + setupService.validatePaperlessConfig = async () => true; + setupService.validateAnthropicConfig = async (...args) => { + calls.push(args); + return args[0] === GOOD_KEY; + }; + try { + await setupService.validateConfig({ + PAPERLESS_API_URL: 'http://paperless', + PAPERLESS_API_TOKEN: 't', + AI_PROVIDER: 'anthropic', + ANTHROPIC_API_KEY: GOOD_KEY, + ANTHROPIC_API_URL: 'https://proxy.example', + }); + assert.deepStrictEqual(calls[0], [ + GOOD_KEY, + ANTHROPIC_DEFAULT_MODEL, + 'https://proxy.example', + ]); + await assert.rejects( + setupService.validateConfig({ + PAPERLESS_API_URL: 'http://paperless', + PAPERLESS_API_TOKEN: 't', + AI_PROVIDER: 'anthropic', + ANTHROPIC_API_KEY: 'sk-ant-wrong', + ANTHROPIC_MODEL: 'claude-live-two', + }), + /Invalid Anthropic configuration/ + ); + assert.strictEqual(calls[1][1], 'claude-live-two'); + } finally { + setupService.validatePaperlessConfig = originalPaperless; + setupService.validateAnthropicConfig = originalAnthropic; + } + }); + + // ── Model discovery ───────────────────────────────────────────────────── + await test('model discovery returns the live list when the key works', async () => { + setScenario({ PAPERLESS_AI_SETUP_ALLOW_LOCALHOST: 'yes' }); + const result = await setupService.discoverAnthropicModels({ + apiKey: GOOD_KEY, + apiUrl: fake.url, + }); + assert.strictEqual(result.live, true); + assert.deepStrictEqual(result.models, LIVE_MODELS); + }); + + await test('model discovery falls back to the suggested models', async () => { + setScenario({ PAPERLESS_AI_SETUP_ALLOW_LOCALHOST: 'yes' }); + const refused = await setupService.discoverAnthropicModels({ + apiKey: 'sk-ant-wrong', + apiUrl: fake.url, + }); + assert.strictEqual(refused.live, false); + assert.deepStrictEqual(refused.models, SUGGESTED_IDS); + assert.ok(refused.error, 'the reason travels with the fallback'); + + const keyless = await setupService.discoverAnthropicModels({ + apiKey: '', + }); + assert.strictEqual(keyless.live, false); + assert.deepStrictEqual(keyless.models, SUGGESTED_IDS); + + setScenario({}); + const blocked = await setupService.discoverAnthropicModels({ + apiKey: GOOD_KEY, + apiUrl: fake.url, + }); + assert.strictEqual( + blocked.live, + false, + 'a URL the SSRF rules refuse is not contacted' + ); + assert.match(blocked.error, /URL validation failed/); + }); + + await test('POST /api/settings/ai/models: stored key, then the fallback with a message', async () => { + setScenario({ + PAPERLESS_AI_SETUP_ALLOW_LOCALHOST: 'yes', + ANTHROPIC_API_KEY: GOOD_KEY, + ANTHROPIC_API_URL: fake.url, + }); + let response = await request('POST', '/api/settings/ai/models', { + aiProvider: 'anthropic', + apiUrl: fake.url, + token: '', + }); + let body = await response.json(); + assert.strictEqual(response.status, 200); + assert.strictEqual(body.live, true, 'an empty field uses the stored key'); + assert.deepStrictEqual(body.models, LIVE_MODELS); + + setScenario({ PAPERLESS_AI_SETUP_ALLOW_LOCALHOST: 'yes' }); + response = await request('POST', '/api/settings/ai/models', { + aiProvider: 'anthropic', + apiUrl: fake.url, + token: 'sk-ant-wrong', + }); + body = await response.json(); + assert.strictEqual(response.status, 200); + assert.strictEqual(body.success, true); + assert.strictEqual(body.live, false); + assert.deepStrictEqual(body.models, SUGGESTED_IDS); + assert.match(body.message, /suggested Claude models/); + }); + + await test('the stored Claude key never goes to a host that is not configured', async () => { + // The fake answers on 127.0.0.1 and localhost alike; only the stored + // URL names one of them, so the other counts as somebody else's host. + const otherHost = fake.url.includes('127.0.0.1') + ? fake.url.replace('127.0.0.1', 'localhost') + : fake.url.replace('localhost', '127.0.0.1'); + setScenario({ + PAPERLESS_AI_SETUP_ALLOW_LOCALHOST: 'yes', + ANTHROPIC_API_KEY: GOOD_KEY, + ANTHROPIC_API_URL: fake.url, + OCR_PROVIDER: 'anthropic', + OCR_API_KEY: GOOD_KEY, + }); + + let response = await request('POST', '/api/settings/ai/models', { + aiProvider: 'anthropic', + apiUrl: otherHost, + token: '', + }); + let body = await response.json(); + assert.strictEqual(response.status, 200); + assert.strictEqual(body.live, false, 'AI: no stored key for a new host'); + + response = await request('POST', '/api/settings/ocr/models', { + provider: 'anthropic', + apiUrl: otherHost, + apiKey: '', + }); + body = await response.json(); + assert.strictEqual(response.status, 200); + assert.strictEqual(body.live, false, 'OCR: no stored key for a new host'); + + const connection = setupService.resolveAnthropicOcrConnection({ + anthropicApiUrl: otherHost, + }); + assert.strictEqual(connection.apiKey, ''); + assert.strictEqual( + setupService.resolveAnthropicOcrConnection({}).apiKey, + GOOD_KEY, + 'the configured URL still gets the stored key' + ); + }); + + // ── validateOcrConfig for Claude OCR ──────────────────────────────────── + await test('validateOcrConfig(anthropic) runs one OCR round trip with the typed key', async () => { + setScenario({ ANTHROPIC_API_KEY: AI_KEY }); + ocrCalls.length = 0; + const valid = await setupService.validateOcrConfig({ + enabled: 'yes', + provider: 'anthropic', + apiKey: 'sk-ant-ocr-key', + model: 'claude-sonnet-5-5', + }); + assert.strictEqual(valid, true); + assert.strictEqual(ocrCalls.length, 1); + const call = ocrCalls[0]; + assert.strictEqual( + call.apiKey, + 'sk-ant-ocr-key', + 'the OCR key wins over the AI key' + ); + assert.strictEqual(call.model, 'claude-sonnet-5-5'); + assert.strictEqual(call.mimeType, 'image/png'); + assert.ok( + !call.base64.startsWith('data:'), + 'plain base64, not a data URL' + ); + assert.ok(Buffer.from(call.base64, 'base64').length > 0); + assert.ok(call.timeoutMs > 0); + }); + + await test('validateOcrConfig(anthropic) falls back to the Anthropic AI key, URL and model', async () => { + setScenario({ + ANTHROPIC_API_KEY: AI_KEY, + PAPERLESS_AI_SETUP_ALLOW_LOCALHOST: 'yes', + }); + ocrCalls.length = 0; + assert.strictEqual( + await setupService.validateOcrConfig({ + enabled: 'yes', + provider: 'anthropic', + apiKey: '', + // The shared OCR model field still holds the Mistral default. + model: 'mistral-ocr-latest', + anthropicApiKey: 'sk-ant-typed-ai-key', + anthropicApiUrl: fake.url, + anthropicModel: 'claude-opus-5-5', + }), + true + ); + assert.strictEqual(ocrCalls[0].apiKey, 'sk-ant-typed-ai-key'); + assert.strictEqual(ocrCalls[0].apiUrl, fake.url); + assert.strictEqual( + ocrCalls[0].model, + 'claude-opus-5-5', + 'a Mistral model is never sent to Claude' + ); + + ocrCalls.length = 0; + assert.strictEqual( + await setupService.validateOcrConfig({ + enabled: 'yes', + provider: 'anthropic', + apiKey: '', + model: '', + }), + true + ); + assert.strictEqual( + ocrCalls[0].apiKey, + AI_KEY, + 'the stored ANTHROPIC_API_KEY is the last fallback' + ); + assert.strictEqual(ocrCalls[0].model, ANTHROPIC_DEFAULT_MODEL); + }); + + await test('validateOcrConfig(anthropic) fails without a key, on a wrong answer and on a blocked URL', async () => { + setScenario({}); + ocrCalls.length = 0; + assert.strictEqual( + await setupService.validateOcrConfig({ + enabled: 'yes', + provider: 'anthropic', + }), + false + ); + assert.strictEqual(ocrCalls.length, 0, 'no key, no request'); + + ocrAnswer = () => ''; + try { + assert.strictEqual( + await setupService.validateOcrConfig({ + enabled: 'yes', + provider: 'anthropic', + apiKey: 'sk-ant-ocr-key', + }), + false, + 'an empty transcript ("no text found") is no proof' + ); + } finally { + ocrAnswer = () => setupService.getOcrValidationToken(); + } + + ocrCalls.length = 0; + assert.strictEqual( + await setupService.validateOcrConfig({ + enabled: 'yes', + provider: 'anthropic', + apiKey: 'sk-ant-ocr-key', + apiUrl: 'http://127.0.0.1:9', + }), + false, + 'a proxy URL goes through the SSRF rules' + ); + assert.strictEqual(ocrCalls.length, 0); + }); + + await test('POST /api/settings/ocr/test never hands a stored Mistral key to Claude', async () => { + setScenario({ + ANTHROPIC_API_KEY: AI_KEY, + OCR_PROVIDER: 'mistral', + OCR_API_KEY: STALE_MISTRAL_KEY, + MISTRAL_API_KEY: STALE_MISTRAL_KEY, + }); + ocrCalls.length = 0; + const response = await request('POST', '/api/settings/ocr/test', { + enabled: 'yes', + provider: 'anthropic', + apiKey: '', + model: '', + }); + const body = await response.json(); + assert.strictEqual(body.success, true, body.message); + assert.strictEqual(ocrCalls[0].apiKey, AI_KEY); + }); + + // ── Bootstrap sanitizer ───────────────────────────────────────────────── + await test('the setup bootstrap drops ANTHROPIC_API_KEY and keeps the rest', async () => { + const sanitized = sanitizeConfigForBootstrap({ + ANTHROPIC_API_KEY: GOOD_KEY, + ANTHROPIC_MODEL: 'claude-live-one', + ANTHROPIC_API_URL: 'https://proxy.example', + }); + assert.ok(!('ANTHROPIC_API_KEY' in sanitized)); + assert.strictEqual(sanitized.ANTHROPIC_MODEL, 'claude-live-one'); + assert.strictEqual(sanitized.ANTHROPIC_API_URL, 'https://proxy.example'); + }); + + // ── The settings page ─────────────────────────────────────────────────── + await test('GET /settings renders the Anthropic option, block and Claude OCR option', async () => { + setScenario({ + AI_PROVIDER: 'anthropic', + ANTHROPIC_API_KEY: GOOD_KEY, + ANTHROPIC_MODEL: 'claude-sonnet-5-5', + ANTHROPIC_EFFORT: 'high', + ANTHROPIC_MAX_TOKENS: '24000', + ANTHROPIC_STRUCTURED_OUTPUT: 'no', + ANTHROPIC_PROMPT_CACHING: 'yes', + MISTRAL_OCR_ENABLED: 'yes', + OCR_PROVIDER: 'anthropic', + }); + const response = await request('GET', '/settings'); + assert.strictEqual(response.status, 200); + const html = await response.text(); + + assert.ok(!html.includes(GOOD_KEY), 'the key never reaches the page'); + assert.match( + html, + /