diff --git a/apps/desktop/.maka-shots/3129-facts-backed-model-selector-open.png b/apps/desktop/.maka-shots/3129-facts-backed-model-selector-open.png new file mode 100644 index 0000000000..ab29ffbe0c Binary files /dev/null and b/apps/desktop/.maka-shots/3129-facts-backed-model-selector-open.png differ diff --git a/apps/desktop/.maka-shots/3129-facts-backed-model-selector.png b/apps/desktop/.maka-shots/3129-facts-backed-model-selector.png new file mode 100644 index 0000000000..fbf80f4acc Binary files /dev/null and b/apps/desktop/.maka-shots/3129-facts-backed-model-selector.png differ diff --git a/packages/core/package.json b/packages/core/package.json index ffc9eec2e1..3c1894a6af 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -81,6 +81,7 @@ "./llm-connections": "./dist/llm-connections.js", "./provider-registry": "./dist/provider-registry.js", "./model-catalog": "./dist/model-catalog.js", + "./model-facts": "./dist/model-facts.js", "./model-metadata": "./dist/model-metadata.js", "./model-web-search": "./dist/model-web-search.js", "./model-thinking": "./dist/model-thinking.js", diff --git a/packages/core/src/__tests__/llm-connections.test.ts b/packages/core/src/__tests__/llm-connections.test.ts index e6cf144aed..302f950952 100644 --- a/packages/core/src/__tests__/llm-connections.test.ts +++ b/packages/core/src/__tests__/llm-connections.test.ts @@ -131,6 +131,33 @@ test('model reconciliation never invents a default the user cleared', () => { ); }); +test('model reconciliation preserves enabled model-fact overrides outside live inventory', () => { + assert.deepEqual( + reconcileConnectionAfterModelFetch( + { + defaultModel: 'custom-model', + enabledModelIds: ['custom-model'], + hasModelInventory: true, + }, + [{ id: 'live-model' }], + { factBackedModelIds: new Set(['custom-model']) }, + ), + { defaultModel: 'custom-model', enabledModelIds: ['custom-model'] }, + ); + assert.deepEqual( + reconcileConnectionAfterModelFetch( + { + defaultModel: 'live-model', + enabledModelIds: ['live-model', 'custom-model'], + hasModelInventory: true, + }, + [{ id: 'live-model' }, { id: 'other-live-model' }], + { factBackedModelIds: new Set(['custom-model']) }, + ), + { defaultModel: 'live-model', enabledModelIds: ['live-model', 'custom-model'] }, + ); +}); + test('a renamed id follows its model, and only for a caller that supplies the table', () => { const curated = [{ id: 'claude-opus-5' }, { id: 'claude-haiku-4-5' }]; const stored = { diff --git a/packages/core/src/__tests__/model-catalog.test.ts b/packages/core/src/__tests__/model-catalog.test.ts index 3a6c57a9ac..4ed5b685e0 100644 --- a/packages/core/src/__tests__/model-catalog.test.ts +++ b/packages/core/src/__tests__/model-catalog.test.ts @@ -132,6 +132,27 @@ test('connection catalogs preserve user-choice provenance without inventing avai assert.deepEqual(entries[2]?.provenance.sources?.userChoice, ['session_model']); }); +test('catalog provenance follows the projected model facts marker used in production', () => { + const [entry] = buildConnectionModelCatalogEntries({ + connection: { + slug: 'facts', + providerType: 'openai', + defaultModel: 'custom-model', + models: [ + { + id: 'custom-model', + contextWindow: 200_000, + capabilities: { chat: true }, + factOverriddenFields: ['contextWindow', 'capabilities'], + }, + ], + modelSource: 'fetched', + }, + }); + assert.equal(entry?.capabilitySource, 'user_override'); + assert.equal(entry?.contextWindow, 200_000); +}); + test('unknown persisted provider ids return an empty catalog', () => { assert.deepEqual( buildConnectionModelCatalogEntries({ @@ -174,3 +195,22 @@ test('Alibaba Token Plan catalogs the formal Qwen3.8 model instead of its retire assert.equal(model?.canUseAsChatDefault, true, providerType); } }); + +test('saved model choices do not expose unrelated entries', () => { + const entries = buildConnectionModelCatalogEntries({ + connection: { + slug: 'zai-live', + providerType: 'zai-coding-plan', + defaultModel: 'glm-4.7', + models: [{ id: 'glm-4.7' }], + modelSource: 'fetched', + }, + savedModelIds: [{ id: 'saved-custom', source: 'session_model' }], + }); + const saved = entries.find((entry) => entry.id === 'saved-custom'); + assert.equal(saved?.displayName, undefined); + assert.equal( + entries.some((entry) => entry.id === 'hidden'), + false, + ); +}); diff --git a/packages/core/src/__tests__/model-facts.test.ts b/packages/core/src/__tests__/model-facts.test.ts new file mode 100644 index 0000000000..6b29b02d4e --- /dev/null +++ b/packages/core/src/__tests__/model-facts.test.ts @@ -0,0 +1,121 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + applyModelFactOverride, + applyModelFactOverridesToConnection, + decodeModelFactsDocument, + modelFactKey, +} from '../model-facts.js'; + +test('model facts use provider:model keys and merge fields without replacing provider facts', () => { + const key = modelFactKey('openai', 'o4-mini'); + const document = decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { [key]: { contextWindow: 200_000, capabilities: { vision: false } } }, + }); + const model = applyModelFactOverride( + { + id: 'o4-mini', + displayName: 'Provider name', + maxOutputTokens: 4_000, + capabilities: { chat: true, vision: true }, + }, + document.overrides[key], + ); + assert.equal(model.displayName, 'Provider name'); + assert.equal(model.contextWindow, 200_000); + assert.deepEqual(model.capabilities, { chat: true, vision: false }); +}); + +test('malformed and unknown model fact fields are rejected', () => { + assert.throws(() => + decodeModelFactsDocument({ schemaVersion: 1, overrides: { 'openai:o4-mini': { nope: true } } }), + ); + assert.throws(() => + decodeModelFactsDocument({ schemaVersion: 1, overrides: { 'o4-mini': { contextWindow: 1 } } }), + ); + assert.throws(() => + decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { 'openai:o4-mini': { contextWindow: 0 } }, + }), + ); + assert.throws(() => + decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { 'openai:o4-mini': { capabilities: { toString: true } } }, + }), + ); + assert.throws(() => + decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { 'toString:model': { contextWindow: 1 } }, + }), + ); +}); + +test('model fact keys preserve colons in provider model ids', () => { + const key = modelFactKey('ollama-cloud', 'gpt-oss:120b'); + assert.equal(key, 'ollama-cloud:gpt-oss:120b'); + const document = decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { [key]: { contextWindow: 131_072 } }, + }); + assert.equal(document.overrides[key]?.contextWindow, 131_072); +}); + +test('override-only models are projected only when enabled', () => { + const connection = { + slug: 'openai', + providerType: 'openai' as const, + defaultModel: 'custom', + enabledModelIds: ['custom'], + models: [{ id: 'provider-model' }], + }; + const result = applyModelFactOverridesToConnection(connection, { + 'openai:custom': { contextWindow: 64_000 }, + 'openai:hidden': { contextWindow: 1_000 }, + }); + assert.deepEqual(result.models, [ + { id: 'provider-model' }, + { + id: 'custom', + contextWindow: 64_000, + inputLimit: 64_000, + factOverriddenFields: ['contextWindow', 'inputLimit'], + }, + ]); +}); + +test('context window facts cannot be truncated by an older input limit', () => { + const result = applyModelFactOverride( + { id: 'model', contextWindow: 8_192, inputLimit: 8_192 }, + { contextWindow: 200_000 }, + ); + assert.equal(result.contextWindow, 200_000); + assert.equal(result.inputLimit, 200_000); +}); + +test('overrides replace fields on discovered models while preserving untouched provider facts', () => { + const result = applyModelFactOverridesToConnection( + { + providerType: 'openai', + defaultModel: 'provider-model', + enabledModelIds: ['provider-model'], + models: [ + { id: 'provider-model', contextWindow: 8_000, capabilities: { chat: true, vision: true } }, + ], + }, + { 'openai:provider-model': { contextWindow: 64_000, capabilities: { vision: false } } }, + ); + assert.equal(result.models?.[0]?.contextWindow, 64_000); + assert.deepEqual(result.models?.[0]?.capabilities, { chat: true, vision: false }); +}); + +test('catalog capabilities preserve web search facts from metadata and overrides', () => { + const result = applyModelFactOverride( + { id: 'web-model', capabilities: { webSearch: true } }, + { capabilities: { chat: true } }, + ); + assert.deepEqual(result.capabilities, { webSearch: true, chat: true }); +}); diff --git a/packages/core/src/__tests__/runtime-policy-codec.test.ts b/packages/core/src/__tests__/runtime-policy-codec.test.ts index 491bb2d340..ac80209a3c 100644 --- a/packages/core/src/__tests__/runtime-policy-codec.test.ts +++ b/packages/core/src/__tests__/runtime-policy-codec.test.ts @@ -373,6 +373,50 @@ test('normalizes exact bounded model discovery results', () => { } }); +test('normalizes extended model facts used by the runtime host catalog', () => { + const result = normalizeConnectionModelDiscoveryResult({ + models: [ + { + id: 'custom-model', + description: 'A custom model', + inputLimit: 120_000, + knowledgeCutoff: '2025-01', + structuredOutput: true, + lastUpdated: '2026-01-01', + modalities: { input: ['text', 'image'], output: ['text'] }, + }, + ], + source: 'fetched', + fetchedAt: 42, + }); + assert.deepEqual(result.models[0], { + id: 'custom-model', + description: 'A custom model', + inputLimit: 120_000, + knowledgeCutoff: '2025-01', + structuredOutput: true, + lastUpdated: '2026-01-01', + modalities: { input: ['text', 'image'], output: ['text'] }, + }); +}); + +test('rejects sparse model modality arrays', () => { + assert.throws( + () => + normalizeConnectionModelDiscoveryResult({ + models: [ + { + id: 'custom-model', + modalities: { input: Array(1), output: ['text'] }, + }, + ], + source: 'fetched', + fetchedAt: 42, + }), + RuntimePolicyDomainDecodeError, + ); +}); + test('credential domain validation requires material but leaves capacity to callers', () => { const input = normalizeSetCredentialInput({ locator: { diff --git a/packages/core/src/llm-connections.ts b/packages/core/src/llm-connections.ts index 5f0cf88707..a8abdcde3d 100644 --- a/packages/core/src/llm-connections.ts +++ b/packages/core/src/llm-connections.ts @@ -86,8 +86,27 @@ export interface ModelInfo { input: Array<'text' | 'image' | 'audio' | 'pdf'>; output: Array<'text' | 'image' | 'audio'>; }; + /** + * Read-time provenance for values overlaid from model-facts.json. This is + * never persisted in a provider inventory; it lets catalog consumers show + * where a projected value came from. + */ + factOverriddenFields?: readonly ModelFactField[]; } +export type ModelFactField = + | 'displayName' + | 'description' + | 'apiProtocol' + | 'contextWindow' + | 'inputLimit' + | 'maxOutputTokens' + | 'knowledgeCutoff' + | 'structuredOutput' + | 'lastUpdated' + | 'capabilities' + | 'modalities'; + export type ModelDiscoverySource = 'fetched' | 'fallback'; export interface ModelDiscoveryResult { @@ -258,6 +277,12 @@ export function reconcileConnectionAfterModelFetch( * caller that knows the provider's naming supplies the table. */ readonly aliases?: Readonly>; + /** + * Model-fact-backed ids that were already enabled on this connection. + * These remain selectable when a provider omits them from live discovery; + * the ids are never added to the provider inventory itself. + */ + readonly factBackedModelIds?: ReadonlySet; }, ): { defaultModel: string; @@ -290,6 +315,7 @@ export function reconcileConnectionAfterModelFetch( ), ), ]; + const factBackedModelIds = options?.factBackedModelIds ?? new Set(); if (liveIds.length === 0) { const defaultModel = previousDefault; @@ -308,20 +334,24 @@ export function reconcileConnectionAfterModelFetch( if (connection.hasModelInventory || previousEnabled.length > 0) { return { defaultModel: '', - enabledModelIds: previousEnabled.filter((id) => live.has(id)), + enabledModelIds: previousEnabled.filter((id) => live.has(id) || factBackedModelIds.has(id)), }; } return { defaultModel: liveIds[0]!, enabledModelIds: [liveIds[0]!] }; } const defaultModel = - (live.has(previousDefault) ? previousDefault : undefined) ?? + (live.has(previousDefault) || factBackedModelIds.has(previousDefault) + ? previousDefault + : undefined) ?? previousEnabled.find((id) => live.has(id)) ?? liveIds[0]!; // Keep previously enabled ids that still exist live, plus the (possibly // repaired) default. Do not auto-enable the entire discovered catalog. - const keptEnabled = previousEnabled.filter((id) => live.has(id) || id === defaultModel); + const keptEnabled = previousEnabled.filter( + (id) => live.has(id) || factBackedModelIds.has(id) || id === defaultModel, + ); return { defaultModel, enabledModelIds: connectionEnabledModelIds({ diff --git a/packages/core/src/model-catalog.ts b/packages/core/src/model-catalog.ts index bbb789cb0b..9320f06073 100644 --- a/packages/core/src/model-catalog.ts +++ b/packages/core/src/model-catalog.ts @@ -290,11 +290,13 @@ function makeEntry( providerType: input.providerType, ...(input.connectionSlug ? { connectionSlug: input.connectionSlug } : {}), source, - capabilitySource: normalizedModel.capabilities - ? source - : metadata.capabilities - ? 'static_catalog' - : 'unknown', + capabilitySource: normalizedModel.factOverriddenFields?.includes('capabilities') + ? 'user_override' + : normalizedModel.capabilities + ? source + : metadata.capabilities + ? 'static_catalog' + : 'unknown', unavailableReason, availability: availabilityOf(unavailableReason), canUseAsChatDefault: canUseUnavailableReasonAsDefault(unavailableReason), @@ -353,36 +355,53 @@ function makeMissingDefaultEntry( ): ModelCatalogEntry { const unavailableReason = missingEntryUnavailableReason(input, modelSource); const metadata = lookupModelMetadata(input.providerType, id); + const model: ModelInfo = { id }; const recommendedRank = recommendedRanks.get(id); return { id, - ...displayNameForKnownModel(input.providerType, id), - ...(metadata.description !== undefined ? { description: metadata.description } : {}), + ...displayNameForModel(input.providerType, model), + ...((model.description ?? metadata.description) + ? { description: model.description ?? metadata.description } + : {}), providerType: input.providerType, ...(input.connectionSlug ? { connectionSlug: input.connectionSlug } : {}), source: 'unknown', - capabilitySource: metadata.capabilities ? 'static_catalog' : 'unknown', + capabilitySource: model.capabilities + ? 'user_override' + : metadata.capabilities + ? 'static_catalog' + : 'unknown', unavailableReason, availability: availabilityOf(unavailableReason), canUseAsChatDefault: canUseUnavailableReasonAsDefault(unavailableReason), isDefault: true, - capabilities: normalizeCapabilities(metadata.capabilities), + capabilities: normalizeCapabilities( + mergeCapabilities(model.capabilities, metadata.capabilities), + ), lifecycle: metadata.lifecycle ?? 'unknown', ...(recommendedRank ? { recommendedRank } : {}), ...(metadata.docsUrl ? { docsUrl: metadata.docsUrl } : {}), - ...(metadata.contextWindow !== undefined ? { contextWindow: metadata.contextWindow } : {}), - ...(metadata.inputLimit !== undefined ? { inputLimit: metadata.inputLimit } : {}), - ...(metadata.maxOutputTokens !== undefined - ? { maxOutputTokens: metadata.maxOutputTokens } + ...((model.contextWindow ?? metadata.contextWindow) !== undefined + ? { contextWindow: model.contextWindow ?? metadata.contextWindow } + : {}), + ...((model.inputLimit ?? metadata.inputLimit) !== undefined + ? { inputLimit: model.inputLimit ?? metadata.inputLimit } + : {}), + ...((model.maxOutputTokens ?? metadata.maxOutputTokens) !== undefined + ? { maxOutputTokens: model.maxOutputTokens ?? metadata.maxOutputTokens } + : {}), + ...((model.knowledgeCutoff ?? metadata.knowledgeCutoff) !== undefined + ? { knowledgeCutoff: model.knowledgeCutoff ?? metadata.knowledgeCutoff } : {}), - ...(metadata.knowledgeCutoff !== undefined - ? { knowledgeCutoff: metadata.knowledgeCutoff } + ...((model.structuredOutput ?? metadata.structuredOutput) !== undefined + ? { structuredOutput: model.structuredOutput ?? metadata.structuredOutput } : {}), - ...(metadata.structuredOutput !== undefined - ? { structuredOutput: metadata.structuredOutput } + ...((model.lastUpdated ?? metadata.lastUpdated) !== undefined + ? { lastUpdated: model.lastUpdated ?? metadata.lastUpdated } + : {}), + ...((model.modalities ?? metadata.modalities) !== undefined + ? { modalities: model.modalities ?? metadata.modalities } : {}), - ...(metadata.lastUpdated !== undefined ? { lastUpdated: metadata.lastUpdated } : {}), - ...(metadata.modalities !== undefined ? { modalities: metadata.modalities } : {}), provenance: { modelSource, ...(input.modelsFetchedAt ? { modelsFetchedAt: input.modelsFetchedAt } : {}), @@ -401,36 +420,53 @@ function makeMissingUserChoiceEntry( ): ModelCatalogEntry { const unavailableReason = missingEntryUnavailableReason(input, modelSource); const metadata = lookupModelMetadata(input.providerType, id); + const model: ModelInfo = { id }; const recommendedRank = recommendedRanks.get(id); return { id, - ...displayNameForKnownModel(input.providerType, id), - ...(metadata.description !== undefined ? { description: metadata.description } : {}), + ...displayNameForModel(input.providerType, model), + ...((model.description ?? metadata.description) + ? { description: model.description ?? metadata.description } + : {}), providerType: input.providerType, ...(input.connectionSlug ? { connectionSlug: input.connectionSlug } : {}), source: 'unknown', - capabilitySource: metadata.capabilities ? 'static_catalog' : 'unknown', + capabilitySource: model.capabilities + ? 'user_override' + : metadata.capabilities + ? 'static_catalog' + : 'unknown', unavailableReason, availability: availabilityOf(unavailableReason), canUseAsChatDefault: canUseUnavailableReasonAsDefault(unavailableReason), isDefault: id === normalizedDefaultModel, - capabilities: normalizeCapabilities(metadata.capabilities), + capabilities: normalizeCapabilities( + mergeCapabilities(model.capabilities, metadata.capabilities), + ), lifecycle: metadata.lifecycle ?? 'unknown', ...(recommendedRank ? { recommendedRank } : {}), ...(metadata.docsUrl ? { docsUrl: metadata.docsUrl } : {}), - ...(metadata.contextWindow !== undefined ? { contextWindow: metadata.contextWindow } : {}), - ...(metadata.inputLimit !== undefined ? { inputLimit: metadata.inputLimit } : {}), - ...(metadata.maxOutputTokens !== undefined - ? { maxOutputTokens: metadata.maxOutputTokens } + ...((model.contextWindow ?? metadata.contextWindow) !== undefined + ? { contextWindow: model.contextWindow ?? metadata.contextWindow } + : {}), + ...((model.inputLimit ?? metadata.inputLimit) !== undefined + ? { inputLimit: model.inputLimit ?? metadata.inputLimit } + : {}), + ...((model.maxOutputTokens ?? metadata.maxOutputTokens) !== undefined + ? { maxOutputTokens: model.maxOutputTokens ?? metadata.maxOutputTokens } + : {}), + ...((model.knowledgeCutoff ?? metadata.knowledgeCutoff) !== undefined + ? { knowledgeCutoff: model.knowledgeCutoff ?? metadata.knowledgeCutoff } + : {}), + ...((model.structuredOutput ?? metadata.structuredOutput) !== undefined + ? { structuredOutput: model.structuredOutput ?? metadata.structuredOutput } : {}), - ...(metadata.knowledgeCutoff !== undefined - ? { knowledgeCutoff: metadata.knowledgeCutoff } + ...((model.lastUpdated ?? metadata.lastUpdated) !== undefined + ? { lastUpdated: model.lastUpdated ?? metadata.lastUpdated } : {}), - ...(metadata.structuredOutput !== undefined - ? { structuredOutput: metadata.structuredOutput } + ...((model.modalities ?? metadata.modalities) !== undefined + ? { modalities: model.modalities ?? metadata.modalities } : {}), - ...(metadata.lastUpdated !== undefined ? { lastUpdated: metadata.lastUpdated } : {}), - ...(metadata.modalities !== undefined ? { modalities: metadata.modalities } : {}), provenance: { modelSource, ...(input.modelsFetchedAt ? { modelsFetchedAt: input.modelsFetchedAt } : {}), diff --git a/packages/core/src/model-facts.ts b/packages/core/src/model-facts.ts new file mode 100644 index 0000000000..ffdc8facd2 --- /dev/null +++ b/packages/core/src/model-facts.ts @@ -0,0 +1,293 @@ +import { PROVIDER_REGISTRY, type ProviderType } from './provider-registry.js'; +import type { ModelFactField, ModelInfo } from './llm-connections.js'; +import type { ConnectionCatalogEntry, ConnectionCatalogSnapshot } from './runtime-policy.js'; + +export const MODEL_FACTS_SCHEMA_VERSION = 1 as const; +export const MODEL_FACT_KEY_MAX_LENGTH = 512; +export const MODEL_FACTS_MAX_OVERRIDES = 512; + +export type ModelFactOverride = Readonly< + Omit>, 'modalities'> & { + readonly modalities?: Readonly>; + } +>; +export type ModelFactOverrides = Readonly>; + +export interface ModelFactsDocument { + readonly schemaVersion: typeof MODEL_FACTS_SCHEMA_VERSION; + readonly overrides: ModelFactOverrides; +} + +export class UnsupportedModelFactsSchemaError extends Error { + constructor(readonly schemaVersion: number) { + super(`model-facts.json schema version ${schemaVersion} is not supported`); + this.name = 'UnsupportedModelFactsSchemaError'; + } +} + +const PROVIDER_ID_PATTERN = /^[^:\s]{1,128}$/; +// Model ids may contain colons (for example, Ollama's `gpt-oss:120b`). The +// provider is the only component that is constrained to the first separator. +const MODEL_ID_PATTERN = /^[^\s]{1,256}$/; +const PROVIDER_MODEL_KEY_PATTERN = /^([^:\s]{1,128}):([^\s]{1,256})$/; +const MAX_FACT_NUMBER = 10_000_000_000; + +export function modelFactKey(providerType: ProviderType | string, modelId: string): string { + const provider = providerType.trim(); + const model = modelId.trim(); + if (!provider || !model || !PROVIDER_ID_PATTERN.test(provider) || !MODEL_ID_PATTERN.test(model)) { + throw new Error('Model fact keys must use a non-empty provider:model identifier'); + } + if (!Object.hasOwn(PROVIDER_REGISTRY, provider)) { + throw new Error(`Unknown model-facts provider: ${provider}`); + } + const key = `${provider}:${model}`; + if (key.length > MODEL_FACT_KEY_MAX_LENGTH) throw new Error('Model fact key is too long'); + return key; +} + +export function lookupModelFactOverride( + overrides: ModelFactOverrides | undefined, + providerType: ProviderType | string, + modelId: string, +): ModelFactOverride | undefined { + if (!overrides) return undefined; + try { + return overrides[modelFactKey(providerType, modelId)]; + } catch { + return undefined; + } +} + +/** Return model ids with facts for one provider without exposing other providers. */ +export function modelFactOverrideIdsForProvider( + overrides: ModelFactOverrides | undefined, + providerType: ProviderType | string, +): string[] { + if (!overrides) return []; + const prefix = `${providerType.trim()}:`; + return Object.keys(overrides) + .filter((key) => key.startsWith(prefix)) + .map((key) => key.slice(prefix.length)); +} + +export function decodeModelFactsDocument(value: unknown): ModelFactsDocument { + if (!isRecord(value)) throw new Error('model-facts.json must be an object'); + if (!Number.isSafeInteger(value.schemaVersion)) { + throw new Error('model-facts.json schemaVersion must be an integer'); + } + if (value.schemaVersion !== MODEL_FACTS_SCHEMA_VERSION) + throw new UnsupportedModelFactsSchemaError(value.schemaVersion); + if (!isRecord(value.overrides)) throw new Error('model-facts.json.overrides must be an object'); + const keys = Object.keys(value.overrides); + if (keys.length > MODEL_FACTS_MAX_OVERRIDES) + throw new Error('model-facts.json has too many overrides'); + const overrides: Record = {}; + for (const key of keys) { + const match = PROVIDER_MODEL_KEY_PATTERN.exec(key); + if (!match || key.length > MODEL_FACT_KEY_MAX_LENGTH) throw new Error('Invalid model fact key'); + modelFactKey(match[1]!, match[2]!); + overrides[key] = normalizeModelFactOverride(value.overrides[key]); + } + return { schemaVersion: MODEL_FACTS_SCHEMA_VERSION, overrides }; +} + +export function normalizeModelFactOverride(value: unknown): ModelFactOverride { + if (!isRecord(value)) throw new Error('Model fact override must be an object'); + const allowed = new Set([ + 'displayName', + 'description', + 'apiProtocol', + 'contextWindow', + 'inputLimit', + 'maxOutputTokens', + 'knowledgeCutoff', + 'structuredOutput', + 'lastUpdated', + 'capabilities', + 'modalities', + ]); + for (const key of Object.keys(value)) + if (!allowed.has(key)) throw new Error(`Unknown model fact field: ${key}`); + const result: Record = {}; + for (const key of ['displayName', 'description', 'knowledgeCutoff', 'lastUpdated'] as const) { + if (key in value) { + if (typeof value[key] !== 'string' || value[key].length > 2048) + throw new Error(`Invalid ${key}`); + result[key] = value[key]; + } + } + if ('apiProtocol' in value) { + if ( + value.apiProtocol !== 'openai-chat' && + value.apiProtocol !== 'openai-responses' && + value.apiProtocol !== 'anthropic-messages' + ) + throw new Error('Invalid apiProtocol'); + result.apiProtocol = value.apiProtocol; + } + for (const key of ['contextWindow', 'inputLimit', 'maxOutputTokens'] as const) { + if (key in value) { + const number = value[key]; + if (!isPositiveBoundedInteger(number)) throw new Error(`Invalid ${key}`); + result[key] = number; + } + } + for (const key of ['structuredOutput'] as const) { + if (key in value) { + if (typeof value[key] !== 'boolean') throw new Error(`Invalid ${key}`); + result[key] = value[key]; + } + } + if ('capabilities' in value) result.capabilities = normalizeCapabilities(value.capabilities); + if ('modalities' in value) result.modalities = normalizeModalities(value.modalities); + return result as ModelFactOverride; +} + +function normalizeCapabilities(value: unknown): NonNullable { + if (!isRecord(value)) throw new Error('Invalid capabilities'); + const result: Record = {}; + const allowed = [ + 'chat', + 'vision', + 'reasoning', + 'functionCalling', + 'imageGeneration', + 'webSearch', + ] as const; + for (const key of allowed) { + if (key in value) { + if (typeof value[key] !== 'boolean') throw new Error(`Invalid capability: ${key}`); + result[key] = value[key]; + } + } + for (const key of Object.keys(value)) + if (!allowed.includes(key as (typeof allowed)[number])) { + throw new Error(`Unknown capability: ${key}`); + } + return result; +} + +function normalizeModalities(value: unknown): NonNullable { + if (!isRecord(value)) throw new Error('Invalid modalities'); + if (value.input === undefined && value.output === undefined) + throw new Error('Invalid modalities'); + const input = normalizeModalityDirection(value.input, isModality); + const output = normalizeModalityDirection(value.output, isOutputModality); + return { + ...(input === undefined ? {} : { input }), + ...(output === undefined ? {} : { output }), + }; +} + +function normalizeModalityDirection( + value: unknown, + allowed: (value: unknown) => value is T, +): T[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) throw new Error('Invalid modality value'); + const entries = Array.from(value); + if (!entries.every(allowed)) throw new Error('Invalid modality value'); + return [...new Set(entries)]; +} + +function isModality(value: unknown): value is 'text' | 'image' | 'audio' | 'pdf' { + return value === 'text' || value === 'image' || value === 'audio' || value === 'pdf'; +} +function isOutputModality(value: unknown): value is 'text' | 'image' | 'audio' { + return value === 'text' || value === 'image' || value === 'audio'; +} +function isPositiveBoundedInteger(value: unknown): value is number { + return ( + typeof value === 'number' && + Number.isSafeInteger(value) && + value > 0 && + value <= MAX_FACT_NUMBER + ); +} +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function applyModelFactOverride( + model: ModelInfo, + override: ModelFactOverride | undefined, +): ModelInfo { + if (!override) return { ...model }; + const overriddenFields = new Set(model.factOverriddenFields); + for (const field of Object.keys(override) as ModelFactField[]) overriddenFields.add(field); + // An authoritative context-window correction must not leave a stale, + // narrower provider input limit to silently win in the runtime resolver. + if (override.contextWindow !== undefined && override.inputLimit === undefined) { + overriddenFields.add('inputLimit'); + } + const modalities = override.modalities + ? { + input: override.modalities.input ?? model.modalities?.input ?? ['text'], + output: override.modalities.output ?? model.modalities?.output ?? ['text'], + } + : model.modalities; + const { modalities: _ignoredModalities, ...scalarOverride } = override; + return { + ...model, + ...scalarOverride, + id: model.id, + factOverriddenFields: [...overriddenFields], + ...(override.contextWindow === undefined || override.inputLimit !== undefined + ? {} + : { inputLimit: override.contextWindow }), + ...(override.capabilities === undefined + ? {} + : { capabilities: { ...model.capabilities, ...override.capabilities } }), + ...(modalities === undefined ? {} : { modalities }), + } satisfies ModelInfo; +} + +type ModelFactConnectionLike = { + readonly providerType: ProviderType; + readonly defaultModel?: string; + readonly models?: readonly ModelInfo[]; + readonly enabledModelIds?: readonly string[]; +}; + +export function applyModelFactOverridesToConnection( + connection: T, + overrides: ModelFactOverrides, +): T { + const models = (connection.models ?? []).map((model) => + applyModelFactOverride( + model, + lookupModelFactOverride(overrides, connection.providerType, model.id), + ), + ); + const existing = new Set(models.map((model) => model.id)); + const enabled = new Set( + connection.enabledModelIds ?? + (connection.defaultModel === undefined ? [] : [connection.defaultModel]), + ); + for (const modelId of enabled) { + if (existing.has(modelId)) continue; + const override = lookupModelFactOverride(overrides, connection.providerType, modelId); + if (override) { + models.push(applyModelFactOverride({ id: modelId }, override)); + existing.add(modelId); + } + } + return { ...connection, models } as T; +} + +export function applyModelFactOverridesToCatalogSnapshot( + snapshot: ConnectionCatalogSnapshot, + overrides: ModelFactOverrides, +): ConnectionCatalogSnapshot { + return { + ...snapshot, + connections: snapshot.connections.map( + (connection) => + applyModelFactOverridesToConnection( + connection, + overrides, + ) as unknown as ConnectionCatalogEntry, + ), + }; +} diff --git a/packages/core/src/runtime-policy.ts b/packages/core/src/runtime-policy.ts index 5ba11406d1..3148cc4228 100644 --- a/packages/core/src/runtime-policy.ts +++ b/packages/core/src/runtime-policy.ts @@ -221,6 +221,8 @@ export interface ConnectionCatalogEntry extends ConnectionConfiguration { readonly modelSource?: ConnectionModelDiscoveryResult['source']; readonly modelsFetchedAt?: ConnectionModelDiscoveryResult['fetchedAt']; readonly lastTest?: ConnectionTestSummary; + /** Digest of the model-facts subset used when `lastTest` was recorded. */ + readonly lastTestModelFactsFingerprint?: string; } export type ConnectionCatalogEntryDraft = ConnectionConfiguration; diff --git a/packages/core/src/runtime-policy/connection-catalog-codec.ts b/packages/core/src/runtime-policy/connection-catalog-codec.ts index 9164336979..b6a3d35816 100644 --- a/packages/core/src/runtime-policy/connection-catalog-codec.ts +++ b/packages/core/src/runtime-policy/connection-catalog-codec.ts @@ -322,6 +322,7 @@ export function decodeCanonicalConnectionCatalogEntry(value: unknown): Connectio 'modelSource', 'modelsFetchedAt', 'lastTest', + 'lastTestModelFactsFingerprint', ], [ 'connectionId', @@ -390,6 +391,15 @@ export function decodeCanonicalConnectionCatalogEntry(value: unknown): Connectio ...(item.lastTest === undefined ? {} : { lastTest: decodeConnectionTestSummary(item.lastTest) }), + ...(item.lastTestModelFactsFingerprint === undefined + ? {} + : { + lastTestModelFactsFingerprint: stringValue( + item.lastTestModelFactsFingerprint, + 'connection test model facts fingerprint', + 128, + ), + }), }; assertCanonicalValue(value, decoded, 'connection catalog entry'); return decoded; @@ -415,7 +425,20 @@ export function decodeConnectionModel(value: unknown): ConnectionModel { const item = exactRecord( value, 'connection model', - ['id', 'displayName', 'apiProtocol', 'contextWindow', 'maxOutputTokens', 'capabilities'], + [ + 'id', + 'displayName', + 'description', + 'apiProtocol', + 'contextWindow', + 'inputLimit', + 'maxOutputTokens', + 'knowledgeCutoff', + 'structuredOutput', + 'lastUpdated', + 'capabilities', + 'modalities', + ], ['id'], ); if ( @@ -442,11 +465,16 @@ export function decodeConnectionModel(value: unknown): ConnectionModel { ); } } + const modalities = + item.modalities === undefined ? undefined : decodeModelModalities(item.modalities); return { id: decodeConnectionModelId(item.id), ...(item.displayName === undefined ? {} : { displayName: stringValue(item.displayName, 'model display name', 512) }), + ...(item.description === undefined + ? {} + : { description: stringValue(item.description, 'model description', 2048) }), ...(item.apiProtocol === undefined ? {} : { apiProtocol: item.apiProtocol }), ...(item.contextWindow === undefined ? {} @@ -458,6 +486,16 @@ export function decodeConnectionModel(value: unknown): ConnectionModel { Number.MAX_SAFE_INTEGER, ), }), + ...(item.inputLimit === undefined + ? {} + : { + inputLimit: integerValue( + item.inputLimit, + 'model input limit', + 1, + Number.MAX_SAFE_INTEGER, + ), + }), ...(item.maxOutputTokens === undefined ? {} : { @@ -468,10 +506,46 @@ export function decodeConnectionModel(value: unknown): ConnectionModel { Number.MAX_SAFE_INTEGER, ), }), + ...(item.knowledgeCutoff === undefined + ? {} + : { knowledgeCutoff: stringValue(item.knowledgeCutoff, 'model knowledge cutoff', 2048) }), + ...(item.structuredOutput === undefined + ? {} + : { structuredOutput: booleanValue(item.structuredOutput, 'model structured output') }), + ...(item.lastUpdated === undefined + ? {} + : { lastUpdated: stringValue(item.lastUpdated, 'model last updated', 2048) }), ...(capabilities === undefined ? {} : { capabilities }), + ...(modalities === undefined ? {} : { modalities }), }; } +function decodeModelModalities(value: unknown): NonNullable { + const item = exactRecord(value, 'connection model modalities', ['input', 'output']); + if (!Array.isArray(item.input) || !Array.isArray(item.output)) { + throw domainError('connection model modalities must contain input and output arrays'); + } + const input = Array.from(item.input, (entry) => decodeModelInputModality(entry)); + const output = Array.from(item.output, (entry) => decodeModelOutputModality(entry)); + return { input, output }; +} + +function decodeModelInputModality(value: unknown): 'text' | 'image' | 'audio' | 'pdf' { + const modality = stringValue(value, 'connection model input modality', 16); + if (modality !== 'text' && modality !== 'image' && modality !== 'audio' && modality !== 'pdf') { + throw domainError('connection model input modality is invalid'); + } + return modality; +} + +function decodeModelOutputModality(value: unknown): 'text' | 'image' | 'audio' { + const modality = stringValue(value, 'connection model output modality', 16); + if (modality !== 'text' && modality !== 'image' && modality !== 'audio') { + throw domainError('connection model output modality is invalid'); + } + return modality; +} + export function decodeConnectionTestSummary(value: unknown): ConnectionTestSummary { const item = exactRecord( value, diff --git a/packages/runtime/src/__tests__/context-budget-model-facts.test.ts b/packages/runtime/src/__tests__/context-budget-model-facts.test.ts index c003892ca5..9d2507d253 100644 --- a/packages/runtime/src/__tests__/context-budget-model-facts.test.ts +++ b/packages/runtime/src/__tests__/context-budget-model-facts.test.ts @@ -42,3 +42,22 @@ test('a relay user declaration remains ahead of runtime and static model facts', assert.equal(resolveSelectedModelContextWindow(connection, undefined), 32_000); }); + +test('a model-facts context window is the authoritative user declaration', () => { + const connection = { + slug: 'relay', + providerType: 'openai-compatible' as const, + defaultModel: 'relay-model', + models: [ + { + id: 'relay-model', + contextWindow: 200_000, + inputLimit: 200_000, + factOverriddenFields: ['contextWindow', 'inputLimit'] as const, + }, + ], + relayModelProfiles: { 'relay-model': { contextWindow: 8_192 } }, + }; + + assert.equal(resolveSelectedModelContextWindow(connection, undefined), 200_000); +}); diff --git a/packages/runtime/src/context-budget-policy.ts b/packages/runtime/src/context-budget-policy.ts index 886c1780ce..a4b429fe19 100644 --- a/packages/runtime/src/context-budget-policy.ts +++ b/packages/runtime/src/context-budget-policy.ts @@ -419,12 +419,16 @@ export function resolveSelectedModelContextWindow( ): number | undefined { const selectedModelId = modelId ?? connection.defaultModel; if (selectedModelId === undefined) return undefined; - // A user declaration outranks both the relay's /models report and generated - // metadata — mirrors the declared-vision precedence in model-metadata.ts, - // through the same provider-gated seam (relay declarations only). + const model = connection.models?.find((candidate) => candidate.id === selectedModelId); + // A model-facts pin is the cross-provider correction authority. It must win + // over the older relay-only declaration so catalog display and execution use + // the same window. Relay declarations retain their existing precedence when + // there is no facts pin for this field. + if (model?.factOverriddenFields?.includes('contextWindow')) { + return narrowestPositiveLimit(model.contextWindow, model.inputLimit); + } const declared = relayModelProfile(connection, selectedModelId)?.contextWindow; if (declared !== undefined) return declared; - const model = connection.models?.find((candidate) => candidate.id === selectedModelId); const metadata = lookupModelMetadata(connection.providerType, selectedModelId); // Provider/access-path facts outrank static metadata. Within one source, // use the narrowest positive bound: models.dev's input limit can be lower diff --git a/packages/storage/src/__tests__/model-facts-store.test.ts b/packages/storage/src/__tests__/model-facts-store.test.ts new file mode 100644 index 0000000000..b03b87ccc6 --- /dev/null +++ b/packages/storage/src/__tests__/model-facts-store.test.ts @@ -0,0 +1,105 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, readdir, readFile, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { ModelFactsDocumentOwner } from '../model-facts-store.js'; +import { RuntimePolicyStoreError } from '../runtime-policy/errors.js'; +import { cleanupRuntimePolicyDocumentTemps } from '../runtime-policy/document-io.js'; + +test('model facts persist and malformed documents fail closed with a bounded diagnostic', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-model-facts-')); + try { + const owner = new ModelFactsDocumentOwner(); + assert.deepEqual((await owner.read(root)).overrides, {}); + await owner.replace(root, { 'openai:o4-mini': { contextWindow: 200_000 } }); + assert.equal((await owner.read(root)).overrides['openai:o4-mini']?.contextWindow, 200_000); + await writeFile(join(root, 'model-facts.json'), '{not-json}', 'utf8'); + const result = await owner.readWithDiagnostics(root); + assert.equal(result.diagnostic, 'malformed'); + assert.deepEqual(result.document.overrides, {}); + await writeFile( + join(root, 'model-facts.json'), + JSON.stringify({ schemaVersion: 1, overrides: { 'openai:o4-mini': { unknown: true } } }), + 'utf8', + ); + assert.equal((await owner.readWithDiagnostics(root)).diagnostic, 'malformed'); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('model facts reject own prototype keys from JSON input', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-model-facts-prototype-')); + try { + const owner = new ModelFactsDocumentOwner(); + const overrides = JSON.parse('{"__proto__":{"contextWindow":200000}}'); + await assert.rejects( + () => owner.replace(root, overrides), + (error: unknown) => + error instanceof RuntimePolicyStoreError && error.code === 'invalid_policy_input', + ); + assert.deepEqual((await owner.read(root)).overrides, {}); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('model facts temporary writes are removed by runtime policy recovery', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-model-facts-recovery-')); + try { + await writeFile( + join(root, 'model-facts.json.00000000-0000-4000-8000-000000000000.tmp'), + '{}', + 'utf8', + ); + await cleanupRuntimePolicyDocumentTemps(root); + assert.deepEqual(await readdir(root), []); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('future model facts schemas are preserved and cannot be overwritten', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-model-facts-future-')); + try { + const owner = new ModelFactsDocumentOwner(); + const future = JSON.stringify({ + schemaVersion: 2, + overrides: { 'openai:o4-mini': { contextWindow: 1 } }, + }); + await writeFile(join(root, 'model-facts.json'), future, 'utf8'); + const read = await owner.readWithDiagnostics(root); + assert.equal(read.diagnostic, 'unsupported_schema'); + await assert.rejects( + () => owner.replace(root, { 'openai:o4-mini': { contextWindow: 200_000 } }), + (error: unknown) => + error instanceof RuntimePolicyStoreError && error.code === 'invalid_policy_input', + ); + assert.equal(await readFile(join(root, 'model-facts.json'), 'utf8'), future); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('model facts replacement supports fingerprint compare-and-set', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-model-facts-cas-')); + try { + const owner = new ModelFactsDocumentOwner(); + const initial = await owner.readWithDiagnostics(root); + await owner.replace( + root, + { 'openai:o4-mini': { contextWindow: 100_000 } }, + initial.fingerprint, + ); + await assert.rejects( + () => + owner.replace(root, { 'openai:o4-mini': { contextWindow: 200_000 } }, initial.fingerprint), + (error: unknown) => + error instanceof RuntimePolicyStoreError && error.code === 'revision_conflict', + ); + assert.equal((await owner.read(root)).overrides['openai:o4-mini']?.contextWindow, 100_000); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/storage/src/__tests__/runtime-policy-model-facts.test.ts b/packages/storage/src/__tests__/runtime-policy-model-facts.test.ts new file mode 100644 index 0000000000..2904d1ddd3 --- /dev/null +++ b/packages/storage/src/__tests__/runtime-policy-model-facts.test.ts @@ -0,0 +1,368 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { RuntimePolicyCoordinator } from '../runtime-policy/coordinator.js'; +import { RuntimePolicyStoreError } from '../runtime-policy/errors.js'; +import { MODEL_FACTS_DOCUMENT_MAX_BYTES, ModelFactsDocumentOwner } from '../model-facts-store.js'; +import { writeJsonDocument } from '../runtime-policy/document-io.js'; + +test('runtime policy catalog overlays enabled custom model facts without changing the raw catalog', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-')); + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const created = await coordinator.createConnection({ + expectedCatalogRevision: 0, + connection: { + slug: 'custom-openai', + name: 'Custom OpenAI', + providerType: 'ollama', + enabled: true, + enabledModelIds: ['custom-model'], + }, + }); + assert.equal(created.kind, 'committed'); + assert.equal(Object.isFrozen(created), true); + if (created.kind === 'committed') assert.equal(Object.isFrozen(created.snapshot), true); + const replaced = await coordinator.replaceModelFacts({ + 'ollama:custom-model': { contextWindow: 64_000 }, + }); + assert.equal(Object.isFrozen(replaced), true); + assert.equal(Object.isFrozen(replaced.overrides), true); + const facts = await coordinator.getModelFacts(); + assert.equal(Object.isFrozen(facts), true); + assert.equal(Object.isFrozen(facts.document), true); + assert.equal(Object.isFrozen(facts.document.overrides), true); + const snapshot = await coordinator.getCatalogSnapshot(); + const model = snapshot.connections[0]?.models.find( + (candidate) => candidate.id === 'custom-model', + ); + assert.equal(model?.contextWindow, 64_000); + const prepared = await coordinator.beginConnectionTest( + snapshot.connections[0]!.connectionId, + null, + ); + assert.equal(prepared.kind, 'ready'); + if (prepared.kind === 'ready') { + const tested = await coordinator.completeConnectionTest(prepared.ticket, { + status: 'verified', + checkedAt: '2026-08-01T00:00:00.000Z', + }); + assert.equal(tested.kind, 'committed'); + } + assert.equal( + (await coordinator.getCatalogSnapshot()).connections[0]?.lastTest?.status, + 'verified', + ); + const staleTest = await coordinator.beginConnectionTest( + snapshot.connections[0]!.connectionId, + null, + ); + assert.equal(staleTest.kind, 'ready'); + await coordinator.replaceModelFacts({ + 'ollama:custom-model': { contextWindow: 65_000 }, + }); + if (staleTest.kind === 'ready') { + assert.deepEqual( + await coordinator.completeConnectionTest(staleTest.ticket, { + status: 'verified', + checkedAt: '2026-08-01T00:01:00.000Z', + }), + { kind: 'superseded', changed: ['connection'] }, + ); + } + assert.equal((await coordinator.getCatalogSnapshot()).connections[0]?.lastTest, undefined); + const restarted = new RuntimePolicyCoordinator((operation) => operation(root)); + const persisted = await restarted.getCatalogSnapshot(); + assert.equal( + persisted.connections[0]?.models.find((candidate) => candidate.id === 'custom-model') + ?.contextWindow, + 65_000, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('model fetch keeps an enabled facts-backed model outside provider inventory', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-refresh-')); + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const connectionId = await createTestConnection(coordinator); + await coordinator.replaceModelFacts({ + 'ollama:custom-model': { contextWindow: 64_000 }, + 'ollama:unselected-model': { contextWindow: 128_000 }, + }); + const beforeRefresh = await coordinator.getCatalogSnapshot(); + const defaulted = await coordinator.setDefaultTarget({ + expectedCatalogRevision: beforeRefresh.revision, + target: { connectionId, modelId: 'custom-model' }, + }); + assert.equal(defaulted.kind, 'committed'); + + const fetch = await coordinator.beginModelFetch(connectionId); + assert.equal(fetch.kind, 'ready'); + if (fetch.kind !== 'ready') return; + const refreshed = await coordinator.completeModelFetch(fetch.ticket, { + models: [{ id: 'live-model' }], + source: 'fetched', + fetchedAt: 1, + }); + assert.equal(refreshed.kind, 'committed'); + if (refreshed.kind !== 'committed') return; + + const raw = await ( + coordinator as unknown as { + catalog: { + read(root: string): Promise<{ + connections: readonly { models: readonly unknown[] }[]; + }>; + }; + } + ).catalog.read(root); + assert.deepEqual(raw.connections[0]?.models, [{ id: 'live-model' }]); + const projected = refreshed.snapshot.connections[0]; + assert.deepEqual(projected?.enabledModelIds, ['custom-model']); + assert.deepEqual(refreshed.snapshot.defaultTarget, { + connectionId, + modelId: 'custom-model', + }); + assert.equal( + projected?.models.find((model) => model.id === 'custom-model')?.contextWindow, + 64_000, + ); + assert.equal( + projected?.models.some((model) => model.id === 'unselected-model'), + false, + ); + + const execution = await coordinator.resolveExecutionConnection('custom-openai'); + assert.equal(execution.kind, 'ready'); + if (execution.kind === 'ready') { + assert.equal( + execution.connection.models?.some((model) => model.id === 'custom-model'), + true, + ); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('model facts are not persisted when verification invalidation fails', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-invalidation-')); + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const connectionId = await createTestConnection(coordinator); + const testTicket = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(testTicket.kind, 'ready'); + if (testTicket.kind === 'ready') { + assert.equal( + ( + await coordinator.completeConnectionTest( + testTicket.ticket, + verifiedAt('2026-08-01T00:00:00.000Z'), + ) + ).kind, + 'committed', + ); + } + const catalogOwner = ( + coordinator as unknown as { + catalog: { clearConnectionLastTest: () => Promise }; + } + ).catalog; + const original = catalogOwner.clearConnectionLastTest; + catalogOwner.clearConnectionLastTest = async () => { + throw new Error('injected invalidation failure'); + }; + try { + await assert.rejects( + () => coordinator.replaceModelFacts({ 'ollama:custom-model': { contextWindow: 64_000 } }), + (error: unknown) => + error instanceof Error && error.message === 'injected invalidation failure', + ); + } finally { + catalogOwner.clearConnectionLastTest = original; + } + assert.deepEqual((await coordinator.getModelFacts()).document.overrides, {}); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('oversized replacements preserve existing connection verification', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-oversized-')); + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const connectionId = await createTestConnection(coordinator); + const prepared = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(prepared.kind, 'ready'); + if (prepared.kind === 'ready') { + assert.equal( + ( + await coordinator.completeConnectionTest( + prepared.ticket, + verifiedAt('2026-08-01T00:00:00.000Z'), + ) + ).kind, + 'committed', + ); + } + + await assert.rejects( + () => coordinator.replaceModelFacts(oversizedOverrides()), + (error: unknown) => + error instanceof RuntimePolicyStoreError && error.code === 'invalid_policy_input', + ); + assert.equal( + (await coordinator.getCatalogSnapshot()).connections[0]?.lastTest?.status, + 'verified', + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('external model facts edits clear verification, supersede tickets, and warn on malformed input', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-external-edit-')); + const emitWarning = process.emitWarning; + const warnings: string[] = []; + process.emitWarning = ((warning: string | Error) => { + warnings.push(String(warning)); + }) as typeof process.emitWarning; + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const connectionId = await createTestConnection(coordinator); + await coordinator.replaceModelFacts({ 'ollama:custom-model': { contextWindow: 64_000 } }); + const verified = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(verified.kind, 'ready'); + if (verified.kind === 'ready') { + assert.equal( + ( + await coordinator.completeConnectionTest( + verified.ticket, + verifiedAt('2026-08-01T00:00:00.000Z'), + ) + ).kind, + 'committed', + ); + } + const ticket = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(ticket.kind, 'ready'); + await writeFile( + join(root, 'model-facts.json'), + JSON.stringify({ + schemaVersion: 1, + overrides: { 'ollama:custom-model': { contextWindow: 65_000 } }, + }), + 'utf8', + ); + if (ticket.kind === 'ready') { + assert.deepEqual( + await coordinator.completeConnectionTest( + ticket.ticket, + verifiedAt('2026-08-01T00:01:00.000Z'), + ), + { kind: 'superseded', changed: ['connection'] }, + ); + } + assert.equal((await coordinator.getCatalogSnapshot()).connections[0]?.lastTest, undefined); + + await writeFile(join(root, 'model-facts.json'), '{not-json}', 'utf8'); + const snapshot = await coordinator.getCatalogSnapshot(); + assert.equal( + snapshot.connections[0]?.models.find((model) => model.id === 'custom-model')?.contextWindow, + undefined, + ); + assert.equal( + warnings.some((warning) => warning.includes('model-facts.json')), + true, + ); + } finally { + process.emitWarning = emitWarning; + await rm(root, { recursive: true, force: true }); + } +}); + +test('a post-publication model facts failure supersedes existing connection-test tickets', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-unknown-write-')); + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const connectionId = await createTestConnection(coordinator); + await coordinator.replaceModelFacts({ 'ollama:custom-model': { contextWindow: 64_000 } }); + const ticket = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(ticket.kind, 'ready'); + const owner = (coordinator as unknown as { modelFacts: ModelFactsDocumentOwner }).modelFacts; + const original = owner.writeReplacement; + owner.writeReplacement = async (writeRoot, document) => { + await writeJsonDocument( + writeRoot, + 'model-facts.json', + document, + MODEL_FACTS_DOCUMENT_MAX_BYTES, + async () => { + throw new Error('injected directory sync failure'); + }, + ); + return document; + }; + try { + await assert.rejects( + () => coordinator.replaceModelFacts({ 'ollama:custom-model': { contextWindow: 65_000 } }), + (error: unknown) => + error instanceof RuntimePolicyStoreError && error.code === 'commit_outcome_unknown', + ); + } finally { + owner.writeReplacement = original; + } + if (ticket.kind === 'ready') { + assert.deepEqual( + await coordinator.completeConnectionTest( + ticket.ticket, + verifiedAt('2026-08-01T00:01:00.000Z'), + ), + { kind: 'superseded', changed: ['connection'] }, + ); + } + const restarted = new RuntimePolicyCoordinator((operation) => operation(root)); + assert.equal( + (await restarted.getCatalogSnapshot()).connections[0]?.models.find( + (model) => model.id === 'custom-model', + )?.contextWindow, + 65_000, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +async function createTestConnection(coordinator: RuntimePolicyCoordinator): Promise { + const created = await coordinator.createConnection({ + expectedCatalogRevision: 0, + connection: { + slug: 'custom-openai', + name: 'Custom OpenAI', + providerType: 'ollama', + enabled: true, + enabledModelIds: ['custom-model'], + }, + }); + assert.equal(created.kind, 'committed'); + if (created.kind !== 'committed') throw new Error('Expected connection creation to commit'); + return created.snapshot.connections[0]!.connectionId; +} + +function verifiedAt(checkedAt: string) { + return { status: 'verified' as const, checkedAt }; +} + +function oversizedOverrides() { + return Object.fromEntries( + Array.from({ length: 512 }, (_, index) => [ + `ollama:custom-model-${index}`, + { description: 'x'.repeat(2_048) }, + ]), + ); +} diff --git a/packages/storage/src/__tests__/runtime-policy-stores.test.ts b/packages/storage/src/__tests__/runtime-policy-stores.test.ts index 64cc2a356d..2b95a77f44 100644 --- a/packages/storage/src/__tests__/runtime-policy-stores.test.ts +++ b/packages/storage/src/__tests__/runtime-policy-stores.test.ts @@ -953,6 +953,7 @@ describe('runtime policy stores', () => { ); const fetch = await stores.operations.beginModelFetch(connection.connectionId); + await stores.modelFacts.replace({ 'openai:gpt-5': { apiProtocol: 'openai-responses' } }); const testTicket = await stores.operations.beginConnectionTest( connection.connectionId, 'gpt-5', @@ -961,6 +962,7 @@ describe('runtime policy stores', () => { assert.equal(testTicket.kind, 'ready'); if (fetch.kind !== 'ready' || testTicket.kind !== 'ready') return; assert.equal(testTicket.modelId, 'gpt-5'); + assert.equal(testTicket.connection.models?.[0]?.apiProtocol, 'openai-responses'); assert.equal(fetch.secretMaterial.connection?.secret, 'effect-secret'); await assert.rejects( @@ -995,8 +997,16 @@ describe('runtime policy stores', () => { if (discovered.kind !== 'committed') return; const afterDiscovery = discovered.snapshot.connections[0]; assert.ok(afterDiscovery); - assert.deepEqual(afterDiscovery.models, [{ id: 'gpt-5.1' }, { id: 'gpt-5.2' }]); - assert.deepEqual(afterDiscovery.enabledModelIds, ['gpt-5.1']); + assert.deepEqual(afterDiscovery.models, [ + { id: 'gpt-5.1' }, + { id: 'gpt-5.2' }, + { + id: 'gpt-5', + apiProtocol: 'openai-responses', + factOverriddenFields: ['apiProtocol'], + }, + ]); + assert.deepEqual(afterDiscovery.enabledModelIds, ['gpt-5']); assert.equal(afterDiscovery.modelSource, 'fetched'); assert.equal(afterDiscovery.modelsFetchedAt, 42); diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 1ff5b9a193..a9fcf0ae08 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -140,6 +140,7 @@ export type { } from './operational-state-store.js'; export * from './operational-state-backup.js'; export * from './mcp-config-store.js'; +export * from './model-facts-store.js'; export * from './workspace-identity.js'; export * from './memory-bundle-store.js'; export * from './long-term-memory-store.js'; diff --git a/packages/storage/src/model-facts-store.ts b/packages/storage/src/model-facts-store.ts new file mode 100644 index 0000000000..e2bce008cd --- /dev/null +++ b/packages/storage/src/model-facts-store.ts @@ -0,0 +1,159 @@ +import { createHash } from 'node:crypto'; +import { + decodeModelFactsDocument, + MODEL_FACTS_MAX_OVERRIDES, + MODEL_FACTS_SCHEMA_VERSION, + normalizeModelFactOverride, + UnsupportedModelFactsSchemaError, + type ModelFactOverrides, + type ModelFactsDocument, +} from '@maka/core/model-facts'; +import { RuntimePolicyStoreError } from './runtime-policy/errors.js'; +import type { ConnectionCatalogEntry } from '@maka/core/runtime-policy'; +import { + readBoundedDocumentBytes, + serializeJsonDocument, + writeJsonDocument, +} from './runtime-policy/document-io.js'; + +export const MODEL_FACTS_DOCUMENT_MAX_BYTES = 256 * 1024; +const FILE = 'model-facts.json'; + +export interface ModelFactsReadResult { + readonly document: ModelFactsDocument; + readonly diagnostic?: 'malformed' | 'oversized' | 'unsupported_schema'; + readonly fingerprint: string; +} + +export class ModelFactsDocumentOwner { + async read(root: string): Promise { + return (await this.readWithDiagnostics(root)).document; + } + + async readWithDiagnostics(root: string): Promise { + let bytes: Buffer | undefined; + try { + bytes = await readBoundedDocumentBytes(root, FILE, MODEL_FACTS_DOCUMENT_MAX_BYTES); + } catch (error) { + if (error instanceof RuntimePolicyStoreError && error.code === 'invalid_document') { + return { + document: emptyDocument(), + diagnostic: error.message.includes('exceeds') ? 'oversized' : 'malformed', + fingerprint: `invalid:${error.message}`, + }; + } + throw error; + } + if (bytes === undefined) return { document: emptyDocument(), fingerprint: 'missing' }; + const fingerprint = fingerprintBytes(bytes); + let value: unknown; + try { + value = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) as unknown; + return { document: decodeModelFactsDocument(value), fingerprint }; + } catch (error) { + if (error instanceof UnsupportedModelFactsSchemaError) { + return { document: emptyDocument(), diagnostic: 'unsupported_schema', fingerprint }; + } + return { document: emptyDocument(), diagnostic: 'malformed', fingerprint }; + } + } + + async replace( + root: string, + overrides: ModelFactOverrides, + expectedFingerprint?: string, + ): Promise { + const validated = this.prepareReplacement(overrides); + return this.writeReplacement(root, validated, expectedFingerprint); + } + + prepareReplacement(overrides: ModelFactOverrides): ModelFactsDocument { + const entries = Object.entries(overrides); + if (entries.length > MODEL_FACTS_MAX_OVERRIDES) { + throw new RuntimePolicyStoreError('invalid_policy_input', 'Too many model fact overrides'); + } + let validated: ModelFactsDocument; + try { + const normalized: Record< + string, + ReturnType + > = Object.create(null); + for (const [key, value] of entries) normalized[key] = normalizeModelFactOverride(value); + const document: ModelFactsDocument = { + schemaVersion: MODEL_FACTS_SCHEMA_VERSION, + overrides: normalized, + }; + // Reuse the canonical decoder for key grammar and exact bounded shape. + validated = decodeModelFactsDocument(document); + if (serializeJsonDocument(validated).length > MODEL_FACTS_DOCUMENT_MAX_BYTES) { + throw new RuntimePolicyStoreError( + 'invalid_policy_input', + `model-facts.json exceeds its ${MODEL_FACTS_DOCUMENT_MAX_BYTES} byte limit`, + ); + } + } catch (error) { + if (error instanceof RuntimePolicyStoreError) throw error; + throw new RuntimePolicyStoreError( + 'invalid_policy_input', + error instanceof Error ? error.message : 'Invalid model fact overrides', + { cause: error }, + ); + } + return validated; + } + + async writeReplacement( + root: string, + document: ModelFactsDocument, + expectedFingerprint?: string, + ): Promise { + const current = await this.readWithDiagnostics(root); + if (current.diagnostic === 'unsupported_schema') { + throw new RuntimePolicyStoreError( + 'invalid_policy_input', + 'model-facts.json uses a newer unsupported schema and cannot be overwritten', + ); + } + if (expectedFingerprint !== undefined && current.fingerprint !== expectedFingerprint) { + throw new RuntimePolicyStoreError( + 'revision_conflict', + 'model-facts.json changed before replacement; reload before retrying', + ); + } + await writeJsonDocument(root, FILE, document, MODEL_FACTS_DOCUMENT_MAX_BYTES); + return document; + } + + fingerprint(document: ModelFactsDocument): string { + return fingerprintBytes(serializeJsonDocument(document)); + } + + fingerprintForConnection( + document: ModelFactsDocument, + connection: Pick, + ): string { + const modelIds = new Set([ + ...(connection.models ?? []).map((model) => model.id), + ...connection.enabledModelIds, + ]); + const entries = Object.entries(document.overrides) + .filter(([key]) => { + const separator = key.indexOf(':'); + return ( + separator > 0 && + key.slice(0, separator) === connection.providerType && + modelIds.has(key.slice(separator + 1)) + ); + }) + .sort(([left], [right]) => left.localeCompare(right)); + return fingerprintBytes(Buffer.from(JSON.stringify(entries), 'utf8')); + } +} + +function emptyDocument(): ModelFactsDocument { + return { schemaVersion: MODEL_FACTS_SCHEMA_VERSION, overrides: {} }; +} + +function fingerprintBytes(bytes: Buffer): string { + return createHash('sha256').update(bytes).digest('hex'); +} diff --git a/packages/storage/src/runtime-policy-stores.ts b/packages/storage/src/runtime-policy-stores.ts index 7eb93556e4..2447327fc4 100644 --- a/packages/storage/src/runtime-policy-stores.ts +++ b/packages/storage/src/runtime-policy-stores.ts @@ -14,6 +14,8 @@ import type { SetDefaultConnectionTargetInput, UpdateCatalogConnectionInput, } from '@maka/core/runtime-policy'; +import type { ModelFactOverrides, ModelFactsDocument } from '@maka/core/model-facts'; +import type { ModelFactsReadResult } from './model-facts-store.js'; import { assertStorageRootLease, runWithStorageRootLease, @@ -89,6 +91,14 @@ export interface ConnectionCatalogWriter extends ConnectionCatalogReader { ): Promise; } +export interface ModelFactsReader { + get(): Promise; +} + +export interface ModelFactsWriter extends ModelFactsReader { + replace(overrides: ModelFactOverrides, expectedFingerprint?: string): Promise; +} + export interface CredentialVaultReader { getSnapshot(): Promise; getStatus(locator: CredentialLocator): Promise; @@ -105,6 +115,7 @@ export interface RuntimePolicyStoresReader { readonly [readerBrand]: true; readonly runtimePolicy: Readonly; readonly connectionCatalog: Readonly; + readonly modelFacts: Readonly; readonly credentialVault: Readonly; } @@ -114,6 +125,7 @@ export interface RuntimePolicyStoresWriter { readonly [writerBrand]: true; readonly runtimePolicy: Readonly; readonly connectionCatalog: Readonly; + readonly modelFacts: Readonly; readonly credentialVault: Readonly; readonly operations: Readonly; } @@ -145,6 +157,7 @@ export async function openInteractiveRuntimePolicyStoresForRead( [readerBrand]: true, runtimePolicy: { getSnapshot: () => coordinator.getPolicySnapshot() }, connectionCatalog: { getSnapshot: () => coordinator.getCatalogSnapshot() }, + modelFacts: { get: () => coordinator.getModelFacts() }, credentialVault: { getSnapshot: () => coordinator.getVaultSnapshot(), getStatus: (locator) => coordinator.getCredentialStatus(locator), @@ -201,6 +214,11 @@ function createWriterFacade(coordinator: RuntimePolicyCoordinator): RuntimePolic remove: (input) => coordinator.removeConnection(input), setDefaultTarget: (input) => coordinator.setDefaultTarget(input), }, + modelFacts: { + get: () => coordinator.getModelFacts(), + replace: (overrides, expectedFingerprint) => + coordinator.replaceModelFacts(overrides, expectedFingerprint), + }, credentialVault: { getSnapshot: () => coordinator.getVaultSnapshot(), getStatus: (locator) => coordinator.getCredentialStatus(locator), @@ -247,11 +265,13 @@ function invalidFacade(access: 'read' | 'write'): StorageRootAuthorityError { function freezeFacade(stores: { readonly runtimePolicy: object; readonly connectionCatalog: object; + readonly modelFacts: object; readonly credentialVault: object; readonly operations?: object; }): void { Object.freeze(stores.runtimePolicy); Object.freeze(stores.connectionCatalog); + Object.freeze(stores.modelFacts); Object.freeze(stores.credentialVault); if (stores.operations) Object.freeze(stores.operations); Object.freeze(stores); diff --git a/packages/storage/src/runtime-policy/connection-catalog-document.ts b/packages/storage/src/runtime-policy/connection-catalog-document.ts index 07dd10b0b0..bf42aee372 100644 --- a/packages/storage/src/runtime-policy/connection-catalog-document.ts +++ b/packages/storage/src/runtime-policy/connection-catalog-document.ts @@ -301,6 +301,9 @@ export class ConnectionCatalogDocumentOwner { current: ConnectionCatalogDocument, expected: ConnectionVersionBasis, rawResult: ConnectionModelDiscoveryResult, + options?: { + readonly factBackedModelIds?: ReadonlySet; + }, ): Promise { const result = decodeConnectionInput(() => normalizeConnectionModelDiscoveryResult(rawResult)); if (result.models.length === 0) { @@ -328,7 +331,10 @@ export class ConnectionCatalogDocumentOwner { hasModelInventory: previous.models.length > 0, }, result.models, - { aliases: modelIdAliasesForProvider(previous.providerType) }, + { + aliases: modelIdAliasesForProvider(previous.providerType), + factBackedModelIds: options?.factBackedModelIds, + }, ) : { defaultModel: currentDefaultTarget?.modelId ?? previous.enabledModelIds[0] ?? '', @@ -497,6 +503,7 @@ export class ConnectionCatalogDocumentOwner { current: ConnectionCatalogDocument, expected: ConnectionVersionBasis, rawResult: ConnectionTestSummary, + modelFactsFingerprint: string, ): Promise { const result = decodeConnectionInput(() => decodeConnectionTestSummary(rawResult)); const index = findConnectionIndex(current, expected); @@ -508,6 +515,7 @@ export class ConnectionCatalogDocumentOwner { ...previous, revision: nextRevision(previous.revision), lastTest: result, + lastTestModelFactsFingerprint: modelFactsFingerprint, }); } @@ -522,7 +530,11 @@ export class ConnectionCatalogDocumentOwner { throw codecError('invalid_document', 'Coordinator admitted an unknown connection'); } if (previous.lastTest === undefined) return false; - const { lastTest: _lastTest, ...withoutLastTest } = previous; + const { + lastTest: _lastTest, + lastTestModelFactsFingerprint: _lastTestModelFactsFingerprint, + ...withoutLastTest + } = previous; await this.writePatchedResult(root, current, index, { ...withoutLastTest, revision: nextRevision(previous.revision), @@ -537,7 +549,11 @@ export class ConnectionCatalogDocumentOwner { if (current.connections.every((connection) => connection.lastTest === undefined)) return false; const connections = current.connections.map((connection) => { if (connection.lastTest === undefined) return connection; - const { lastTest: _lastTest, ...withoutLastTest } = connection; + const { + lastTest: _lastTest, + lastTestModelFactsFingerprint: _lastTestModelFactsFingerprint, + ...withoutLastTest + } = connection; return { ...withoutLastTest, revision: nextRevision(connection.revision), diff --git a/packages/storage/src/runtime-policy/coordinator.ts b/packages/storage/src/runtime-policy/coordinator.ts index b63bcc7bf1..10d76d03c2 100644 --- a/packages/storage/src/runtime-policy/coordinator.ts +++ b/packages/storage/src/runtime-policy/coordinator.ts @@ -32,6 +32,13 @@ import { type SetDefaultConnectionTargetInput, type UpdateCatalogConnectionInput, } from '@maka/core/runtime-policy'; +import { + applyModelFactOverridesToConnection, + applyModelFactOverridesToCatalogSnapshot, + modelFactOverrideIdsForProvider, + type ModelFactOverrides, + type ModelFactsDocument, +} from '@maka/core/model-facts'; import { deriveProviderAuthContract, type ProviderAuthAction } from '@maka/core/provider-auth'; import { deriveConnectionSlug, @@ -103,6 +110,7 @@ import { } from './onboarding-transaction.js'; import { policySnapshot, RuntimePolicyDocumentOwner } from './policy-document.js'; import { SerializedOperationLane } from '../serialized-operation-lane.js'; +import { ModelFactsDocumentOwner } from '../model-facts-store.js'; type RootExecutor = (operation: (root: string) => Promise) => Promise; @@ -152,6 +160,7 @@ type SemanticConnectionBasis = readonly kind: 'connection_test'; readonly requestBodyOverlayJson: string; readonly model: ConnectionTestModelBasis; + readonly modelFactsFingerprint: string; }); interface ConnectionTicketRecord { @@ -175,6 +184,8 @@ export class RuntimePolicyCoordinator { private readonly policy = new RuntimePolicyDocumentOwner(); private readonly catalog = new ConnectionCatalogDocumentOwner(); private readonly vault = new CredentialVaultDocumentOwner(); + private readonly modelFacts = new ModelFactsDocumentOwner(); + private warnedModelFactsFingerprint: string | undefined; private readonly tickets = new WeakMap(); private onboardingRecoveryRequired = false; @@ -201,7 +212,60 @@ export class RuntimePolicyCoordinator { } getCatalogSnapshot() { - return this.inLane(async (root) => catalogSnapshot(await this.catalog.read(root))); + return this.inLane(async (root) => this.projectCatalogSnapshot(root)); + } + + getModelFacts() { + return this.inLane(async (root) => deepFreeze(await this.readModelFacts(root))); + } + + replaceModelFacts(overrides: ModelFactOverrides, expectedFingerprint?: string) { + return this.inLane(async (root) => { + const document = this.modelFacts.prepareReplacement(overrides); + const currentFacts = await this.modelFacts.readWithDiagnostics(root); + if (currentFacts.diagnostic === 'unsupported_schema') { + throw new RuntimePolicyStoreError( + 'invalid_policy_input', + 'model-facts.json uses a newer unsupported schema and cannot be overwritten', + ); + } + if (expectedFingerprint !== undefined && currentFacts.fingerprint !== expectedFingerprint) { + throw new RuntimePolicyStoreError( + 'revision_conflict', + 'model-facts.json changed before replacement; reload before retrying', + ); + } + const currentCatalog = await this.catalog.read(root); + const affected = currentCatalog.connections.filter( + (connection) => + connection.lastTest !== undefined && + this.modelFacts.fingerprintForConnection(currentFacts.document, connection) !== + this.modelFacts.fingerprintForConnection(document, connection), + ); + let cleared = false; + try { + for (const connection of affected) { + const latest = await this.catalog.read(root); + cleared = + (await this.catalog.clearConnectionLastTest(root, latest, connection.connectionId)) || + cleared; + } + const persisted = await this.modelFacts.writeReplacement( + root, + document, + currentFacts.fingerprint, + ); + return deepFreeze(persisted); + } catch (error) { + if (cleared) { + throw commitOutcomeUnknown( + 'Connection verification was cleared before model facts replacement completed', + error, + ); + } + throw error; + } + }); } getVaultSnapshot() { @@ -249,11 +313,15 @@ export class RuntimePolicyCoordinator { } createConnection(input: CreateCatalogConnectionInput) { - return this.inLane((root) => this.catalog.create(root, input)); + return this.inLane(async (root) => + this.projectCatalogMutation(root, await this.catalog.create(root, input)), + ); } updateConnection(input: UpdateCatalogConnectionInput) { - return this.inLane((root) => this.catalog.update(root, input)); + return this.inLane(async (root) => + this.projectCatalogMutation(root, await this.catalog.update(root, input)), + ); } removeConnection(rawInput: RemoveCatalogConnectionInput) { @@ -274,7 +342,10 @@ export class RuntimePolicyCoordinator { const vault = await this.vault.read(root); if (!connection) { await this.vault.deleteConnectionCredentials(root, vault, expected.connectionId); - return deepFreeze({ kind: 'committed' as const, snapshot: catalogSnapshot(catalog) }); + return deepFreeze({ + kind: 'committed' as const, + snapshot: await this.projectCatalogSnapshot(root), + }); } const result = await this.catalog.remove(root, { expected }); if (result.kind === 'committed') { @@ -287,12 +358,14 @@ export class RuntimePolicyCoordinator { ); } } - return result; + return this.projectCatalogMutation(root, result); }); } setDefaultTarget(input: SetDefaultConnectionTargetInput) { - return this.inLane((root) => this.catalog.setDefaultTarget(root, input)); + return this.inLane(async (root) => + this.projectCatalogMutation(root, await this.catalog.setDefaultTarget(root, input)), + ); } setCredential(rawInput: SetCredentialInput) { @@ -581,7 +654,10 @@ export class RuntimePolicyCoordinator { if (prepared.kind !== 'ready') return prepared; return deepFreeze({ kind: 'ready' as const, - connection: structuredClone(connection), + connection: applyModelFactOverridesToConnection( + structuredClone(connection), + (await this.readModelFacts(root)).document.overrides, + ), secretMaterial: prepared.secretMaterial, networkProxy: structuredClone(prepared.networkProxy), }); @@ -868,18 +944,34 @@ export class RuntimePolicyCoordinator { const claimed = this.claimTicket(ticket, 'model_fetch'); return this.completeClaimedTicket(claimed, () => this.inLane(async (root) => { + const facts = await this.readModelFacts(root); const catalog = await this.catalog.read(root); const checked = await this.checkSemanticConnectionBasis(root, catalog, claimed.basis); if (checked.changed.length > 0 || !checked.connection) { return deepFreeze({ kind: 'superseded' as const, changed: checked.changed }); } + const selectedModelIds = new Set(checked.connection.enabledModelIds); + if (catalog.defaultTarget?.connectionId === checked.connection.connectionId) { + selectedModelIds.add(catalog.defaultTarget.modelId); + } const snapshot = await this.catalog.writeModelFetchResult( root, catalog, connectionBasis(checked.connection), result, + { + factBackedModelIds: new Set( + modelFactOverrideIdsForProvider( + facts.document.overrides, + checked.connection.providerType, + ).filter((modelId) => selectedModelIds.has(modelId)), + ), + }, ); - return deepFreeze({ kind: 'committed' as const, snapshot }); + return deepFreeze({ + kind: 'committed' as const, + snapshot: await this.projectCatalogSnapshot(root), + }); }), ); } @@ -947,7 +1039,11 @@ export class RuntimePolicyCoordinator { const result = await this.applyConnectionOnboarding(root, intent); await clearConnectionOnboardingIntent(root); this.onboardingRecoveryRequired = false; - return deepFreeze({ kind: 'committed' as const, ...result }); + return deepFreeze({ + kind: 'committed' as const, + ...result, + snapshot: await this.projectCatalogSnapshot(root), + }); } catch (error) { this.onboardingRecoveryRequired = true; if (isCommitOutcomeUnknown(error)) throw error; @@ -973,21 +1069,33 @@ export class RuntimePolicyCoordinator { 'test_credentials', ); if (prepared.kind !== 'ready') return prepared; + const facts = await this.readModelFacts(root); + const projectedConnection = applyModelFactOverridesToConnection( + structuredClone(prepared.connection), + facts.document.overrides, + ); + const projected = { ...prepared, connection: projectedConnection }; const modelId = rawModelId === null ? null : decodeConnectionInput(() => decodeConnectionModelId(rawModelId)); - if (modelId !== null && !isCanonicalConnectionTestModel(prepared.connection, modelId)) { + if (modelId !== null && !isCanonicalConnectionTestModel(projectedConnection, modelId)) { throw codecError( 'invalid_connection_input', 'Connection test model is not in the canonical model set', ); } - const ticket = this.issueTicket('connection_test', connectionTestSemanticBasis(prepared)); + const ticket = this.issueTicket( + 'connection_test', + connectionTestSemanticBasis( + projected, + this.modelFacts.fingerprintForConnection(facts.document, prepared.connection), + ), + ); return deepFreeze({ kind: 'ready' as const, ticket: ticket as ConnectionTestTicket, - connection: structuredClone(prepared.connection), + connection: projectedConnection, modelId, secretMaterial: prepared.secretMaterial, networkProxy: structuredClone(prepared.networkProxy), @@ -1002,6 +1110,9 @@ export class RuntimePolicyCoordinator { const claimed = this.claimTicket(ticket, 'connection_test'); return this.completeClaimedTicket(claimed, () => this.inLane(async (root) => { + if (claimed.basis.kind !== 'connection_test') { + throw new Error('Coordinator admitted a non-connection-test ticket'); + } const catalog = await this.catalog.read(root); const checked = await this.checkSemanticConnectionBasis(root, catalog, claimed.basis); if (checked.changed.length > 0 || !checked.connection) { @@ -1012,8 +1123,12 @@ export class RuntimePolicyCoordinator { catalog, connectionBasis(checked.connection), result, + claimed.basis.modelFactsFingerprint, ); - return deepFreeze({ kind: 'committed' as const, snapshot }); + return deepFreeze({ + kind: 'committed' as const, + snapshot: await this.projectCatalogSnapshot(root), + }); }), ); } @@ -1157,17 +1272,31 @@ export class RuntimePolicyCoordinator { }> { const connection = findConnection(catalog, { connectionId: basis.connectionId }); const changed: ConnectionEffectChangedDomain[] = []; + const facts = basis.kind === 'connection_test' ? await this.readModelFacts(root) : undefined; + if ( + basis.kind === 'connection_test' && + (!connection || + this.modelFacts.fingerprintForConnection(facts!.document, connection) !== + basis.modelFactsFingerprint) + ) { + changed.push('connection'); + } + const effectiveConnection = + connection && basis.kind === 'connection_test' + ? applyModelFactOverridesToConnection(connection, facts!.document.overrides) + : connection; if ( - !connection || - connection.providerType !== basis.providerType || - !connection.enabled || - canonicalEffectiveEndpoint(connection) !== basis.effectiveEndpoint || + !effectiveConnection || + effectiveConnection.providerType !== basis.providerType || + !effectiveConnection.enabled || + canonicalEffectiveEndpoint(effectiveConnection) !== basis.effectiveEndpoint || (basis.kind === 'model_fetch' && - !sameStringArray(connection.enabledModelIds, basis.enabledModelIds)) || + !sameStringArray(effectiveConnection.enabledModelIds, basis.enabledModelIds)) || (basis.kind === 'connection_test' && - JSON.stringify(connection.requestBodyOverlay ?? {}) !== basis.requestBodyOverlayJson) || + JSON.stringify(effectiveConnection.requestBodyOverlay ?? {}) !== + basis.requestBodyOverlayJson) || (basis.kind === 'connection_test' && - !sameConnectionTestModelBasis(connectionTestModelBasis(connection), basis.model)) + !sameConnectionTestModelBasis(connectionTestModelBasis(effectiveConnection), basis.model)) ) { changed.push('connection'); } @@ -1343,6 +1472,77 @@ export class RuntimePolicyCoordinator { return operation(root); }); } + + private async projectCatalogSnapshot(root: string): Promise { + const facts = await this.readModelFacts(root); + const snapshot = catalogSnapshot(await this.catalog.read(root)); + return deepFreeze( + hideStaleModelFactsVerification( + applyModelFactOverridesToCatalogSnapshot(snapshot, facts.document.overrides), + snapshot, + facts.document, + this.modelFacts, + ), + ); + } + + private async readModelFacts(root: string) { + const facts = await this.modelFacts.readWithDiagnostics(root); + if (facts.diagnostic !== undefined && this.warnedModelFactsFingerprint !== facts.fingerprint) { + process.emitWarning(`model-facts.json is ${facts.diagnostic}; ignoring its overrides`, { + type: 'RuntimePolicyWarning', + }); + this.warnedModelFactsFingerprint = facts.fingerprint; + } + return facts; + } + + private async projectCatalogMutation( + root: string, + result: T, + ): Promise { + if (result.kind !== 'committed' || !('snapshot' in result)) return result; + return deepFreeze({ + ...result, + snapshot: await this.projectCatalogSnapshot(root), + }) as T; + } +} + +function hideStaleModelFactsVerification( + projected: ConnectionCatalogSnapshot, + persisted: ConnectionCatalogSnapshot, + document: ModelFactsDocument, + owner: ModelFactsDocumentOwner, +): ConnectionCatalogSnapshot { + const persistedById = new Map( + persisted.connections.map((connection) => [connection.connectionId, connection] as const), + ); + return { + ...projected, + connections: projected.connections.map((connection) => { + if (connection.lastTest === undefined) return connection; + const raw = persistedById.get(connection.connectionId); + if (!raw) return connection; + const current = owner.fingerprintForConnection(document, raw); + // Catalogs written before model facts existed have no marker. They remain + // valid until facts actually exist; every test recorded by this feature + // carries a connection-scoped marker and is checked exactly. + if ( + raw.lastTestModelFactsFingerprint === current || + (raw.lastTestModelFactsFingerprint === undefined && + Object.keys(document.overrides).length === 0) + ) { + return connection; + } + const { + lastTest: _lastTest, + lastTestModelFactsFingerprint: _lastTestModelFactsFingerprint, + ...withoutLastTest + } = connection; + return withoutLastTest; + }), + }; } function isCommitOutcomeUnknown(error: unknown): error is RuntimePolicyStoreError { @@ -1376,12 +1576,14 @@ function modelFetchSemanticBasis( function connectionTestSemanticBasis( prepared: PreparedConnectionMaterial, + modelFactsFingerprint: string, ): Extract { return { kind: 'connection_test', ...commonSemanticConnectionBasis(prepared), requestBodyOverlayJson: JSON.stringify(prepared.connection.requestBodyOverlay ?? {}), model: connectionTestModelBasis(prepared.connection), + modelFactsFingerprint, }; } diff --git a/packages/storage/src/runtime-policy/document-io.ts b/packages/storage/src/runtime-policy/document-io.ts index 5632836253..372af9cc23 100644 --- a/packages/storage/src/runtime-policy/document-io.ts +++ b/packages/storage/src/runtime-policy/document-io.ts @@ -16,7 +16,7 @@ export const VAULT_DOCUMENT_MAX_BYTES = 2 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const RUNTIME_POLICY_TEMP_PATTERN = - /^(?:runtime-policy|connection-catalog|credential-vault|runtime-policy-onboarding)\.json\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.tmp$/; + /^(?:runtime-policy|connection-catalog|credential-vault|runtime-policy-onboarding|model-facts)\.json\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.tmp$/; export async function cleanupRuntimePolicyDocumentTemps(root: string): Promise { let failure: unknown; @@ -65,6 +65,26 @@ export async function readBoundedJsonDocument( file: string, maxBytes: number, ): Promise { + const bytes = await readBoundedDocumentBytes(root, file, maxBytes); + if (bytes === undefined) return undefined; + let text: string; + try { + text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch (error) { + throw invalidDocument(`${file} is not valid UTF-8`, error); + } + try { + return JSON.parse(text) as unknown; + } catch (error) { + throw invalidDocument(`${file} is not valid JSON`, error); + } +} + +export async function readBoundedDocumentBytes( + root: string, + file: string, + maxBytes: number, +): Promise { const path = join(root, file); const flags = process.platform === 'win32' @@ -81,7 +101,7 @@ export async function readBoundedJsonDocument( throw ioFailed(`${file} could not be opened`, error); } - let result: unknown | undefined; + let result: Buffer | undefined; let failure: unknown; try { const metadata = await handle.stat(); @@ -102,17 +122,7 @@ export async function readBoundedJsonDocument( } if (total > maxBytes) throw invalidDocument(`${file} exceeds its ${maxBytes} byte limit`); - let text: string; - try { - text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks, total)); - } catch (error) { - throw invalidDocument(`${file} is not valid UTF-8`, error); - } - try { - result = JSON.parse(text) as unknown; - } catch (error) { - throw invalidDocument(`${file} is not valid JSON`, error); - } + result = Buffer.concat(chunks, total); } catch (error) { failure = error; } finally { @@ -135,6 +145,7 @@ export async function writeJsonDocument( file: string, value: unknown, maxBytes: number, + synchronizeDirectory: (root: string) => Promise = syncDirectory, ): Promise { const bytes = serializeJsonDocument(value); if (bytes.length > maxBytes) throw invalidDocument(`${file} exceeds its ${maxBytes} byte limit`); @@ -154,7 +165,7 @@ export async function writeJsonDocument( handle = undefined; await rename(temporaryPath, path); published = true; - await syncDirectory(root); + await synchronizeDirectory(root); } catch (error) { failure = error; } finally { diff --git a/packages/storage/src/runtime-policy/errors.ts b/packages/storage/src/runtime-policy/errors.ts index dd8caf95cf..67c50fb3c9 100644 --- a/packages/storage/src/runtime-policy/errors.ts +++ b/packages/storage/src/runtime-policy/errors.ts @@ -5,6 +5,7 @@ export type RuntimePolicyStoreErrorCode = | 'invalid_policy_input' | 'invalid_connection_input' | 'invalid_credential_input' + | 'revision_conflict' | 'io_failed' | 'commit_outcome_unknown';