diff --git a/src/routing/capability.ts b/src/routing/capability.ts index a27713c1e4..7c11b2ae16 100644 --- a/src/routing/capability.ts +++ b/src/routing/capability.ts @@ -2,8 +2,9 @@ * Candidate capability evidence for policy routing (RI-05). * * Evidence comes from canonical local sources only - provider config maps, - * the provider registry, the cached Codex catalog file, and the native-model - * metadata helpers. No live network fetch happens at routing time. + * explicit custom-model metadata, the provider registry, and native-model + * metadata helpers. No live network fetch or generated-catalog inference + * happens at routing time. * * "Unknown is not zero": any dimension without canonical evidence stays * `undefined` (unknown) and the profile's `unknownEvidence` policy decides @@ -11,69 +12,18 @@ */ import type { CodexCommanderConfig } from "../types"; +import { modelInList } from "../types"; import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; -import { PROVIDER_REGISTRY } from "../providers/registry"; +import { PROVIDER_REGISTRY, providerMatchesRegistryTransport } from "../providers/registry"; +import { modelRecordValue, sanitizeCodexReasoningEfforts } from "../reasoning-effort"; +import { applyProviderContextCap, providerContextCap } from "../providers/context-cap"; import { nativeInputModalities, nativeOpenAiContextWindow, - nativeParallelToolCalls, nativeReasoningEfforts, } from "../codex/catalog/metadata"; -import { readCatalog, readCodexCatalogPath } from "../codex/catalog/parsing"; -import { statSync } from "node:fs"; import type { RouteCapabilityEvidence } from "./trace"; -type CatalogModelRow = { - provider: string; - id: string; - contextWindow?: number; - inputModalities?: string[]; - reasoningEfforts?: string[]; - capabilities?: string[]; -}; - -/** - * Catalog rows memoized by path + mtime: the cached Codex catalog is stable - * between refreshes, and re-reading/parsing the whole file per candidate on - * the request path would multiply a synchronous disk + JSON cost by the - * profile candidate count for every policy-routed request. - */ -let catalogCache: { path: string; mtimeMs: number; rows: CatalogModelRow[] } | null = null; - -function cachedCatalogModels(): CatalogModelRow[] { - try { - const path = readCodexCatalogPath(); - const mtimeMs = statSync(path).mtimeMs; - if (catalogCache && catalogCache.path === path && catalogCache.mtimeMs === mtimeMs) { - return catalogCache.rows; - } - const catalog = readCatalog(path); - const models = catalog?.models; - if (!Array.isArray(models)) return []; - const rows = models - .filter((model): model is Record & { id: string; provider: string } => - typeof model === "object" && model !== null && typeof model.id === "string" && typeof model.provider === "string") - .map(model => ({ - provider: model.provider, - id: model.id, - ...(typeof model.contextWindow === "number" ? { contextWindow: model.contextWindow } : {}), - ...(Array.isArray(model.inputModalities) - ? { inputModalities: model.inputModalities.filter((value): value is string => typeof value === "string") } - : {}), - ...(Array.isArray(model.reasoningEfforts) - ? { reasoningEfforts: model.reasoningEfforts.filter((value): value is string => typeof value === "string") } - : {}), - ...(Array.isArray(model.capabilities) - ? { capabilities: model.capabilities.filter((value): value is string => typeof value === "string") } - : {}), - })); - catalogCache = { path, mtimeMs, rows }; - return rows; - } catch { - return []; - } -} - /** * Classify a hostname for locality evidence. `URL.hostname` keeps IPv6 * literals bracketed (`[::1]`), so strip the brackets before matching. @@ -133,8 +83,8 @@ function localRemoteEvidence(baseUrl: string | undefined): Pick entry.id === providerName); - const catalogRow = cachedCatalogModels().find(model => model.provider === providerName && model.id === modelId); + const registryEntry = provider && providerMatchesRegistryTransport(providerName, provider) + ? PROVIDER_REGISTRY.find(entry => entry.id === providerName) + : undefined; + const customModel = config.customModels?.find(model => + model.provider === providerName && model.modelId === modelId); const isNative = providerName === "openai" && !modelId.includes("/"); - const contextWindow = provider?.modelContextWindows?.[modelId] + const providerModelContext = modelRecordValue(provider?.modelContextWindows, modelId); + const registryModelContext = modelRecordValue(registryEntry?.modelContextWindows, modelId); + const uncappedContextWindow = customModel?.contextWindow + ?? (providerName === "openai-apikey" + && providerModelContext !== undefined + && registryModelContext !== undefined + ? Math.min(providerModelContext, registryModelContext) + : providerModelContext ?? registryModelContext) ?? provider?.contextWindow - ?? registryEntry?.modelContextWindows?.[modelId] - ?? catalogRow?.contextWindow + ?? registryEntry?.contextWindow ?? (isNative ? nativeOpenAiContextWindow(modelId) : undefined); + const contextWindow = customModel?.contextWindow !== undefined + ? uncappedContextWindow + : applyProviderContextCap(uncappedContextWindow, providerContextCap(config, providerName)); - const modalities = provider?.modelInputModalities?.[modelId] - ?? registryEntry?.modelInputModalities?.[modelId] - ?? catalogRow?.inputModalities + const modalities = customModel?.inputModalities + ?? modelRecordValue(provider?.modelInputModalities, modelId) + ?? modelRecordValue(registryEntry?.modelInputModalities, modelId) ?? (isNative ? nativeInputModalities(modelId) : undefined); const image = Array.isArray(modalities) ? modalities.includes("image") : undefined; - const capabilities = catalogRow?.capabilities ?? []; - // The catalog `capabilities` list is a positive per-model signal; a row - // without "tools" is treated as unknown, never as a negative. Without a - // catalog row the adapter protocol itself is the signal: tool-capable - // adapters run single tool calls even when the parallel-call opt-in is - // unset or false. `parallelToolCalls` stays a positive provider-level - // override. - const tools = capabilities.includes("tools") - || isNative - || (catalogRow === undefined && provider !== undefined && TOOL_CAPABLE_ADAPTERS.has(provider.adapter)) + // Adapter protocol support is positive tool evidence even when the provider + // does not opt into parallel calls. `parallelToolCalls` remains an explicit + // positive override; neither source infers a negative capability. + const tools = isNative + || (provider !== undefined && TOOL_CAPABLE_ADAPTERS.has(provider.adapter)) || provider?.parallelToolCalls === true || undefined; - const reasoningEfforts = provider?.modelReasoningEfforts?.[modelId] - ?? registryEntry?.modelReasoningEfforts?.[modelId] - ?? catalogRow?.reasoningEfforts - ?? (isNative ? nativeReasoningEfforts(modelId) : undefined); + const reasoningDisabled = modelInList(provider?.noReasoningModels, modelId) + || modelInList(registryEntry?.noReasoningModels, modelId); + const configuredReasoning = reasoningDisabled + ? [] + : modelRecordValue(provider?.modelReasoningEfforts, modelId) + ?? modelRecordValue(registryEntry?.modelReasoningEfforts, modelId) + ?? provider?.reasoningEfforts + ?? registryEntry?.reasoningEfforts; + const reasoningEfforts = configuredReasoning === undefined + ? (isNative ? nativeReasoningEfforts(modelId) : undefined) + : sanitizeCodexReasoningEfforts(configuredReasoning) ?? []; const tierSupport = provider?.supportsServiceTier ?? registryEntry?.supportsServiceTier; @@ -196,7 +160,7 @@ export function candidateCapabilityEvidence( ...(typeof contextWindow === "number" ? { contextWindow } : {}), ...(typeof image === "boolean" ? { image } : {}), ...(typeof tools === "boolean" ? { tools } : {}), - ...(reasoningEfforts !== undefined && reasoningEfforts.length > 0 ? { reasoningEfforts } : {}), + ...(reasoningEfforts !== undefined ? { reasoningEfforts } : {}), ...(serviceTier !== "unknown" ? { serviceTier } : {}), ...localRemote, ...(typeof encryptedCodexTasks === "boolean" ? { encryptedCodexTasks } : {}), diff --git a/tests/route-explainability.test.ts b/tests/route-explainability.test.ts index 4f78fc9479..34a62e3ecf 100644 --- a/tests/route-explainability.test.ts +++ b/tests/route-explainability.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { handleManagementAPI } from "../src/server/management-api"; @@ -8,6 +8,7 @@ import { appendUsageEntry, resetUsageReadCacheForTests, type PersistedUsageEntry import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; import { candidateCapabilityEvidence } from "../src/routing/capability"; import type { CodexCommanderConfig } from "../src/types"; +import { installIsolatedCodexHome } from "./helpers/isolated-codex-home"; let testDir = ""; let previousHome: string | undefined; @@ -200,6 +201,224 @@ describe("route explainability (RI-09)", () => { expect(Object.prototype.hasOwnProperty.call(evidence, "encryptedCodexTasks")).toBe(false); }); + test("explicit custom-model metadata is trusted without generated-catalog inference", () => { + const cfg = config(); + cfg.providers.custom = { + adapter: "openai-chat", + baseUrl: "https://custom.example/v1", + apiKey: "kc", + models: ["vendor/model"], + contextWindow: 64_000, + modelContextWindows: { "vendor/model": 96_000 }, + modelInputModalities: { "vendor/model": ["text"] }, + }; + cfg.customModels = [{ + id: "custom-row", + provider: "custom", + modelId: "vendor/model", + contextWindow: 250_000, + inputModalities: ["text", "image"], + }]; + + expect(candidateCapabilityEvidence(cfg, "custom", "vendor/model")).toMatchObject({ + contextWindow: 250_000, + image: true, + tools: true, + }); + }); + + test("model metadata lookup keeps family and case-fold semantics", () => { + const cfg = config(); + cfg.providers.custom = { + adapter: "bare", + baseUrl: "https://custom.example/v1", + apiKey: "kc", + models: ["Model:tag"], + modelContextWindows: { Model: 180_000 }, + modelInputModalities: { "model:TAG": ["text", "image"] }, + modelReasoningEfforts: { Model: ["low", "high"] }, + }; + + expect(candidateCapabilityEvidence(cfg, "custom", "Model:tag")).toMatchObject({ + contextWindow: 180_000, + image: true, + reasoningEfforts: ["low", "high"], + }); + }); + + test("provider-wide reasoning metadata remains canonical evidence", () => { + const cfg = config(); + cfg.providers.custom = { + adapter: "bare", + baseUrl: "https://custom.example/v1", + apiKey: "kc", + models: ["mystery"], + reasoningEfforts: ["high", "low", "not-a-codex-tier"], + }; + + expect(candidateCapabilityEvidence(cfg, "custom", "mystery").reasoningEfforts) + .toEqual(["low", "high"]); + }); + + test("transport-owned registry-wide reasoning metadata remains canonical evidence", () => { + const cfg = config(); + cfg.providers["cline-pass"] = { + adapter: "openai-chat", + baseUrl: "https://api.cline.bot/api/v1", + apiKey: "kc", + models: ["cline-pass/kimi-k3"], + }; + + expect(candidateCapabilityEvidence(cfg, "cline-pass", "cline-pass/kimi-k3").reasoningEfforts) + .toEqual(["low"]); + }); + + test("provider-wide reasoning takes precedence over registry-wide defaults", () => { + const cfg = config(); + cfg.providers["cline-pass"] = { + adapter: "openai-chat", + baseUrl: "https://api.cline.bot/api/v1", + apiKey: "kc", + models: ["cline-pass/kimi-k3"], + reasoningEfforts: ["high"], + }; + + expect(candidateCapabilityEvidence(cfg, "cline-pass", "cline-pass/kimi-k3").reasoningEfforts) + .toEqual(["high"]); + }); + + test("trusted model reasoning and no-reasoning facts beat provider-wide defaults", () => { + const cfg = config(); + cfg.providers["opencode-go"] = { + adapter: "openai-chat", + baseUrl: "https://opencode.ai/zen/go/v1", + apiKey: "kc", + models: ["deepseek-v4-flash", "kimi-k2.7-code"], + reasoningEfforts: ["high"], + }; + + expect(candidateCapabilityEvidence(cfg, "opencode-go", "deepseek-v4-flash").reasoningEfforts) + .toEqual(["low", "high", "max"]); + expect(candidateCapabilityEvidence(cfg, "opencode-go", "kimi-k2.7-code").reasoningEfforts) + .toEqual([]); + }); + + test("trusted registry per-model context beats a provider-wide fallback", () => { + const cfg = config(); + cfg.providers["cline-pass"] = { + adapter: "openai-chat", + baseUrl: "https://api.cline.bot/api/v1", + apiKey: "kc", + models: ["cline-pass/kimi-k3"], + contextWindow: 2_000_000, + }; + + expect(candidateCapabilityEvidence(cfg, "cline-pass", "cline-pass/kimi-k3").contextWindow) + .toBe(1_048_576); + }); + + test("provider context caps lower trusted policy evidence", () => { + const cfg = config(); + cfg.providerContextCaps = { a: 150_000 }; + + expect(candidateCapabilityEvidence(cfg, "a", "m1").contextWindow).toBe(150_000); + }); + + test("explicit custom-model context remains the user's routing assertion", () => { + const cfg = config(); + cfg.providers.custom = { + adapter: "bare", + baseUrl: "https://custom.example/v1", + apiKey: "kc", + models: ["mystery"], + }; + cfg.customModels = [{ + id: "custom-row", + provider: "custom", + modelId: "mystery", + contextWindow: 250_000, + }]; + cfg.providerContextCaps = { custom: 150_000 }; + + expect(candidateCapabilityEvidence(cfg, "custom", "mystery").contextWindow).toBe(250_000); + }); + + test("same-named custom transports do not inherit registry capabilities", () => { + const cfg = config(); + cfg.providers["opencode-go"] = { + adapter: "openai-chat", + baseUrl: "https://custom.example/v1", + apiKey: "kc", + models: ["deepseek-v4-flash"], + }; + + const evidence = candidateCapabilityEvidence(cfg, "opencode-go", "deepseek-v4-flash"); + expect(Object.prototype.hasOwnProperty.call(evidence, "contextWindow")).toBe(false); + expect(Object.prototype.hasOwnProperty.call(evidence, "image")).toBe(false); + expect(Object.prototype.hasOwnProperty.call(evidence, "reasoningEfforts")).toBe(false); + expect(evidence.tools).toBe(true); + }); + + test("unknown custom-model facts stay unknown", () => { + const cfg = config(); + cfg.providers.custom = { + adapter: "bare", + baseUrl: "https://custom.example/v1", + apiKey: "kc", + models: ["mystery"], + }; + cfg.customModels = [{ id: "custom-row", provider: "custom", modelId: "mystery" }]; + + const evidence = candidateCapabilityEvidence(cfg, "custom", "mystery"); + expect(Object.prototype.hasOwnProperty.call(evidence, "contextWindow")).toBe(false); + expect(Object.prototype.hasOwnProperty.call(evidence, "image")).toBe(false); + expect(Object.prototype.hasOwnProperty.call(evidence, "reasoningEfforts")).toBe(false); + expect(Object.prototype.hasOwnProperty.call(evidence, "tools")).toBe(false); + }); + + test("persisted catalog rows cannot become routing capability authority", () => { + const home = installIsolatedCodexHome("ccx-route-evidence-"); + try { + // Deliberately uses the obsolete in-memory shape the removed reader + // accepted. Generated catalog artifacts are presentation state, not + // verified provider capability evidence. + writeFileSync(join(home.path, "codexcommander-catalog.json"), JSON.stringify({ + models: [ + { + provider: "custom", + id: "vendor/model", + contextWindow: 900_000, + inputModalities: ["image"], + reasoningEfforts: ["low", "max"], + capabilities: ["tools"], + }, + { provider: "chat", id: "plain", capabilities: [] }, + ], + })); + const cfg = config(); + cfg.providers.custom = { + adapter: "bare", + baseUrl: "https://custom.example/v1", + apiKey: "kc", + models: ["vendor/model"], + }; + cfg.providers.chat = { + adapter: "openai-chat", + baseUrl: "https://chat.example/v1", + apiKey: "kchat", + models: ["plain"], + }; + + const untrusted = candidateCapabilityEvidence(cfg, "custom", "vendor/model"); + for (const key of ["contextWindow", "image", "reasoningEfforts", "tools"]) { + expect(Object.prototype.hasOwnProperty.call(untrusted, key)).toBe(false); + } + expect(candidateCapabilityEvidence(cfg, "chat", "plain").tools).toBe(true); + } finally { + home.restore(); + } + }); + test("CLI logs explain encodes request ids and supports --json", async () => { const { handleObserveCommand } = await import("../src/cli/observe"); const calls: Array<{ path: string; init?: RequestInit }> = [];