From 5abf553a8b67fe484611717ccf2b34124f27c67e Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 16 Aug 2026 10:01:05 +0900 Subject: [PATCH 1/4] fix(catalog): ingest llama.cpp multimodal and dual-envelope metadata (#1797) Closes the two halves #1799 deliberately deferred. 1. `multimodal` is now recognized as an image signal. llama.cpp and Ollama-compatible servers emit it instead of `vision`/`image-input`, so a server that truthfully advertised image support still produced an image-blind row. Mapped to the closed text|image enum rather than passed through: an out-of-enum modality makes Codex reject the whole catalog file. 2. A `data[]` row is enriched from a sibling `models[]` entry for the same id. llama.cpp splits one model's metadata across both arrays - capabilities in `models[]`, `meta` in `data[]` - so the two halves never met. The conservative boundary that refuses a stray `models` key is preserved deliberately, because that refusal exists for a reason: - membership is decided entirely by `data[]`; a sibling-only entry is dropped - matching is exact id equality (id, or Ollama's model/name), never fuzzy - an id claimed twice in the sibling array is skipped rather than guessed - only keys ABSENT from the `data[]` row are filled, so `data[]` always wins - the sibling array is bounded by the same limit as the primary envelope Verified against the verbatim payload from the reporting server: contextWindow 262144 and inputModalities ["text","image"], where before it was context-only. The characterization test that pinned the gap is flipped, and three guards pin the boundary: data[] wins a conflict, sibling-only rows stay ignored, ambiguous ids are skipped. --- src/codex/catalog/provider-fetch.ts | 5 ++ src/providers/model-discovery.ts | 62 +++++++++++++++++++++ tests/catalog-llamacpp-capabilities.test.ts | 42 ++++++++++++-- 3 files changed, 104 insertions(+), 5 deletions(-) diff --git a/src/codex/catalog/provider-fetch.ts b/src/codex/catalog/provider-fetch.ts index 899429440e..4ca9052355 100644 --- a/src/codex/catalog/provider-fetch.ts +++ b/src/codex/catalog/provider-fetch.ts @@ -990,6 +990,11 @@ function modelInputModalities( if (capabilityRecord?.vision === false) return ["text"]; if (capabilityRecord?.vision === true || capabilities?.some(value => ( value === "vision" || value === "image-input" || value === "image_input" + // llama.cpp and Ollama-compatible servers report vision as "multimodal" — + // it is the only image signal those servers emit (#1797). Mapped to the + // closed `text|image` enum rather than passed through: an out-of-enum + // modality makes Codex reject the entire catalog file. + || value === "multimodal" ))) { return ["text", "image"]; } diff --git a/src/providers/model-discovery.ts b/src/providers/model-discovery.ts index bd3795105f..301d941f5c 100644 --- a/src/providers/model-discovery.ts +++ b/src/providers/model-discovery.ts @@ -331,6 +331,56 @@ export function extractModelEnvelopeRows( } /** Validate, bound, deduplicate, and declaratively filter OpenAI `{data:[...]}` or top-level arrays (Together `#617`). */ +/** + * Enrich `data[]` rows with metadata a sibling `models[]` array carries for the + * SAME model id (#1797). + * + * Deliberately conservative, because `models` is exactly the key catalog + * discovery refuses to trust as a source of models: + * - membership is decided entirely by `data[]`; a sibling-only entry is dropped, + * - matching is exact id equality (`id`, or Ollama's `model`/`name`), never fuzzy, + * - an ambiguous id appearing twice in the sibling array is skipped rather than guessed, + * - only keys ABSENT from the `data[]` row are filled, so `data[]` always wins, + * - the sibling array is bounded by the same limit as the primary envelope. + */ +function mergeSiblingModelMetadata(value: unknown, data: unknown[], limit: number): unknown[] { + const record = plainObject(value); + const sibling = record?.models; + if (!Array.isArray(sibling) || sibling.length === 0 || sibling.length > limit) return data; + + const byId = new Map | null>(); + for (const raw of sibling) { + const entry = plainObject(raw); + if (!entry) continue; + for (const key of ["id", "model", "name"]) { + const id = entry[key]; + if (typeof id !== "string" || id.length === 0) continue; + // `null` marks an ambiguous id: two sibling entries claim it, so neither + // can be attributed with confidence and both are ignored. + byId.set(id, byId.has(id) && byId.get(id) !== entry ? null : entry); + } + } + if (byId.size === 0) return data; + + return data.map(raw => { + const row = plainObject(raw); + if (!row || typeof row.id !== "string") return raw; + const extra = byId.get(row.id); + if (!extra) return raw; + const merged: Record = { ...row }; + for (const [key, val] of Object.entries(extra)) { + if (!(key in merged)) merged[key] = val; + } + return merged; + }); +} + +function plainObject(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : null; +} + export function extractProviderModelItems( value: unknown, discovery: ResolvedProviderModelDiscovery, @@ -346,6 +396,18 @@ export function extractProviderModelItems( const envelope = extractModelEnvelopeRows(value, discovery.maxModels, ["data"]); if (!envelope.ok) return envelope; data = envelope.rows; + // llama.cpp serves a dual-envelope body: an Ollama-style `models[]` array + // carrying `capabilities` alongside the OpenAI-style `data[]` array carrying + // `meta`. The two halves of one model's metadata therefore never meet, and a + // server that truthfully advertises "multimodal" still produced an + // image-blind row (#1797). + // + // The existing refusal to trust a stray `models` key stays intact: this does + // NOT add `models` as an envelope source, and a row present only there is + // still ignored. It enriches a row `data[]` ALREADY published, matching on + // exact id, and fills only keys the `data[]` row does not define — so an + // authoritative `data[]` entry can never be overridden. + data = mergeSiblingModelMetadata(value, data, limit); } const items: ProviderModelsApiItem[] = []; diff --git a/tests/catalog-llamacpp-capabilities.test.ts b/tests/catalog-llamacpp-capabilities.test.ts index 153999128e..b94aa0293b 100644 --- a/tests/catalog-llamacpp-capabilities.test.ts +++ b/tests/catalog-llamacpp-capabilities.test.ts @@ -62,9 +62,10 @@ describe("llama.cpp served context ingestion (#1797)", () => { expect(hints.contextWindow).toBe(32768); }); - test("the dual-envelope body yields context but still no image evidence", () => { - // Characterization of the KNOWN remaining gap in #1797, so the follow-up fix - // has a live witness and a test to flip rather than a prose claim. + test("the dual-envelope body now yields BOTH context and image evidence", () => { + // Was a characterization of the #1797 gap: the multimodal token lived in + // models[] while discovery read only data[], so the row stayed image-blind. + // Both halves are now joined on exact id, and multimodal maps to image. const extracted = extractProviderModelItems(VERBATIM_LLAMACPP_BODY, { maxModels: 100, } as never); @@ -74,7 +75,38 @@ describe("llama.cpp served context ingestion (#1797)", () => { const hints = catalogHintsFromModelsApiItem("lidge", items[0] as never); expect(hints.contextWindow).toBe(262144); - // The "multimodal" token was discarded with models[]; unknown, never false. - expect(hints.inputModalities).toBeUndefined(); + expect(hints.inputModalities).toEqual(["text", "image"]); + }); + + test("a data[] value is never overridden by its sibling", () => { + // models[] is exactly the key discovery refuses to trust as a source of + // models. Enrichment fills only ABSENT keys, so an authoritative data[] + // entry always wins. + const extracted = extractProviderModelItems({ + models: [{ id: "m", context_length: 999 }], + data: [{ id: "m", context_length: 111 }], + }, { maxModels: 100 } as never); + const items = (extracted as { ok: true; items: Array> }).items; + expect(items[0]!.context_length).toBe(111); + }); + + test("a model present only in the sibling array is still ignored", () => { + // Membership is decided entirely by data[]; the conservative boundary that + // refuses a stray models key is preserved. + const extracted = extractProviderModelItems({ + models: [{ id: "ghost" }], + data: [{ id: "real" }], + }, { maxModels: 100 } as never); + const items = (extracted as { ok: true; items: Array> }).items; + expect(items.map(i => i.id)).toEqual(["real"]); + }); + + test("an ambiguous sibling id is skipped rather than guessed", () => { + const extracted = extractProviderModelItems({ + models: [{ id: "m", context_length: 999 }, { id: "m", context_length: 555 }], + data: [{ id: "m" }], + }, { maxModels: 100 } as never); + const items = (extracted as { ok: true; items: Array> }).items; + expect(items[0]!.context_length).toBeUndefined(); }); }); From ad1a89595aa8b1e1068f79e53f62ca71d7cb417a Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 16 Aug 2026 10:11:13 +0900 Subject: [PATCH 2/4] fix(discovery): enrich only after admission, and only capability keys Review found the first attempt unsound in a way that crossed a real safety boundary. mergeSiblingModelMetadata ran BEFORE providerModelMatchesDiscoveryFilter and copied every absent key, so a models[] entry could supply the exact field a provider filter required. Reproduced against the real Chutes policy: withoutSibling -> items: [] withSibling -> items: [{id:not-proven-tool-capable, supported_features:[tools]}] That contradicts the claimed membership boundary and affects every filtered provider, not just llama.cpp. Two corrections: - Enrichment now runs AFTER admission, on already-published rows only, so it can change what is KNOWN about a model but never WHICH models are published. - Only capability keys are copied (capabilities, features, supported_features, modalities, input_modalities) instead of every absent key. Verified: the reproduction now yields [], #1797 still resolves 262144 and [text,image], and a non-whitelisted sibling key is ignored. --- src/providers/model-discovery.ts | 86 +++++++++++---------- tests/catalog-llamacpp-capabilities.test.ts | 29 +++++++ 2 files changed, 76 insertions(+), 39 deletions(-) diff --git a/src/providers/model-discovery.ts b/src/providers/model-discovery.ts index 301d941f5c..a66f052434 100644 --- a/src/providers/model-discovery.ts +++ b/src/providers/model-discovery.ts @@ -332,23 +332,36 @@ export function extractModelEnvelopeRows( /** Validate, bound, deduplicate, and declaratively filter OpenAI `{data:[...]}` or top-level arrays (Together `#617`). */ /** - * Enrich `data[]` rows with metadata a sibling `models[]` array carries for the - * SAME model id (#1797). + * Metadata a sibling `models[]` array may contribute to an ALREADY-ADMITTED + * `data[]` row (#1797). * - * Deliberately conservative, because `models` is exactly the key catalog - * discovery refuses to trust as a source of models: - * - membership is decided entirely by `data[]`; a sibling-only entry is dropped, - * - matching is exact id equality (`id`, or Ollama's `model`/`name`), never fuzzy, - * - an ambiguous id appearing twice in the sibling array is skipped rather than guessed, - * - only keys ABSENT from the `data[]` row are filled, so `data[]` always wins, - * - the sibling array is bounded by the same limit as the primary envelope. + * llama.cpp serves a dual-envelope body: an Ollama-style `models[]` array + * carrying `capabilities` alongside the OpenAI-style `data[]` array carrying + * `meta`, so the two halves of one model's metadata never meet and a server + * that truthfully advertises "multimodal" produced an image-blind row. + * + * Two boundaries make this safe, and both were added after review found the + * first attempt unsound: + * + * 1. It runs AFTER admission filtering. Enriching first let a sibling supply + * the exact field a provider filter requires — reproduced against the real + * Chutes policy, where a row lacking `supported_features: ["tools"]` was + * admitted once a same-id sibling provided it. Enrichment may change what is + * KNOWN about a model, never WHICH models are published. + * 2. Only the capability keys #1797 needs are copied. A blanket "fill every + * absent key" made a key the untrusted `models[]` array could reach into any + * field the pipeline consumes. */ -function mergeSiblingModelMetadata(value: unknown, data: unknown[], limit: number): unknown[] { +const SIBLING_ENRICHABLE_KEYS = new Set(["capabilities", "features", "supported_features", "modalities", "input_modalities"]); + +type SiblingIndex = Map | null>; + +function buildSiblingIndex(value: unknown, limit: number): SiblingIndex | null { const record = plainObject(value); const sibling = record?.models; - if (!Array.isArray(sibling) || sibling.length === 0 || sibling.length > limit) return data; + if (!Array.isArray(sibling) || sibling.length === 0 || sibling.length > limit) return null; - const byId = new Map | null>(); + const byId: SiblingIndex = new Map(); for (const raw of sibling) { const entry = plainObject(raw); if (!entry) continue; @@ -360,19 +373,19 @@ function mergeSiblingModelMetadata(value: unknown, data: unknown[], limit: numbe byId.set(id, byId.has(id) && byId.get(id) !== entry ? null : entry); } } - if (byId.size === 0) return data; - - return data.map(raw => { - const row = plainObject(raw); - if (!row || typeof row.id !== "string") return raw; - const extra = byId.get(row.id); - if (!extra) return raw; - const merged: Record = { ...row }; - for (const [key, val] of Object.entries(extra)) { - if (!(key in merged)) merged[key] = val; - } - return merged; - }); + return byId.size > 0 ? byId : null; +} + +function enrichAdmittedModel(item: ProviderModelsApiItem, siblings: SiblingIndex): ProviderModelsApiItem { + const extra = siblings.get(item.id); + if (!extra) return item; + let merged: Record | null = null; + for (const key of SIBLING_ENRICHABLE_KEYS) { + if (!(key in extra) || key in item) continue; + merged ??= { ...(item as Record) }; + merged[key] = extra[key]; + } + return (merged ?? item) as ProviderModelsApiItem; } function plainObject(value: unknown): Record | null { @@ -380,13 +393,13 @@ function plainObject(value: unknown): Record | null { ? value as Record : null; } - export function extractProviderModelItems( value: unknown, discovery: ResolvedProviderModelDiscovery, ): ProviderModelItemsResult { const limit = positiveIntegerAtMost(discovery.maxModels, MODEL_DISCOVERY_MAX_MODELS); let data: unknown[]; + let siblings: SiblingIndex | null = null; if (Array.isArray(value)) { // Together-style top-level /models arrays. Catalog discovery must not treat a stray // `models` key on openai-chat responses as valid — only `data` envelopes or top-level arrays. @@ -396,18 +409,7 @@ export function extractProviderModelItems( const envelope = extractModelEnvelopeRows(value, discovery.maxModels, ["data"]); if (!envelope.ok) return envelope; data = envelope.rows; - // llama.cpp serves a dual-envelope body: an Ollama-style `models[]` array - // carrying `capabilities` alongside the OpenAI-style `data[]` array carrying - // `meta`. The two halves of one model's metadata therefore never meet, and a - // server that truthfully advertises "multimodal" still produced an - // image-blind row (#1797). - // - // The existing refusal to trust a stray `models` key stays intact: this does - // NOT add `models` as an envelope source, and a row present only there is - // still ignored. It enriches a row `data[]` ALREADY published, matching on - // exact id, and fills only keys the `data[]` row does not define — so an - // authoritative `data[]` entry can never be overridden. - data = mergeSiblingModelMetadata(value, data, limit); + siblings = buildSiblingIndex(value, limit); } const items: ProviderModelsApiItem[] = []; @@ -425,9 +427,15 @@ export function extractProviderModelItems( if (!isValidModelDiscoveryModelId(finalId)) continue; } const item = finalId === id ? raw as ProviderModelsApiItem : { ...(raw as ProviderModelsApiItem), id: finalId }; + // Admission is decided on the ORIGINAL `data[]` row, before any sibling + // enrichment. Merging first let a `models[]` entry supply the very field a + // provider filter requires — reproduced against the real Chutes policy, + // where a row lacking `supported_features: ["tools"]` was admitted once a + // same-id sibling provided it. Enrichment must never change WHICH models + // are published, only what is known about an already-admitted one. if (!providerModelMatchesDiscoveryFilter(item, discovery.spec?.filter) || seen.has(finalId)) continue; seen.add(finalId); - items.push(item); + items.push(siblings ? enrichAdmittedModel(item, siblings) : item); } return { ok: true, items, rawCount: data.length }; } diff --git a/tests/catalog-llamacpp-capabilities.test.ts b/tests/catalog-llamacpp-capabilities.test.ts index b94aa0293b..8a31c9f8c8 100644 --- a/tests/catalog-llamacpp-capabilities.test.ts +++ b/tests/catalog-llamacpp-capabilities.test.ts @@ -109,4 +109,33 @@ describe("llama.cpp served context ingestion (#1797)", () => { const items = (extracted as { ok: true; items: Array> }).items; expect(items[0]!.context_length).toBeUndefined(); }); + test("sibling metadata cannot admit a model the provider filter rejects", () => { + // Enrichment used to run BEFORE admission filtering, so a models[] entry + // could supply the exact field a filter required. Reproduced against the + // real Chutes policy: a row without supported_features:["tools"] was + // admitted once a same-id sibling provided it. Enrichment may change what + // is KNOWN about a model, never WHICH models are published. + const extracted = extractProviderModelItems({ + models: [{ id: "not-proven-tool-capable", supported_features: ["tools"] }], + data: [{ id: "not-proven-tool-capable" }], + }, { + maxModels: 100, + spec: { filter: { allOf: [{ path: ["supported_features"], containsAny: ["tools"] }] } }, + } as never); + + const items = (extracted as { ok: true; items: Array> }).items; + expect(items).toEqual([]); + }); + + test("only capability keys are enriched, not arbitrary fields", () => { + // A blanket fill-every-absent-key made the untrusted models[] array a way + // into any field the pipeline consumes. + const extracted = extractProviderModelItems({ + models: [{ id: "m", context_length: 999, owned_by: "hostile" }], + data: [{ id: "m" }], + }, { maxModels: 100 } as never); + + const items = (extracted as { ok: true; items: Array> }).items; + expect(items[0]).toEqual({ id: "m" }); + }); }); From a81a39f1883f67c94d8139411916c5400acd1b12 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 16 Aug 2026 10:12:16 +0900 Subject: [PATCH 3/4] harden(discovery): drop filter-relevant keys from the enrichable set Ordering already prevents a sibling from flipping an admission verdict, but supported_features and features are exactly the two keys real provider filters test (registry.ts:1591 requires supported_features to contain "tools"; registry.ts:1883 tests features.tool_use). A key that is both enrichable and filter-relevant is one refactor away from becoming a bypass again, and #1797 does not need either. Enrichable set is now capabilities, modalities, input_modalities. --- src/providers/model-discovery.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/providers/model-discovery.ts b/src/providers/model-discovery.ts index a66f052434..ada0bd2aec 100644 --- a/src/providers/model-discovery.ts +++ b/src/providers/model-discovery.ts @@ -349,10 +349,18 @@ export function extractModelEnvelopeRows( * admitted once a same-id sibling provided it. Enrichment may change what is * KNOWN about a model, never WHICH models are published. * 2. Only the capability keys #1797 needs are copied. A blanket "fill every - * absent key" made a key the untrusted `models[]` array could reach into any - * field the pipeline consumes. + * absent key" made the untrusted `models[]` array a way into any field the + * pipeline consumes. + * + * The list deliberately EXCLUDES `supported_features` and `features`, even + * though both are capability-shaped: they are the two keys real provider + * filters test (`registry.ts:1591` requires `supported_features` to contain + * "tools"; `registry.ts:1883` tests `features.tool_use`). Ordering already + * prevents a sibling from flipping an admission verdict, but a key that is + * both enrichable and filter-relevant is one refactor away from becoming a + * bypass again. #1797 does not need them. */ -const SIBLING_ENRICHABLE_KEYS = new Set(["capabilities", "features", "supported_features", "modalities", "input_modalities"]); +const SIBLING_ENRICHABLE_KEYS = new Set(["capabilities", "modalities", "input_modalities"]); type SiblingIndex = Map | null>; From a88b781d1bea4613038643bbb10a8d7c95c7378c Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 16 Aug 2026 10:18:08 +0900 Subject: [PATCH 4/4] docs(test): refresh the #1797 header now that both halves resolve The header still described the image half as unfixed and claimed a merged item would stay image-unknown, contradicting the end-to-end assertion below it. --- tests/catalog-llamacpp-capabilities.test.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/catalog-llamacpp-capabilities.test.ts b/tests/catalog-llamacpp-capabilities.test.ts index 8a31c9f8c8..1eb433fd80 100644 --- a/tests/catalog-llamacpp-capabilities.test.ts +++ b/tests/catalog-llamacpp-capabilities.test.ts @@ -3,16 +3,17 @@ import { catalogHintsFromModelsApiItem } from "../src/codex/catalog/provider-fet import { extractProviderModelItems } from "../src/providers/model-discovery"; /** - * Regression coverage for the context half of #1797. + * Regression coverage for #1797 (llama.cpp dual-envelope metadata). * - * A llama.cpp server reports its served context under `meta.n_ctx`, which was in - * none of the recognized context fields, so a correct local server produced no - * context evidence at all. + * A llama.cpp server splits one model across two arrays: an Ollama-style + * `models[]` carrying `capabilities` and an OpenAI-style `data[]` carrying + * `meta`. Discovery reads only `data[]`, so the served context (`meta.n_ctx`) + * and the image signal (`multimodal`) never met and a correct server produced + * a context-unknown, image-blind row. * - * The image half of #1797 is NOT fixed here and is characterized below: the - * `multimodal` token lives in the Ollama-style `models[]` array while discovery - * deliberately reads only `data[]`, and even a merged item would stay - * image-unknown because `multimodal` is not a recognized capability string. + * Both halves now resolve. The tests below also pin the boundary that makes + * the join safe: admission is decided on the original `data[]` row before any + * enrichment, and only capability keys are copied. */ const VERBATIM_LLAMACPP_BODY = {