diff --git a/tests/helpers/responses-conformance.ts b/tests/helpers/responses-conformance.ts new file mode 100644 index 000000000..0991a7afd --- /dev/null +++ b/tests/helpers/responses-conformance.ts @@ -0,0 +1,139 @@ +import { bridgeToResponsesSSE, buildResponseJSON } from "../../src/bridge"; +import type { AdapterEvent } from "../../src/types"; + +/** + * Shared harness for Responses tool round-trip conformance + * (devlog/_plan/260813_routed_tool_discovery_profiles/030-034). + * + * Every existing tool test re-implements `replay`/`collectSse` locally, which is why the + * streaming and non-streaming paths had never been compared: each test only looked at one. + * + * The streamed side is read from BOTH surfaces on purpose. `response.completed` is what a + * client that reconnects or ignores deltas sees; `response.output_item.done` is what a client + * consuming normal incremental frames sees. The bridge builds them separately, so reading only + * the snapshot hides a whole divergence class — an item can be correct in the final snapshot + * and wrong in the incremental frame. devlog 034 requires the incremental assertions. + */ + +export async function* replay(events: readonly AdapterEvent[]): AsyncGenerator { + for (const event of events) yield event; +} + +export interface SseFrame { + event?: string; + data: Record; +} + +export async function collectSse(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + return text.split("\n\n") + .map(frame => frame.trim()) + .filter(frame => frame.length > 0 && frame !== "data: [DONE]") + .map(frame => { + const lines = frame.split("\n"); + const event = lines.find(line => line.startsWith("event: "))?.slice(7); + const dataLine = lines.find(line => line.startsWith("data: ")); + return { event, data: JSON.parse(dataLine?.slice(6) ?? "{}") as Record }; + }); +} + +/** The tool-bearing fields every transport must agree on, in output order. */ +export interface NormalizedToolItem { + type: string; + name?: string; + call_id?: string; + /** `arguments` for function/tool_search, `input` for custom. Objects are preserved. */ + payload?: unknown; + status?: string; + /** Namespace identity, when the restored item carries one. */ + namespace?: string; +} + +function normalizeItem(item: Record): NormalizedToolItem { + const payload = item.arguments !== undefined ? item.arguments : item.input; + return { + type: String(item.type ?? ""), + ...(typeof item.name === "string" ? { name: item.name } : {}), + ...(typeof item.call_id === "string" ? { call_id: item.call_id } : {}), + ...(payload !== undefined ? { payload } : {}), + ...(typeof item.status === "string" ? { status: item.status } : {}), + ...(typeof item.namespace === "string" ? { namespace: item.namespace } : {}), + }; +} + +const isToolItem = (item: Record): boolean => + String(item.type ?? "").includes("call"); + +type BridgeMaps = [ + toolNsMap?: Map, + freeformToolNames?: Set, + toolSearchToolNames?: Set, +]; + +export interface StreamedView { + /** Tool items from the terminal `response.completed` snapshot. */ + snapshot: NormalizedToolItem[]; + /** Tool items from the incremental `response.output_item.done` frames. */ + incremental: NormalizedToolItem[]; + /** Every frame's event name, in order. */ + eventNames: string[]; + /** Ordered payloads of every argument/input delta frame. */ + deltas: string[]; + /** Every non-call output item type from the snapshot, e.g. "message". */ + snapshotItemTypes: string[]; +} + +export async function streamedView( + events: readonly AdapterEvent[], + modelId: string, + ...maps: BridgeMaps +): Promise { + const frames = await collectSse(bridgeToResponsesSSE(replay(events), modelId, ...maps)); + const completed = frames.find(frame => frame.event === "response.completed"); + const response = completed?.data.response as Record | undefined; + const output = Array.isArray(response?.output) ? response.output as Record[] : []; + + const doneItems = frames + .filter(frame => frame.event === "response.output_item.done") + .map(frame => frame.data.item) + .filter((item): item is Record => !!item && typeof item === "object"); + + return { + snapshot: output.filter(isToolItem).map(normalizeItem), + incremental: doneItems.filter(isToolItem).map(normalizeItem), + eventNames: frames.map(frame => frame.event ?? ""), + deltas: frames + .filter(frame => frame.event?.endsWith(".delta") && typeof frame.data.delta === "string") + .map(frame => String(frame.data.delta)), + snapshotItemTypes: output.map(item => String(item.type ?? "")), + }; +} + +/** Tool items from the non-streaming transport. */ +export function jsonToolItems( + events: readonly AdapterEvent[], + modelId: string, + options?: Parameters[2], +): NormalizedToolItem[] { + const body = buildResponseJSON([...events], modelId, options); + const output = Array.isArray(body.output) ? body.output as Record[] : []; + return output.filter(isToolItem).map(normalizeItem); +} + +/** Every output item type from the non-streaming transport, including non-call items. */ +export function jsonItemTypes( + events: readonly AdapterEvent[], + modelId: string, + options?: Parameters[2], +): string[] { + const body = buildResponseJSON([...events], modelId, options); + const output = Array.isArray(body.output) ? body.output as Record[] : []; + return output.map(item => String(item.type ?? "")); +} diff --git a/tests/responses-tool-conformance.test.ts b/tests/responses-tool-conformance.test.ts new file mode 100644 index 000000000..95bf30e55 --- /dev/null +++ b/tests/responses-tool-conformance.test.ts @@ -0,0 +1,402 @@ +import { describe, expect, it } from "bun:test"; +import { parseRequest } from "../src/responses/parser"; +import type { AdapterEvent } from "../src/types"; +import { jsonItemTypes, jsonToolItems, streamedView } from "./helpers/responses-conformance"; + +/** + * Responses tool round-trip conformance + * (devlog/_plan/260813_routed_tool_discovery_profiles/030-034). + * + * The plan's premise was that a translator reading only top-level `tools` can silently erase + * terminal, custom and namespace tools, and that the model then emits ordinary text and + * completes normally — a failure that looks like model behavior rather than protocol loss. + * That premise is partly obsolete here: `additional_tools` IS parsed and merged. This suite + * covers PART of the residual exposure — malformed input, unknown tool kinds, and + * stream/non-stream divergence. It is a STARTING slice of the 030-034 programme, not a + * complete conformance layer: the end-to-end declaration/call/result/second-turn execution + * loop, compaction and resume with a discovered tool, the full collision matrix, per-adapter + * declaration comparison, transport-error parity, and the conformance artifacts are absent. + * + * Several cases below deliberately pin CURRENT degradation rather than desired behavior. + * They are labeled as such, so a future change to that behavior fails here loudly instead of + * being discovered by a user whose tool vanished. + */ + +const MODEL = "deepseek/glm-5.2"; + +function request(input: unknown[], tools?: unknown[]): Record { + return { + model: MODEL, + input, + ...(tools ? { tools } : {}), + }; +} + +function toolNames(parsed: ReturnType): string[] { + return (parsed.context.tools ?? []).map(tool => tool.name); +} + +describe("Responses Lite additional_tools declaration merge", () => { + const fnTool = { type: "function", name: "read_file", parameters: { type: "object", properties: {} } }; + const nsTool = { + type: "namespace", + name: "github", + tools: [{ type: "function", name: "search", parameters: { type: "object", properties: {} } }], + }; + const customTool = { type: "custom", name: "apply_patch" }; + + it("merges top-level tools and additional_tools input items", () => { + const parsed = parseRequest(request( + [{ type: "additional_tools", role: "developer", tools: [customTool] }], + [fnTool], + )); + expect(toolNames(parsed)).toEqual(["read_file", "apply_patch"]); + }); + + it("accepts an additional_tools-only request, the Codex Desktop responses_lite shape", () => { + // The tool surface rides INSIDE input rather than body.tools. A translator that reads + // only body.tools sees an empty catalog and the turn silently loses every tool. + const parsed = parseRequest(request( + [{ type: "additional_tools", role: "developer", tools: [fnTool, nsTool, customTool] }], + )); + // Namespaced children keep their BARE name; the namespace rides alongside on the tool. + expect(toolNames(parsed)).toEqual(["read_file", "search", "apply_patch"]); + expect(parsed.context.tools?.find(tool => tool.name === "search")?.namespace).toBe("github"); + }); + + it("preserves wire order across multiple additional_tools groups", () => { + const parsed = parseRequest(request([ + { type: "additional_tools", role: "developer", tools: [fnTool] }, + { type: "additional_tools", role: "developer", tools: [customTool] }, + ])); + expect(toolNames(parsed)).toEqual(["read_file", "apply_patch"]); + }); + + it("lets the top-level declaration win a qualified-name collision", () => { + const shadowed = { type: "function", name: "read_file", parameters: { type: "object", properties: { shadow: { type: "string" } } } }; + const parsed = parseRequest(request( + [{ type: "additional_tools", role: "developer", tools: [shadowed] }], + [fnTool], + )); + expect(toolNames(parsed)).toEqual(["read_file"]); + // The surviving entry is the TOP-LEVEL one, not the nested shadow. + expect(parsed.context.tools?.[0]?.parameters).toEqual(fnTool.parameters as never); + }); + + it("CURRENT BEHAVIOR: a malformed additional_tools item is ignored, not rejected", () => { + // devlog 031 asks for explicit failure here. Today the item is skipped silently, so a + // typo in the tool surface degrades to "model has no tools" with no diagnostic. Pinned + // so that changing it to a hard error is a visible, deliberate decision. + // + // A WELL-FORMED sibling group rides along deliberately: without it this case would also + // pass if additional_tools support were deleted outright, and "ignored one bad item" + // would be indistinguishable from "lost the whole feature". + const parsed = parseRequest(request([ + { type: "additional_tools", role: "developer", tools: "not-an-array" }, + { type: "additional_tools", role: "developer", tools: [fnTool] }, + ])); + expect(toolNames(parsed)).toEqual(["read_file"]); + }); +}); + +describe("Responses tool-kind discrimination", () => { + it("maps each known kind onto its internal marker", () => { + const parsed = parseRequest(request([], [ + { type: "function", name: "fn", parameters: { type: "object", properties: { path: { type: "string" } } } }, + { type: "custom", name: "freeform" }, + { type: "tool_search", execution: "client", description: "search", parameters: { type: "object", properties: {} } }, + { type: "namespace", name: "ns", tools: [{ type: "function", name: "child", parameters: { type: "object", properties: {} } }] }, + ])); + const byName = new Map((parsed.context.tools ?? []).map(tool => [tool.name, tool])); + // NOTE: this asserts function-declaration BEHAVIOR, not branch discrimination. The + // explicit `type === "function"` branch and the generic named-tool fallback both call + // pushFn(t) (parser.ts:157 and :194), so they are observably identical for a named tool + // and NO assertion can tell them apart. Deleting the explicit branch keeps this green on + // purpose; what it does pin is that a declared schema reaches the model intact. + expect(byName.get("fn")?.parameters).toEqual({ type: "object", properties: { path: { type: "string" } } } as never); + expect(byName.get("freeform")?.freeform).toBe(true); + expect(byName.get("tool_search")?.toolSearch).toBe(true); + expect(byName.get("child")?.namespace).toBe("ns"); + }); + + it("CURRENT BEHAVIOR: an unknown NAMED kind survives as a callable function", () => { + // Better than the historical silent drop, but the original kind is not recoverable, so + // the response cannot be restored as that kind either. + const parsed = parseRequest(request([], [ + { type: "computer_use_preview", name: "computer", parameters: { type: "object", properties: {} } }, + ])); + expect(toolNames(parsed)).toEqual(["computer"]); + expect(parsed.context.tools?.[0]?.freeform).toBeUndefined(); + }); + + it("CURRENT BEHAVIOR: an unknown UNNAMED kind disappears entirely", () => { + // This is the plan's silent-loss class, still live. There is no name to pass through, so + // the declaration is dropped with no diagnostic. + const parsed = parseRequest(request([], [ + { type: "some_future_hosted_tool", config: { enabled: true } }, + ])); + expect(parsed.context.tools ?? []).toEqual([]); + }); + + it("CURRENT BEHAVIOR: a non-function child inside a namespace disappears", () => { + const parsed = parseRequest(request([], [ + { + type: "namespace", + name: "ns", + tools: [ + { type: "function", name: "kept", parameters: { type: "object", properties: {} } }, + { type: "custom", name: "dropped" }, + ], + }, + ])); + expect(toolNames(parsed)).toEqual(["kept"]); + }); +}); + +describe("tool_search call and output history", () => { + it("loads definitions returned by a tool_search_output into the active catalog", () => { + const parsed = parseRequest(request([ + { type: "message", role: "user", content: [{ type: "input_text", text: "find it" }] }, + { type: "tool_search_call", id: "ts_1", call_id: "ts_1", execution: "client", arguments: "{\"query\":\"repl\"}", status: "completed" }, + { + type: "tool_search_output", + call_id: "ts_1", + status: "completed", + execution: "client", + tools: [{ type: "function", name: "node_repl", parameters: { type: "object", properties: {} } }], + }, + ], [ + { type: "tool_search", execution: "client", description: "search", parameters: { type: "object", properties: {} } }, + ])); + + // The discovered tool must be callable on the NEXT turn, or deferred discovery is a + // one-way trip and the model can never invoke what it just found. + expect(toolNames(parsed)).toContain("node_repl"); + const loaded = parsed.context.tools?.find(tool => tool.name === "node_repl"); + expect(loaded?.loadedFromToolSearch).toBe(true); + }); + + it("preserves the search call itself in assistant history, paired with its result", () => { + // Loading the discovered DEFINITIONS is not enough: if the tool_search_call disappears + // from history, the next upstream request has an orphaned result and providers reject + // the turn. Asserts the call survives with its id, and that the paired result carries + // the same id so the two can be matched. + const parsed = parseRequest(request([ + { type: "message", role: "user", content: [{ type: "input_text", text: "find it" }] }, + { type: "tool_search_call", id: "ts_1", call_id: "ts_1", execution: "client", arguments: "{\"query\":\"repl\"}", status: "completed" }, + { + type: "tool_search_output", + call_id: "ts_1", + status: "completed", + execution: "client", + tools: [{ type: "function", name: "node_repl", parameters: { type: "object", properties: {} } }], + }, + ], [ + { type: "tool_search", execution: "client", description: "search", parameters: { type: "object", properties: {} } }, + ])); + + const messages = parsed.context.messages; + const assistant = messages.find(message => message.role === "assistant"); + const call = Array.isArray(assistant?.content) + ? assistant.content.find(part => (part as { type?: string }).type === "toolCall") as { id?: string; name?: string } | undefined + : undefined; + expect(call?.name).toBe("tool_search"); + expect(call?.id).toBe("ts_1"); + + const result = messages.find(message => message.role === "toolResult") as { toolCallId?: string; toolName?: string } | undefined; + expect(result?.toolCallId).toBe("ts_1"); + expect(result?.toolName).toBe("tool_search"); + }); + + it("flattens a namespaced tool discovered through tool_search", () => { + const parsed = parseRequest(request([ + { + type: "tool_search_output", + call_id: "ts_2", + status: "completed", + execution: "client", + tools: [{ + type: "namespace", + name: "browser", + tools: [{ type: "function", name: "open", parameters: { type: "object", properties: {} } }], + }], + }, + ], [ + { type: "tool_search", execution: "client", description: "search", parameters: { type: "object", properties: {} } }, + ])); + expect(toolNames(parsed)).toContain("open"); + expect(parsed.context.tools?.find(tool => tool.name === "open")?.namespace).toBe("browser"); + }); +}); + +describe("streaming and non-streaming tool parity", () => { + const nsMap = new Map([["ns__child", { namespace: "ns", name: "child" }]]); + const freeform = new Set(["apply_patch"]); + const toolSearch = new Set(["tool_search"]); + + const cases: Array<{ label: string; events: AdapterEvent[] }> = [ + { + label: "function call", + events: [ + { type: "tool_call_start", id: "call_fn", name: "read_file" }, + { type: "tool_call_delta", arguments: "{\"path\"" }, + { type: "tool_call_delta", arguments: ":\"a.txt\"}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + { + label: "custom/freeform call with split escapes and non-ASCII", + events: [ + { type: "tool_call_start", id: "call_custom", name: "apply_patch" }, + { type: "tool_call_delta", arguments: "{\"input\":\"안녕 \\" }, + { type: "tool_call_delta", arguments: "\"quoted\\\" 世界\"}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + { + label: "namespaced call", + events: [ + { type: "tool_call_start", id: "call_ns", name: "ns__child" }, + { type: "tool_call_delta", arguments: "{}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + { + label: "tool_search call", + events: [ + { type: "tool_call_start", id: "call_ts", name: "tool_search" }, + { type: "tool_call_delta", arguments: "{\"query\":\"repl\"}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + { + label: "text before a call", + events: [ + { type: "text_delta", text: "working" }, + { type: "tool_call_start", id: "call_after_text", name: "read_file" }, + { type: "tool_call_delta", arguments: "{}" }, + { type: "tool_call_end" }, + { type: "done" }, + ], + }, + ]; + + for (const { label, events } of cases) { + it(`agrees across snapshot, incremental frames and JSON for a ${label}`, async () => { + const view = await streamedView(events, MODEL, nsMap, freeform, toolSearch); + const json = jsonToolItems(events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); + + // Three surfaces, not two. `response.completed` is what a reconnecting client sees; + // `output_item.done` is what a client consuming normal incremental frames sees. The + // bridge builds them separately, so comparing only the snapshot hides an item that is + // correct at the end and wrong on the wire (devlog 034). + expect(view.incremental).toEqual(view.snapshot); + expect(view.snapshot).toEqual(json); + }); + } + + it("preserves assistant text alongside the call on both transports", async () => { + // The tool-item filter hides non-call output, so without this the "text before a call" + // fixture would be just another function-call parity case. + const textCase = cases.find(entry => entry.label.startsWith("text"))!; + const view = await streamedView(textCase.events, MODEL, nsMap, freeform, toolSearch); + const jsonTypes = jsonItemTypes(textCase.events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); + expect(view.snapshotItemTypes).toContain("message"); + expect(jsonTypes).toContain("message"); + expect(view.snapshotItemTypes).toEqual(jsonTypes); + }); + + it("restores namespace identity identically on both transports", async () => { + // NormalizedToolItem carries `namespace`, so dropping namespace restoration from either + // bridge fails here. An earlier revision omitted the field and could not see it at all. + const nsCase = cases.find(entry => entry.label.startsWith("namespaced"))!; + const view = await streamedView(nsCase.events, MODEL, nsMap, freeform, toolSearch); + const json = jsonToolItems(nsCase.events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); + + // ABSOLUTE assertions first. Equality alone is satisfied by EQUAL DEGRADATION: deleting + // namespace restoration from BOTH bridges keeps the two sides identical, so a pure + // comparison test would stay green while the identity was lost on the wire. + for (const item of [view.snapshot[0], view.incremental[0], json[0]]) { + expect(item?.name).toBe("child"); + expect(item?.namespace).toBe("ns"); + } + expect(view.incremental).toEqual(view.snapshot); + expect(view.snapshot).toEqual(json); + }); + + it("carries the tool_search payload rather than an empty object", async () => { + // tool_search arguments may be an object rather than a string; the normalizer keeps the + // value verbatim so replacing it with {} on both paths cannot pass silently. + const tsCase = cases.find(entry => entry.label.startsWith("tool_search"))!; + const view = await streamedView(tsCase.events, MODEL, nsMap, freeform, toolSearch); + const json = jsonToolItems(tsCase.events, MODEL, { toolNsMap: nsMap, freeformToolNames: freeform, toolSearchToolNames: toolSearch }); + expect(JSON.stringify(view.snapshot[0]?.payload)).toContain("repl"); + expect(view.snapshot).toEqual(json); + }); + + it("restores each kind as its own item type rather than collapsing to function_call", async () => { + const kinds = await Promise.all(cases.map(async ({ events }) => { + const view = await streamedView(events, MODEL, nsMap, freeform, toolSearch); + return view.snapshot[0]?.type; + })); + expect(kinds).toEqual([ + "function_call", + "custom_tool_call", + "function_call", + "tool_search_call", + "function_call", + ]); + }); + + it("emits the exact custom input fragments on the streamed path only", async () => { + const custom = cases.find(entry => entry.label.startsWith("custom"))!; + const view = await streamedView(custom.events, MODEL, nsMap, freeform, toolSearch); + expect(view.eventNames).toContain("response.custom_tool_call_input.delta"); + expect(view.eventNames).not.toContain("response.function_call_arguments.delta"); + // Exact ordered fragments, not just the event name: a corrupted, duplicated or reordered + // delta stream would otherwise pass. + expect(view.deltas.join("")).toBe(String(view.snapshot[0]?.payload ?? "")); + // Exactly one terminal item event for the one call. + expect(view.eventNames.filter(name => name === "response.output_item.done")).toHaveLength(1); + }); +}); + +describe("parallel tool-call capability", () => { + it("CURRENT LIMITATION: interleaved calls cannot be represented by AdapterEvent", async () => { + // `tool_call_start` carries an id, but `tool_call_delta` and `tool_call_end` do not + // (src/types.ts:323). Both bridges therefore track ONE current call, so a provider that + // interleaves two calls has no way to say which fragment belongs to which. + // + // A genuine A/B/A/B interleaving: call A is fragmented, B starts mid-flight, then A's + // continuation arrives. This pins the consequence rather than pretending it works — + // A's later fragment is misattributed to B, because fragments are routed by ARRIVAL + // ORDER, not by call id. Giving delta/end a call id should make this fail. + const interleaved: AdapterEvent[] = [ + { type: "tool_call_start", id: "call_a", name: "read_file" }, + { type: "tool_call_delta", arguments: "{\"path\":\"a" }, + { type: "tool_call_start", id: "call_b", name: "write_file" }, + { type: "tool_call_delta", arguments: "{\"path\":\"b" }, + { type: "tool_call_delta", arguments: ".txt\"}" }, + { type: "tool_call_end" }, + { type: "done" }, + ]; + + const view = await streamedView(interleaved, MODEL); + const json = jsonToolItems(interleaved, MODEL); + // Both transports agree with each other AND with the incremental frames, which is what + // makes this a contract limitation rather than a transport bug. + expect(view.incremental).toEqual(view.snapshot); + expect(view.snapshot).toEqual(json); + + expect(view.snapshot.map(item => item.call_id)).toEqual(["call_a", "call_b"]); + expect(view.snapshot.map(item => item.name)).toEqual(["read_file", "write_file"]); + // A keeps only what arrived before B started; B absorbs A's continuation. Neither + // payload is the JSON its provider actually sent. + expect(view.snapshot[0]?.payload).toBe("{\"path\":\"a"); + expect(view.snapshot[1]?.payload).toBe("{\"path\":\"b.txt\"}"); + }); +});