From dcdae4d1dc828c6b9dd948b71b339b42df219ab2 Mon Sep 17 00:00:00 2001 From: zhouyayu Date: Fri, 18 Sep 2026 19:59:44 +0800 Subject: [PATCH] fix(qoder): preserve user and tool-result images --- CHANGELOG.md | 4 +++ worker/src/plaintext.mjs | 23 ++++++++++++++--- worker/test/plaintext.test.mjs | 46 ++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3817e7..5dba9b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Write each change in both `### English` and `### 中文` under `## Unreleased`. ### English +- Preserve Qoder user images and image-bearing tool results, emitting tool-result images after their complete ordered tool batch. + - Preserve typed upstream stream errors through the OpenAI, Anthropic, and Responses relays so invalid Devin requests do not falsely cool accounts, while transport interruptions remain retryable - Report Devin cache reads and writes in OpenAI-compatible usage, with prompt totals including all upstream input tokens @@ -19,6 +21,8 @@ Write each change in both `### English` and `### 中文` under `## Unreleased`. ### 中文 +- 保留 Qoder 用户消息及工具结果中的图片，并在完整、有序的工具结果批次之后发送工具图片。 + - OpenAI、Anthropic 与 Responses 流式转发会保留上游的类型化错误，避免无效的 Devin 请求被错误地冷却账号，同时传输中断仍可重试 - Devin 的缓存读取与写入会显示在 OpenAI 兼容 usage 中，prompt 总数包含全部上游输入 token - 新增手动更新命令，提取 descriptor 并生成 Devin 聊天与账号状态 protobuf 类型；构建与 CI 直接使用已提交的 Go 文件，不下载发行包或重新生成 schema diff --git a/worker/src/plaintext.mjs b/worker/src/plaintext.mjs index c65e0b4..50e86b4 100644 --- a/worker/src/plaintext.mjs +++ b/worker/src/plaintext.mjs @@ -235,6 +235,7 @@ function normalizeMessagesForUpstream(messages = []) { const flushToolResults = () => { if (!pendingToolResults.length) return; + const images = []; const order = new Map(currentBatch.map((id, index) => [id, index])); pendingToolResults.sort((left, right) => { const leftOrder = order.has(left.toolCallId) ? order.get(left.toolCallId) : Number.MAX_SAFE_INTEGER; @@ -245,8 +246,12 @@ function normalizeMessagesForUpstream(messages = []) { const belongsToCurrentBatch = currentBatch.length > 0 ? order.has(result.toolCallId) : callsById.has(result.toolCallId); - if (belongsToCurrentBatch) normalized.push(result.message); + if (belongsToCurrentBatch) { + normalized.push(result.message); + images.push(...result.images); + } } + if (images.length) normalized.push({ role: "user", content: images }); pendingToolResults = []; }; @@ -288,14 +293,26 @@ function normalizeMessagesForUpstream(messages = []) { tool_call_id: toolCallId, }; if (message?.name) out.name = String(message.name); - pendingToolResults.push({ message: out, toolCallId }); + const images = (Array.isArray(message.content) ? message.content : []).flatMap((part) => { + if (part?.type === "image_url") { + return normalizeContentForUpstream([part]); + } + if (part?.type === "image" && typeof part.data === "string" && part.data && (part.mimeType || part.mime_type)) { + return [{ type: "image_url", image_url: { url: `data:${part.mimeType || part.mime_type};base64,${part.data}` } }]; + } + return []; + }); + pendingToolResults.push({ message: out, toolCallId, images }); continue; } flushToolResults(); currentBatch = []; consumedBatchIds = new Set(); - normalized.push({ role, content: contentToString(message?.content) }); + normalized.push({ + role, + content: role === "user" ? normalizeContentForUpstream(message?.content) : contentToString(message?.content), + }); } flushToolResults(); diff --git a/worker/test/plaintext.test.mjs b/worker/test/plaintext.test.mjs index 147a05f..e017464 100644 --- a/worker/test/plaintext.test.mjs +++ b/worker/test/plaintext.test.mjs @@ -36,6 +36,52 @@ test("normalizes model ids only for stable public/settings keys", () => { assert.equal(canonicalModelID("GLM-5.2"), "glm-5.2"); }); +test("preserves user images in the final Qoder request without changing system text", () => { + const images = [ + { type: "image_url", image_url: { url: "data:image/png;base64,cHJvYmU=", detail: "high" } }, + { type: "image_url", image_url: { url: "https://example.com/probe.png" } }, + ]; + const body = buildPlainChatBody({ + model: "deepseek-flash", + messages: [ + { role: "system", content: [{ type: "text", text: "Inspect images" }] }, + { role: "user", content: [{ type: "text", text: "Compare these" }, ...images] }, + ], + }); + assert.equal(body.system, "Inspect images"); + assert.deepEqual(body.messages[1].content, [{ type: "text", text: "Compare these" }, ...images]); +}); + +test("preserves tool images after the complete parallel result batch and drops orphan images", () => { + const image = { type: "image_url", image_url: { url: "data:image/png;base64,cHJvYmU=" } }; + const body = buildPlainChatBody({ model: "deepseek-flash", messages: [ + { role: "assistant", content: "", tool_calls: [ + { id: "a", function: { name: "view", arguments: "{}" } }, + { id: "b", function: { name: "view", arguments: "{}" } }, + ] }, + { role: "tool", tool_call_id: "orphan", content: [{ type: "image_url", image_url: { url: "orphan" } }] }, + { role: "tool", tool_call_id: "b", content: [image] }, + { role: "tool", tool_call_id: "a", content: [{ type: "image", mimeType: "image/png", data: "bWNw" }] }, + { role: "user", content: "Describe the results" }, + ] }); + assert.deepEqual(body.messages.map(x => x.role), ["assistant", "tool", "tool", "user", "user"]); + assert.deepEqual(body.messages.slice(1, 3).map(x => x.tool_call_id), ["a", "b"]); + assert.deepEqual(body.messages[3].content, [ + { type: "image_url", image_url: { url: "data:image/png;base64,bWNw" } }, image, + ]); + assert.equal(JSON.stringify(body).includes('"url":"orphan"'), false); +}); + +test("preserves images extracted from Responses tool output into a user message", () => { + const image = { type: "image_url", image_url: { url: "data:image/png;base64,cHJvYmU=" } }; + const body = buildPlainChatBody({ model: "deepseek-flash", messages: [ + { role: "assistant", content: "", tool_calls: [{ id: "a", function: { name: "view", arguments: "{}" } }] }, + { role: "tool", tool_call_id: "a", content: "Image attached" }, + { role: "user", content: [image] }, + ] }); + assert.deepEqual(body.messages[2], { role: "user", content: [image] }); +}); + test("detects reasoning flags from OpenAI-style fields", () => { assert.equal(wantsReasoning({ enable_thinking: true }), true); assert.equal(wantsReasoning({ reasoning_effort: "high" }), true);