From 4935751cc379b2af14021021a8c2e523b4f22c0f Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Mon, 28 Sep 2026 20:02:52 +0200 Subject: [PATCH 1/4] fix(task): lead each request user message with its tool_result blocks History saved with image blocks between tool_result blocks (read_file on an image alongside another tool call in the same turn) is rejected by Anthropic with a 400 on every retry, leaving the task stuck. Reorder tool_result blocks to the front of each user message when shaping the API request, after merging consecutive user messages. Stored history is unchanged, so tasks already saved with the broken shape recover on their next retry. This complements #1705, which stops new messages from being built with the interleaved shape. Fixes #1774 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/core/task/Task.ts | 7 +- src/core/task/__tests__/Task.spec.ts | 57 +++++++++++++ .../__tests__/orderToolResultsFirst.spec.ts | 85 +++++++++++++++++++ src/core/task/orderToolResultsFirst.ts | 33 +++++++ 4 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 src/core/task/__tests__/orderToolResultsFirst.spec.ts create mode 100644 src/core/task/orderToolResultsFirst.ts diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index e573a7f107..4988376380 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -140,6 +140,7 @@ import { type AutoDenyDetail, AutoApprovalHandler, checkAutoApproval } from "../ import { MessageManager } from "../message-manager" import { validateAndFixToolResultIds } from "./validateToolResultIds" import { mergeConsecutiveApiMessages } from "./mergeConsecutiveApiMessages" +import { orderToolResultsFirst } from "./orderToolResultsFirst" import { prepareApiConversationMessage } from "./apiConversationHistory" import { shouldAddUserMessageToHistory } from "./messageCounting" import { type TaskExecutionContext } from "./providerHandoff" @@ -5830,7 +5831,11 @@ export class Task extends EventEmitter implements TaskLike { const messagesSinceLastSummary = getMessagesSinceLastSummary(effectiveHistory) // For API only: merge consecutive user messages (excludes summary messages per // mergeConsecutiveApiMessages implementation) without mutating stored history. - const mergedForApi = mergeConsecutiveApiMessages(messagesSinceLastSummary, { roles: ["user"] }) + // Then lead each user message with its tool_result blocks, which also repairs history saved + // with images between results (#1774) so a stuck task can retry successfully. + const mergedForApi = orderToolResultsFirst( + mergeConsecutiveApiMessages(messagesSinceLastSummary, { roles: ["user"] }), + ) const messagesWithoutImages = maybeRemoveImageBlocks(mergedForApi, this.api) const cleanConversationHistory = this.buildCleanConversationHistory( messagesWithoutImages as ApiMessage[], diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index ad07230204..2c5e733b25 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -2971,6 +2971,63 @@ describe("Cline", () => { expect(Object.keys(cleanConversationHistory[0]!)).toEqual(["role", "content"]) }) + it("sends tool_result blocks first when stored history has images between them (#1774)", async () => { + const cline = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(getTaskTestAccess(cline), "getSystemPrompt").mockResolvedValue("mock system prompt") + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { type: "text", text: "response" } + }, + } as AsyncGenerator + const createMessageSpy = vi.spyOn(cline.api, "createMessage").mockReturnValue(mockStream) + + // History as persisted by a turn that read an image alongside another tool call. + cline.apiConversationHistory = [ + { role: "user", content: [{ type: "text", text: "read the logo and list files" }], ts: 1 }, + { + role: "assistant", + content: [ + { type: "tool_use", id: "read", name: "read_file", input: { path: "logo.jpg" } }, + { type: "tool_use", id: "cmd", name: "execute_command", input: { command: "ls" } }, + ], + ts: 2, + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" }, + { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, + { type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" }, + { type: "text", text: "" }, + ], + ts: 3, + }, + ] + + const iterator = cline.attemptApiRequest(0) + await iterator.next() + + const [, sentHistory] = requireDefined(createMessageSpy.mock.calls[0]) + const lastUserContent = requireDefined(sentHistory.at(-1)).content as Array<{ type: string }> + expect(lastUserContent.slice(0, 2)).toMatchObject([ + { type: "tool_result", tool_use_id: "read" }, + { type: "tool_result", tool_use_id: "cmd" }, + ]) + expect(lastUserContent.slice(2).some((block) => block.type === "tool_result")).toBe(false) + // The stored history keeps its original shape. + expect(requireDefined(cline.apiConversationHistory.at(-1)).content).toMatchObject([ + { type: "tool_result" }, + { type: "image" }, + { type: "tool_result" }, + { type: "text" }, + ]) + }) + it("should shape image blocks for API compatibility before request construction", async () => { const conversationHistory = [ { diff --git a/src/core/task/__tests__/orderToolResultsFirst.spec.ts b/src/core/task/__tests__/orderToolResultsFirst.spec.ts new file mode 100644 index 0000000000..1399837ea8 --- /dev/null +++ b/src/core/task/__tests__/orderToolResultsFirst.spec.ts @@ -0,0 +1,85 @@ +// npx vitest run core/task/__tests__/orderToolResultsFirst.spec.ts + +import type { Anthropic } from "@anthropic-ai/sdk" + +import type { ApiMessage } from "../../task-persistence" +import { mergeConsecutiveApiMessages } from "../mergeConsecutiveApiMessages" +import { orderToolResultsFirst } from "../orderToolResultsFirst" + +const toolResult = (id: string, content = `result ${id}`): Anthropic.Messages.ToolResultBlockParam => ({ + type: "tool_result", + tool_use_id: id, + content, +}) +const image = (data: string): Anthropic.Messages.ImageBlockParam => ({ + type: "image", + source: { type: "base64", media_type: "image/jpeg", data }, +}) +const text = (value: string): Anthropic.Messages.TextBlockParam => ({ type: "text", text: value }) + +describe("orderToolResultsFirst", () => { + it("repairs the persisted #1774 shape so every tool_result leads the message", () => { + // [tool_result, image, tool_result, text] as saved when read_file on an image ran + // alongside another tool call in the same turn. + const stuck: ApiMessage = { + role: "user", + ts: 16, + content: [toolResult("read"), image("jpeg"), toolResult("cmd"), text("")], + } + + const [repaired] = orderToolResultsFirst([stuck]) + + expect(repaired.content).toEqual([ + toolResult("read"), + toolResult("cmd"), + image("jpeg"), + text(""), + ]) + expect(repaired.ts).toBe(16) + // Stored history is untouched. + expect(stuck.content).toEqual([ + toolResult("read"), + image("jpeg"), + toolResult("cmd"), + text(""), + ]) + }) + + it("keeps the order of results and of other blocks with several interleaved images", () => { + const [repaired] = orderToolResultsFirst([ + { + role: "user", + content: [toolResult("a"), toolResult("b"), image("1"), toolResult("c"), image("2"), text("t")], + }, + ]) + + expect(repaired.content).toEqual([ + toolResult("a"), + toolResult("b"), + toolResult("c"), + image("1"), + image("2"), + text("t"), + ]) + }) + + it("repairs interleaving introduced by merging consecutive user messages", () => { + const merged = mergeConsecutiveApiMessages([ + { role: "user", content: [toolResult("a"), text("note")] }, + { role: "user", content: [toolResult("b")] }, + ]) + + expect(orderToolResultsFirst(merged)[0].content).toEqual([toolResult("a"), toolResult("b"), text("note")]) + }) + + it("returns already-ordered, string, and assistant messages unchanged", () => { + const ordered: ApiMessage = { role: "user", content: [toolResult("a"), image("1"), text("t")] } + const plain: ApiMessage = { role: "user", content: "hello" } + const noResults: ApiMessage = { role: "user", content: [text("a"), image("1")] } + const assistant: ApiMessage = { role: "assistant", content: [text("a")] } + + const out = orderToolResultsFirst([ordered, plain, noResults, assistant]) + + out.forEach((message, index) => expect(message).toBe([ordered, plain, noResults, assistant][index])) + }) +}) diff --git a/src/core/task/orderToolResultsFirst.ts b/src/core/task/orderToolResultsFirst.ts new file mode 100644 index 0000000000..96a35b7e32 --- /dev/null +++ b/src/core/task/orderToolResultsFirst.ts @@ -0,0 +1,33 @@ +import type { ApiMessage } from "../task-persistence" + +/** + * Moves `tool_result` blocks to the front of each user message, keeping the relative order of + * the results and of the remaining blocks. + * + * Anthropic requires the results for the previous turn's `tool_use` blocks to lead the next user + * message. History saved with image blocks between results (#1307, #1774), or produced by merging + * consecutive user messages, is otherwise rejected with a 400 on every retry. + * + * Used for *API request shaping only* (do not use for storage). Messages already in order are + * returned unchanged. + */ +export function orderToolResultsFirst(messages: ApiMessage[]): ApiMessage[] { + return messages.map((message) => { + if (message.role !== "user" || !Array.isArray(message.content)) { + return message + } + const firstOther = message.content.findIndex((block) => block.type !== "tool_result") + const hasLaterResult = + firstOther !== -1 && message.content.slice(firstOther).some((block) => block.type === "tool_result") + if (!hasLaterResult) { + return message + } + return { + ...message, + content: [ + ...message.content.filter((block) => block.type === "tool_result"), + ...message.content.filter((block) => block.type !== "tool_result"), + ], + } + }) +} From e5de8f171ed5b4dcb3592d965836ff0012609ce9 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Tue, 20 Oct 2026 15:44:19 +0200 Subject: [PATCH 2/4] test(task): assert the complete outgoing user content for reordered tool results Co-Authored-By: Claude Sonnet 5.5 --- src/core/task/__tests__/Task.spec.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index 2c5e733b25..9a05b68aed 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -2979,6 +2979,16 @@ describe("Cline", () => { startTask: false, }) vi.spyOn(getTaskTestAccess(cline), "getSystemPrompt").mockResolvedValue("mock system prompt") + // Image-capable model so the image block is kept in the outgoing request. + vi.spyOn(cline.api, "getModel").mockReturnValue({ + id: "claude-3-sonnet", + info: { + supportsImages: true, + supportsPromptCache: false, + contextWindow: 200000, + maxTokens: 4096, + } as ModelInfo, + }) const mockStream = { async *[Symbol.asyncIterator]() { yield { type: "text", text: "response" } @@ -3014,11 +3024,13 @@ describe("Cline", () => { const [, sentHistory] = requireDefined(createMessageSpy.mock.calls[0]) const lastUserContent = requireDefined(sentHistory.at(-1)).content as Array<{ type: string }> - expect(lastUserContent.slice(0, 2)).toMatchObject([ - { type: "tool_result", tool_use_id: "read" }, - { type: "tool_result", tool_use_id: "cmd" }, + // Both results lead, followed by the image and the environment details, none lost. + expect(lastUserContent).toEqual([ + { type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" }, + { type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" }, + { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, + { type: "text", text: "" }, ]) - expect(lastUserContent.slice(2).some((block) => block.type === "tool_result")).toBe(false) // The stored history keeps its original shape. expect(requireDefined(cline.apiConversationHistory.at(-1)).content).toMatchObject([ { type: "tool_result" }, From 2cfa4b37f6b4eb0be7f010b4e409b09794a29141 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Sun, 25 Oct 2026 11:35:59 +0100 Subject: [PATCH 3/4] fix(task): nest an image that follows a tool_result into that result's content Co-Authored-By: Claude Sonnet 5.5 --- src/core/task/__tests__/Task.spec.ts | 12 ++++-- .../__tests__/orderToolResultsFirst.spec.ts | 42 ++++++++++++++++--- src/core/task/orderToolResultsFirst.ts | 37 ++++++++++++---- 3 files changed, 75 insertions(+), 16 deletions(-) diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index 9a05b68aed..1dda1555a6 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -3024,11 +3024,17 @@ describe("Cline", () => { const [, sentHistory] = requireDefined(createMessageSpy.mock.calls[0]) const lastUserContent = requireDefined(sentHistory.at(-1)).content as Array<{ type: string }> - // Both results lead, followed by the image and the environment details, none lost. + // Both results lead; the image sits inside the read_file result; the environment details follow. expect(lastUserContent).toEqual([ - { type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" }, + { + type: "tool_result", + tool_use_id: "read", + content: [ + { type: "text", text: "File: logo.jpg" }, + { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, + ], + }, { type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" }, - { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, { type: "text", text: "" }, ]) // The stored history keeps its original shape. diff --git a/src/core/task/__tests__/orderToolResultsFirst.spec.ts b/src/core/task/__tests__/orderToolResultsFirst.spec.ts index 1399837ea8..6c6b93595a 100644 --- a/src/core/task/__tests__/orderToolResultsFirst.spec.ts +++ b/src/core/task/__tests__/orderToolResultsFirst.spec.ts @@ -16,6 +16,11 @@ const image = (data: string): Anthropic.Messages.ImageBlockParam => ({ source: { type: "base64", media_type: "image/jpeg", data }, }) const text = (value: string): Anthropic.Messages.TextBlockParam => ({ type: "text", text: value }) +const nested = (id: string, content: Anthropic.Messages.ToolResultBlockParam["content"]) => ({ + type: "tool_result" as const, + tool_use_id: id, + content, +}) describe("orderToolResultsFirst", () => { it("repairs the persisted #1774 shape so every tool_result leads the message", () => { @@ -29,10 +34,10 @@ describe("orderToolResultsFirst", () => { const [repaired] = orderToolResultsFirst([stuck]) + // The image moves into the read_file result, the shape the Anthropic API expects. expect(repaired.content).toEqual([ - toolResult("read"), + nested("read", [text("result read"), image("jpeg")]), toolResult("cmd"), - image("jpeg"), text(""), ]) expect(repaired.ts).toBe(16) @@ -55,14 +60,39 @@ describe("orderToolResultsFirst", () => { expect(repaired.content).toEqual([ toolResult("a"), - toolResult("b"), - toolResult("c"), - image("1"), - image("2"), + nested("b", [text("result b"), image("1")]), + nested("c", [text("result c"), image("2")]), text("t"), ]) }) + it("appends an image to a result whose content is already an array", () => { + const withArray = nested("a", [text("first"), text("second")]) + + const [repaired] = orderToolResultsFirst([{ role: "user", content: [withArray, image("1"), toolResult("b")] }]) + + expect(repaired.content).toEqual([nested("a", [text("first"), text("second"), image("1")]), toolResult("b")]) + // The original result block is not mutated. + expect(withArray.content).toEqual([text("first"), text("second")]) + }) + + it("keeps an image standalone when no result directly precedes it", () => { + const [repaired] = orderToolResultsFirst([ + { + role: "user", + content: [image("lead"), toolResult("a"), text("note"), image("after-text"), toolResult("b")], + }, + ]) + + expect(repaired.content).toEqual([ + toolResult("a"), + toolResult("b"), + image("lead"), + text("note"), + image("after-text"), + ]) + }) + it("repairs interleaving introduced by merging consecutive user messages", () => { const merged = mergeConsecutiveApiMessages([ { role: "user", content: [toolResult("a"), text("note")] }, diff --git a/src/core/task/orderToolResultsFirst.ts b/src/core/task/orderToolResultsFirst.ts index 96a35b7e32..24a0059e65 100644 --- a/src/core/task/orderToolResultsFirst.ts +++ b/src/core/task/orderToolResultsFirst.ts @@ -1,8 +1,19 @@ +import type { Anthropic } from "@anthropic-ai/sdk" + import type { ApiMessage } from "../task-persistence" +type Block = Anthropic.Messages.ContentBlockParam +type ToolResultBlock = Anthropic.Messages.ToolResultBlockParam + +function toContentArray(content: ToolResultBlock["content"]): Exclude { + if (typeof content === "string") return [{ type: "text", text: content }] + return content ?? [] +} + /** * Moves `tool_result` blocks to the front of each user message, keeping the relative order of - * the results and of the remaining blocks. + * the results and of the remaining blocks. An image that directly follows a result (as when + * `read_file` returns an image) is nested into that result's `content`, where Anthropic expects it. * * Anthropic requires the results for the previous turn's `tool_use` blocks to lead the next user * message. History saved with image blocks between results (#1307, #1774), or produced by merging @@ -22,12 +33,24 @@ export function orderToolResultsFirst(messages: ApiMessage[]): ApiMessage[] { if (!hasLaterResult) { return message } - return { - ...message, - content: [ - ...message.content.filter((block) => block.type === "tool_result"), - ...message.content.filter((block) => block.type !== "tool_result"), - ], + + const results: ToolResultBlock[] = [] + const rest: Block[] = [] + // Index of the result an immediately following image belongs to; reset by any other block. + let owner = -1 + for (const block of message.content) { + if (block.type === "tool_result") { + results.push(block) + owner = results.length - 1 + } else if (block.type === "image" && owner !== -1) { + const current = results[owner] + results[owner] = { ...current, content: [...toContentArray(current.content), block] } + } else { + owner = -1 + rest.push(block) + } } + + return { ...message, content: [...results, ...rest] } }) } From 04da4c62e8f3862fd5e501f14798b4f6f5c1591d Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Wed, 4 Nov 2026 16:58:19 +0100 Subject: [PATCH 4/4] fix(task): keep repaired tool_result images top-level so every provider sends them Nesting the image into the preceding tool_result dropped it for the OpenAI chat, AI SDK, and OpenAI Codex converters, and hid it from maybeRemoveImageBlocks for text-only models. Partition results first instead, the shape live turns already store (#1307). --- src/core/task/__tests__/Task.spec.ts | 148 +++++++++--------- .../__tests__/orderToolResultsFirst.spec.ts | 89 +++++++++-- src/core/task/orderToolResultsFirst.ts | 35 +---- 3 files changed, 162 insertions(+), 110 deletions(-) diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index ae451b4b61..f5337f5a04 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -3027,80 +3027,86 @@ describe("Cline", () => { expect(Object.keys(cleanConversationHistory[0]!)).toEqual(["role", "content"]) }) - it("sends tool_result blocks first when stored history has images between them (#1774)", async () => { - const cline = new Task({ - provider: mockProvider, - apiConfiguration: mockApiConfig, - task: "test task", - startTask: false, - }) - vi.spyOn(getTaskTestAccess(cline), "getSystemPrompt").mockResolvedValue("mock system prompt") - // Image-capable model so the image block is kept in the outgoing request. - vi.spyOn(cline.api, "getModel").mockReturnValue({ - id: "claude-3-sonnet", - info: { - supportsImages: true, - supportsPromptCache: false, - contextWindow: 200000, - maxTokens: 4096, - } as ModelInfo, - }) - const mockStream = { - async *[Symbol.asyncIterator]() { - yield { type: "text", text: "response" } - }, - } as AsyncGenerator - const createMessageSpy = vi.spyOn(cline.api, "createMessage").mockReturnValue(mockStream) + it.each([ + { + supportsImages: true, + imageBlock: { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, + }, + { + // A text-only model must not receive the image, including after the repair. + supportsImages: false, + imageBlock: { type: "text", text: "[Referenced image in conversation]" }, + }, + ])( + "sends tool_result blocks first when stored history has images between them (#1774, supportsImages: $supportsImages)", + async ({ supportsImages, imageBlock }) => { + const cline = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(getTaskTestAccess(cline), "getSystemPrompt").mockResolvedValue("mock system prompt") + vi.spyOn(cline.api, "getModel").mockReturnValue({ + id: "claude-3-sonnet", + info: { + supportsImages, + supportsPromptCache: false, + contextWindow: 200000, + maxTokens: 4096, + } as ModelInfo, + }) + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { type: "text", text: "response" } + }, + } as AsyncGenerator + const createMessageSpy = vi.spyOn(cline.api, "createMessage").mockReturnValue(mockStream) - // History as persisted by a turn that read an image alongside another tool call. - cline.apiConversationHistory = [ - { role: "user", content: [{ type: "text", text: "read the logo and list files" }], ts: 1 }, - { - role: "assistant", - content: [ - { type: "tool_use", id: "read", name: "read_file", input: { path: "logo.jpg" } }, - { type: "tool_use", id: "cmd", name: "execute_command", input: { command: "ls" } }, - ], - ts: 2, - }, - { - role: "user", - content: [ - { type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" }, - { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, - { type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" }, - { type: "text", text: "" }, - ], - ts: 3, - }, - ] + // History as persisted by a turn that read an image alongside another tool call. + cline.apiConversationHistory = [ + { role: "user", content: [{ type: "text", text: "read the logo and list files" }], ts: 1 }, + { + role: "assistant", + content: [ + { type: "tool_use", id: "read", name: "read_file", input: { path: "logo.jpg" } }, + { type: "tool_use", id: "cmd", name: "execute_command", input: { command: "ls" } }, + ], + ts: 2, + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" }, + { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, + { type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" }, + { type: "text", text: "" }, + ], + ts: 3, + }, + ] - const iterator = cline.attemptApiRequest(0) - await iterator.next() + const iterator = cline.attemptApiRequest(0) + await iterator.next() - const [, sentHistory] = requireDefined(createMessageSpy.mock.calls[0]) - const lastUserContent = requireDefined(sentHistory.at(-1)).content as Array<{ type: string }> - // Both results lead; the image sits inside the read_file result; the environment details follow. - expect(lastUserContent).toEqual([ - { - type: "tool_result", - tool_use_id: "read", - content: [ - { type: "text", text: "File: logo.jpg" }, - { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, - ], - }, - { type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" }, - { type: "text", text: "" }, - ]) - // The stored history keeps its original shape. - expect(requireDefined(cline.apiConversationHistory.at(-1)).content).toMatchObject([ - { type: "tool_result" }, - { type: "image" }, - { type: "tool_result" }, - { type: "text" }, - ]) - }) + const [, sentHistory] = requireDefined(createMessageSpy.mock.calls[0]) + const lastUserContent = requireDefined(sentHistory.at(-1)).content as Array<{ type: string }> + // Both results lead; the image (or its text stand-in) and the environment details follow. + expect(lastUserContent).toEqual([ + { type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" }, + { type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" }, + imageBlock, + { type: "text", text: "" }, + ]) + // The stored history keeps its original shape. + expect(requireDefined(cline.apiConversationHistory.at(-1)).content).toMatchObject([ + { type: "tool_result" }, + { type: "image" }, + { type: "tool_result" }, + { type: "text" }, + ]) + }, + ) it("should shape image blocks for API compatibility before request construction", async () => { const conversationHistory = [ diff --git a/src/core/task/__tests__/orderToolResultsFirst.spec.ts b/src/core/task/__tests__/orderToolResultsFirst.spec.ts index 6c6b93595a..e6a33e21a5 100644 --- a/src/core/task/__tests__/orderToolResultsFirst.spec.ts +++ b/src/core/task/__tests__/orderToolResultsFirst.spec.ts @@ -2,6 +2,11 @@ import type { Anthropic } from "@anthropic-ai/sdk" +import type { ApiHandler } from "../../../api" +import { OpenAiCodexHandler } from "../../../api/providers/openai-codex" +import { convertToAiSdkMessages } from "../../../api/transform/ai-sdk" +import { maybeRemoveImageBlocks } from "../../../api/transform/image-cleaning" +import { convertToOpenAiMessages } from "../../../api/transform/openai-format" import type { ApiMessage } from "../../task-persistence" import { mergeConsecutiveApiMessages } from "../mergeConsecutiveApiMessages" import { orderToolResultsFirst } from "../orderToolResultsFirst" @@ -22,10 +27,24 @@ const nested = (id: string, content: Anthropic.Messages.ToolResultBlockParam["co content, }) +// [tool_result, image, tool_result, text] as saved when read_file on an image ran alongside +// another tool call in the same turn (#1774). +const stuckTurn = (): ApiMessage[] => [ + { + role: "assistant", + content: [ + { type: "tool_use", id: "read", name: "read_file", input: { path: "logo.jpg" } }, + { type: "tool_use", id: "cmd", name: "execute_command", input: { command: "ls" } }, + ], + }, + { + role: "user", + content: [toolResult("read"), image("jpeg"), toolResult("cmd"), text("")], + }, +] + describe("orderToolResultsFirst", () => { it("repairs the persisted #1774 shape so every tool_result leads the message", () => { - // [tool_result, image, tool_result, text] as saved when read_file on an image ran - // alongside another tool call in the same turn. const stuck: ApiMessage = { role: "user", ts: 16, @@ -34,10 +53,11 @@ describe("orderToolResultsFirst", () => { const [repaired] = orderToolResultsFirst([stuck]) - // The image moves into the read_file result, the shape the Anthropic API expects. + // The results lead; the image stays a top-level block, as live turns store it (#1307). expect(repaired.content).toEqual([ - nested("read", [text("result read"), image("jpeg")]), + toolResult("read"), toolResult("cmd"), + image("jpeg"), text(""), ]) expect(repaired.ts).toBe(16) @@ -60,23 +80,24 @@ describe("orderToolResultsFirst", () => { expect(repaired.content).toEqual([ toolResult("a"), - nested("b", [text("result b"), image("1")]), - nested("c", [text("result c"), image("2")]), + toolResult("b"), + toolResult("c"), + image("1"), + image("2"), text("t"), ]) }) - it("appends an image to a result whose content is already an array", () => { + it("leaves the content of every result block untouched", () => { const withArray = nested("a", [text("first"), text("second")]) const [repaired] = orderToolResultsFirst([{ role: "user", content: [withArray, image("1"), toolResult("b")] }]) - expect(repaired.content).toEqual([nested("a", [text("first"), text("second"), image("1")]), toolResult("b")]) - // The original result block is not mutated. + expect(repaired.content).toEqual([withArray, toolResult("b"), image("1")]) expect(withArray.content).toEqual([text("first"), text("second")]) }) - it("keeps an image standalone when no result directly precedes it", () => { + it("moves leading and mid-message blocks after the results in their original order", () => { const [repaired] = orderToolResultsFirst([ { role: "user", @@ -112,4 +133,52 @@ describe("orderToolResultsFirst", () => { out.forEach((message, index) => expect(message).toBe([ordered, plain, noResults, assistant][index])) }) + + describe("repaired images reach the provider request", () => { + const dataUrl = "data:image/jpeg;base64,jpeg" + + it("OpenAI chat format sends the image as an image_url part", () => { + const converted = convertToOpenAiMessages(orderToolResultsFirst(stuckTurn())) + + expect(converted.map((message) => message.role)).toEqual(["assistant", "tool", "tool", "user"]) + expect(converted.at(-1)?.content).toContainEqual({ type: "image_url", image_url: { url: dataUrl } }) + }) + + it("AI SDK format sends the image as an image part", () => { + const converted = convertToAiSdkMessages(orderToolResultsFirst(stuckTurn())) + const user = converted.find((message) => message.role === "user") + + expect(user?.content).toContainEqual({ type: "image", image: dataUrl, mimeType: "image/jpeg" }) + }) + + it("OpenAI Codex Responses input sends the image as an input_image part", () => { + const handler = new OpenAiCodexHandler({ apiModelId: "gpt-5.1-codex" }) + const input: unknown[] = handler["formatFullConversation"]("", orderToolResultsFirst(stuckTurn())) + + expect(input).toContainEqual({ + role: "user", + content: [ + { type: "input_image", image_url: dataUrl }, + { type: "input_text", text: "" }, + ], + }) + }) + + it("a model without image support gets no image block anywhere in the request", () => { + const textOnly: ApiHandler = { + getModel: vi.fn().mockReturnValue({ id: "text-only", info: { supportsImages: false } }), + createMessage: vi.fn(), + countTokens: vi.fn(), + getModelCacheScope: vi.fn(), + } + const [, user] = maybeRemoveImageBlocks(orderToolResultsFirst(stuckTurn()), textOnly) + + expect(user.content).toEqual([ + toolResult("read"), + toolResult("cmd"), + text("[Referenced image in conversation]"), + text(""), + ]) + }) + }) }) diff --git a/src/core/task/orderToolResultsFirst.ts b/src/core/task/orderToolResultsFirst.ts index 24a0059e65..8faecde182 100644 --- a/src/core/task/orderToolResultsFirst.ts +++ b/src/core/task/orderToolResultsFirst.ts @@ -1,19 +1,11 @@ -import type { Anthropic } from "@anthropic-ai/sdk" - import type { ApiMessage } from "../task-persistence" -type Block = Anthropic.Messages.ContentBlockParam -type ToolResultBlock = Anthropic.Messages.ToolResultBlockParam - -function toContentArray(content: ToolResultBlock["content"]): Exclude { - if (typeof content === "string") return [{ type: "text", text: content }] - return content ?? [] -} - /** * Moves `tool_result` blocks to the front of each user message, keeping the relative order of - * the results and of the remaining blocks. An image that directly follows a result (as when - * `read_file` returns an image) is nested into that result's `content`, where Anthropic expects it. + * the results and of the remaining blocks (images, text). This is the same partition + * `Task.pushToolResultToUserContent` applies to live turns (#1307), so repaired history has the + * shape every provider converter already handles: images stay top-level blocks after the results, + * where image-capable converters forward them and `maybeRemoveImageBlocks` can strip them. * * Anthropic requires the results for the previous turn's `tool_use` blocks to lead the next user * message. History saved with image blocks between results (#1307, #1774), or produced by merging @@ -34,23 +26,8 @@ export function orderToolResultsFirst(messages: ApiMessage[]): ApiMessage[] { return message } - const results: ToolResultBlock[] = [] - const rest: Block[] = [] - // Index of the result an immediately following image belongs to; reset by any other block. - let owner = -1 - for (const block of message.content) { - if (block.type === "tool_result") { - results.push(block) - owner = results.length - 1 - } else if (block.type === "image" && owner !== -1) { - const current = results[owner] - results[owner] = { ...current, content: [...toContentArray(current.content), block] } - } else { - owner = -1 - rest.push(block) - } - } - + const results = message.content.filter((block) => block.type === "tool_result") + const rest = message.content.filter((block) => block.type !== "tool_result") return { ...message, content: [...results, ...rest] } }) }