diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 90cf7bf7fb..2d4debaca3 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -141,6 +141,7 @@ import { type AutoDenyDetail, AutoApprovalHandler, checkAutoApproval } from "../ import { MessageManager } from "../message-manager" import { validateAndFixToolResultIds } from "./validateToolResultIds" import { mergeConsecutiveApiMessages } from "./mergeConsecutiveApiMessages" +import { orderToolResultsFirst } from "./orderToolResultsFirst" import { prepareApiConversationMessage } from "./apiConversationHistory" import { shouldAddUserMessageToHistory } from "./messageCounting" import { type TaskExecutionContext } from "./providerHandoff" @@ -5879,7 +5880,11 @@ export class Task extends EventEmitter implements TaskLike { const messagesSinceLastSummary = getMessagesSinceLastSummary(effectiveHistory) // For API only: merge consecutive user messages (excludes summary messages per // mergeConsecutiveApiMessages implementation) without mutating stored history. - const mergedForApi = mergeConsecutiveApiMessages(messagesSinceLastSummary, { roles: ["user"] }) + // Then lead each user message with its tool_result blocks, which also repairs history saved + // with images between results (#1774) so a stuck task can retry successfully. + const mergedForApi = orderToolResultsFirst( + mergeConsecutiveApiMessages(messagesSinceLastSummary, { roles: ["user"] }), + ) const messagesWithoutImages = maybeRemoveImageBlocks(mergedForApi, this.api) const cleanConversationHistory = this.buildCleanConversationHistory( messagesWithoutImages as ApiMessage[], diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index b73b9afea7..f5337f5a04 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -3027,6 +3027,87 @@ describe("Cline", () => { expect(Object.keys(cleanConversationHistory[0]!)).toEqual(["role", "content"]) }) + it.each([ + { + supportsImages: true, + imageBlock: { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, + }, + { + // A text-only model must not receive the image, including after the repair. + supportsImages: false, + imageBlock: { type: "text", text: "[Referenced image in conversation]" }, + }, + ])( + "sends tool_result blocks first when stored history has images between them (#1774, supportsImages: $supportsImages)", + async ({ supportsImages, imageBlock }) => { + const cline = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(getTaskTestAccess(cline), "getSystemPrompt").mockResolvedValue("mock system prompt") + vi.spyOn(cline.api, "getModel").mockReturnValue({ + id: "claude-3-sonnet", + info: { + supportsImages, + supportsPromptCache: false, + contextWindow: 200000, + maxTokens: 4096, + } as ModelInfo, + }) + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { type: "text", text: "response" } + }, + } as AsyncGenerator + const createMessageSpy = vi.spyOn(cline.api, "createMessage").mockReturnValue(mockStream) + + // History as persisted by a turn that read an image alongside another tool call. + cline.apiConversationHistory = [ + { role: "user", content: [{ type: "text", text: "read the logo and list files" }], ts: 1 }, + { + role: "assistant", + content: [ + { type: "tool_use", id: "read", name: "read_file", input: { path: "logo.jpg" } }, + { type: "tool_use", id: "cmd", name: "execute_command", input: { command: "ls" } }, + ], + ts: 2, + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" }, + { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } }, + { type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" }, + { type: "text", text: "" }, + ], + ts: 3, + }, + ] + + const iterator = cline.attemptApiRequest(0) + await iterator.next() + + const [, sentHistory] = requireDefined(createMessageSpy.mock.calls[0]) + const lastUserContent = requireDefined(sentHistory.at(-1)).content as Array<{ type: string }> + // Both results lead; the image (or its text stand-in) and the environment details follow. + expect(lastUserContent).toEqual([ + { type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" }, + { type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" }, + imageBlock, + { type: "text", text: "" }, + ]) + // The stored history keeps its original shape. + expect(requireDefined(cline.apiConversationHistory.at(-1)).content).toMatchObject([ + { type: "tool_result" }, + { type: "image" }, + { type: "tool_result" }, + { type: "text" }, + ]) + }, + ) + it("should shape image blocks for API compatibility before request construction", async () => { const conversationHistory = [ { diff --git a/src/core/task/__tests__/orderToolResultsFirst.spec.ts b/src/core/task/__tests__/orderToolResultsFirst.spec.ts new file mode 100644 index 0000000000..e6a33e21a5 --- /dev/null +++ b/src/core/task/__tests__/orderToolResultsFirst.spec.ts @@ -0,0 +1,184 @@ +// npx vitest run core/task/__tests__/orderToolResultsFirst.spec.ts + +import type { Anthropic } from "@anthropic-ai/sdk" + +import type { ApiHandler } from "../../../api" +import { OpenAiCodexHandler } from "../../../api/providers/openai-codex" +import { convertToAiSdkMessages } from "../../../api/transform/ai-sdk" +import { maybeRemoveImageBlocks } from "../../../api/transform/image-cleaning" +import { convertToOpenAiMessages } from "../../../api/transform/openai-format" +import type { ApiMessage } from "../../task-persistence" +import { mergeConsecutiveApiMessages } from "../mergeConsecutiveApiMessages" +import { orderToolResultsFirst } from "../orderToolResultsFirst" + +const toolResult = (id: string, content = `result ${id}`): Anthropic.Messages.ToolResultBlockParam => ({ + type: "tool_result", + tool_use_id: id, + content, +}) +const image = (data: string): Anthropic.Messages.ImageBlockParam => ({ + type: "image", + source: { type: "base64", media_type: "image/jpeg", data }, +}) +const text = (value: string): Anthropic.Messages.TextBlockParam => ({ type: "text", text: value }) +const nested = (id: string, content: Anthropic.Messages.ToolResultBlockParam["content"]) => ({ + type: "tool_result" as const, + tool_use_id: id, + content, +}) + +// [tool_result, image, tool_result, text] as saved when read_file on an image ran alongside +// another tool call in the same turn (#1774). +const stuckTurn = (): ApiMessage[] => [ + { + role: "assistant", + content: [ + { type: "tool_use", id: "read", name: "read_file", input: { path: "logo.jpg" } }, + { type: "tool_use", id: "cmd", name: "execute_command", input: { command: "ls" } }, + ], + }, + { + role: "user", + content: [toolResult("read"), image("jpeg"), toolResult("cmd"), text("")], + }, +] + +describe("orderToolResultsFirst", () => { + it("repairs the persisted #1774 shape so every tool_result leads the message", () => { + const stuck: ApiMessage = { + role: "user", + ts: 16, + content: [toolResult("read"), image("jpeg"), toolResult("cmd"), text("")], + } + + const [repaired] = orderToolResultsFirst([stuck]) + + // The results lead; the image stays a top-level block, as live turns store it (#1307). + expect(repaired.content).toEqual([ + toolResult("read"), + toolResult("cmd"), + image("jpeg"), + text(""), + ]) + expect(repaired.ts).toBe(16) + // Stored history is untouched. + expect(stuck.content).toEqual([ + toolResult("read"), + image("jpeg"), + toolResult("cmd"), + text(""), + ]) + }) + + it("keeps the order of results and of other blocks with several interleaved images", () => { + const [repaired] = orderToolResultsFirst([ + { + role: "user", + content: [toolResult("a"), toolResult("b"), image("1"), toolResult("c"), image("2"), text("t")], + }, + ]) + + expect(repaired.content).toEqual([ + toolResult("a"), + toolResult("b"), + toolResult("c"), + image("1"), + image("2"), + text("t"), + ]) + }) + + it("leaves the content of every result block untouched", () => { + const withArray = nested("a", [text("first"), text("second")]) + + const [repaired] = orderToolResultsFirst([{ role: "user", content: [withArray, image("1"), toolResult("b")] }]) + + expect(repaired.content).toEqual([withArray, toolResult("b"), image("1")]) + expect(withArray.content).toEqual([text("first"), text("second")]) + }) + + it("moves leading and mid-message blocks after the results in their original order", () => { + const [repaired] = orderToolResultsFirst([ + { + role: "user", + content: [image("lead"), toolResult("a"), text("note"), image("after-text"), toolResult("b")], + }, + ]) + + expect(repaired.content).toEqual([ + toolResult("a"), + toolResult("b"), + image("lead"), + text("note"), + image("after-text"), + ]) + }) + + it("repairs interleaving introduced by merging consecutive user messages", () => { + const merged = mergeConsecutiveApiMessages([ + { role: "user", content: [toolResult("a"), text("note")] }, + { role: "user", content: [toolResult("b")] }, + ]) + + expect(orderToolResultsFirst(merged)[0].content).toEqual([toolResult("a"), toolResult("b"), text("note")]) + }) + + it("returns already-ordered, string, and assistant messages unchanged", () => { + const ordered: ApiMessage = { role: "user", content: [toolResult("a"), image("1"), text("t")] } + const plain: ApiMessage = { role: "user", content: "hello" } + const noResults: ApiMessage = { role: "user", content: [text("a"), image("1")] } + const assistant: ApiMessage = { role: "assistant", content: [text("a")] } + + const out = orderToolResultsFirst([ordered, plain, noResults, assistant]) + + out.forEach((message, index) => expect(message).toBe([ordered, plain, noResults, assistant][index])) + }) + + describe("repaired images reach the provider request", () => { + const dataUrl = "data:image/jpeg;base64,jpeg" + + it("OpenAI chat format sends the image as an image_url part", () => { + const converted = convertToOpenAiMessages(orderToolResultsFirst(stuckTurn())) + + expect(converted.map((message) => message.role)).toEqual(["assistant", "tool", "tool", "user"]) + expect(converted.at(-1)?.content).toContainEqual({ type: "image_url", image_url: { url: dataUrl } }) + }) + + it("AI SDK format sends the image as an image part", () => { + const converted = convertToAiSdkMessages(orderToolResultsFirst(stuckTurn())) + const user = converted.find((message) => message.role === "user") + + expect(user?.content).toContainEqual({ type: "image", image: dataUrl, mimeType: "image/jpeg" }) + }) + + it("OpenAI Codex Responses input sends the image as an input_image part", () => { + const handler = new OpenAiCodexHandler({ apiModelId: "gpt-5.1-codex" }) + const input: unknown[] = handler["formatFullConversation"]("", orderToolResultsFirst(stuckTurn())) + + expect(input).toContainEqual({ + role: "user", + content: [ + { type: "input_image", image_url: dataUrl }, + { type: "input_text", text: "" }, + ], + }) + }) + + it("a model without image support gets no image block anywhere in the request", () => { + const textOnly: ApiHandler = { + getModel: vi.fn().mockReturnValue({ id: "text-only", info: { supportsImages: false } }), + createMessage: vi.fn(), + countTokens: vi.fn(), + getModelCacheScope: vi.fn(), + } + const [, user] = maybeRemoveImageBlocks(orderToolResultsFirst(stuckTurn()), textOnly) + + expect(user.content).toEqual([ + toolResult("read"), + toolResult("cmd"), + text("[Referenced image in conversation]"), + text(""), + ]) + }) + }) +}) diff --git a/src/core/task/orderToolResultsFirst.ts b/src/core/task/orderToolResultsFirst.ts new file mode 100644 index 0000000000..8faecde182 --- /dev/null +++ b/src/core/task/orderToolResultsFirst.ts @@ -0,0 +1,33 @@ +import type { ApiMessage } from "../task-persistence" + +/** + * Moves `tool_result` blocks to the front of each user message, keeping the relative order of + * the results and of the remaining blocks (images, text). This is the same partition + * `Task.pushToolResultToUserContent` applies to live turns (#1307), so repaired history has the + * shape every provider converter already handles: images stay top-level blocks after the results, + * where image-capable converters forward them and `maybeRemoveImageBlocks` can strip them. + * + * Anthropic requires the results for the previous turn's `tool_use` blocks to lead the next user + * message. History saved with image blocks between results (#1307, #1774), or produced by merging + * consecutive user messages, is otherwise rejected with a 400 on every retry. + * + * Used for *API request shaping only* (do not use for storage). Messages already in order are + * returned unchanged. + */ +export function orderToolResultsFirst(messages: ApiMessage[]): ApiMessage[] { + return messages.map((message) => { + if (message.role !== "user" || !Array.isArray(message.content)) { + return message + } + const firstOther = message.content.findIndex((block) => block.type !== "tool_result") + const hasLaterResult = + firstOther !== -1 && message.content.slice(firstOther).some((block) => block.type === "tool_result") + if (!hasLaterResult) { + return message + } + + const results = message.content.filter((block) => block.type === "tool_result") + const rest = message.content.filter((block) => block.type !== "tool_result") + return { ...message, content: [...results, ...rest] } + }) +}