From 9f0e0a3b6a2a343d2e2af9e9e01f4782c3ebc55d Mon Sep 17 00:00:00 2001 From: Evgeniy Podivilov Date: Sun, 13 Sep 2026 17:21:15 +0100 Subject: [PATCH 1/2] fix(history): restore images during session replay --- src/CodexAcpServer.ts | 54 +++++++++++++++---- src/ContentChunks.ts | 12 +++++ src/ResponseItemHistoryFallback.ts | 38 ++++--------- .../data/load-session-history.json | 35 ++++++++++++ ...ession-response-item-history-fallback.json | 17 ++++++ .../CodexACPAgent/load-session.test.ts | 48 ++++++++++++++--- .../response-item-history-fallback.test.ts | 28 ++++++++++ 7 files changed, 187 insertions(+), 45 deletions(-) diff --git a/src/CodexAcpServer.ts b/src/CodexAcpServer.ts index 4ff0e1b4..64319cad 100644 --- a/src/CodexAcpServer.ts +++ b/src/CodexAcpServer.ts @@ -1,5 +1,8 @@ import * as acp from "@agentclientprotocol/sdk"; import {RequestError, type SessionId, type SessionModeState} from "@agentclientprotocol/sdk"; +import {readFile} from "node:fs/promises"; +import {extname} from "node:path"; +import {fileURLToPath, pathToFileURL} from "node:url"; import {CodexEventHandler, type CompletedPlan} from "./CodexEventHandler"; import {CodexApprovalHandler} from "./permissions/CodexApprovalHandler"; import {PermissionLifecycleContext} from "./permissions/lifecycle"; @@ -112,7 +115,9 @@ import { createAgentTextThoughtChunk, createCodexMessagePhaseMeta, createUserMessageChunk, + visibleUserMessageText, } from "./ContentChunks"; + import {sameThreadGoalSnapshot, type ThreadGoalSnapshot, toThreadGoalSnapshot,} from "./ThreadGoalSnapshot"; import { clientSupportsSubagents, @@ -152,6 +157,19 @@ import { parseAgentFileChangeReportRequest, } from "./AgentFileChangeReport"; +const LOCAL_IMAGE_MIME_TYPES: Readonly> = { + ".avif": "image/avif", + ".bmp": "image/bmp", + ".gif": "image/gif", + ".jpeg": "image/jpeg", + ".jpg": "image/jpeg", + ".png": "image/png", + ".webp": "image/webp", +}; + +function localImageMimeType(localPath: string): string | null { + return LOCAL_IMAGE_MIME_TYPES[extname(localPath).toLowerCase()] ?? null; +} export interface SessionState { sessionId: string, @@ -2282,13 +2300,11 @@ export class CodexAcpServer { } } - private createUserMessageUpdates(item: ThreadItem & { type: "userMessage" }): UpdateSessionEvent[] { + private async createUserMessageUpdates(item: ThreadItem & { type: "userMessage" }): Promise { const updates: UpdateSessionEvent[] = []; - const messageId = item.id; for (const input of item.content) { - const blocks = this.userInputToContentBlocks(input); - for (const block of blocks) { - updates.push(createUserMessageChunk(block, messageId)); + for (const block of await this.userInputToContentBlocks(input)) { + updates.push(createUserMessageChunk(block, item.id)); } } return updates; @@ -2349,20 +2365,36 @@ export class CodexAcpServer { ); } - private userInputToContentBlocks(input: UserInput): acp.ContentBlock[] { + private async userInputToContentBlocks(input: UserInput): Promise { switch (input.type) { - case "text": - return input.text.length > 0 ? [{ type: "text", text: input.text }] : []; - case "image": + case "text": { + const visibleText = visibleUserMessageText(input.text); + return visibleText.length > 0 ? [{ type: "text", text: visibleText }] : []; + } + case "image": { + const match = /^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=\r\n]+)$/i.exec(input.url); + if (match?.[1] && match[2]) { + return [{ type: "image", mimeType: match[1], data: match[2].replace(/\s/g, "") }]; + } return [{ type: "text", text: this.formatUriAsLink("image", input.url) }]; + } case "localImage": { - const uri = input.path.startsWith("file://") ? input.path : `file://${input.path}`; + const localPath = input.path.startsWith("file://") ? fileURLToPath(input.path) : input.path; + const uri = pathToFileURL(localPath).href; + const mimeType = localImageMimeType(localPath); + const data = mimeType ? await readFile(localPath).catch(() => null) : null; + if (mimeType && data) { + return [{ type: "image", mimeType, data: data.toString("base64"), uri }]; + } return [{ type: "text", text: this.formatUriAsLink(null, uri) }]; } case "skill": return [{ type: "text", text: `skill:${input.name} (${input.path})` }]; + case "audio": + case "localAudio": + case "mention": + return []; } - return []; } private formatUriAsLink(name: string | null, uri: string): string { diff --git a/src/ContentChunks.ts b/src/ContentChunks.ts index 2bef82e8..4a52eb39 100644 --- a/src/ContentChunks.ts +++ b/src/ContentChunks.ts @@ -3,6 +3,18 @@ import type {UpdateSessionEvent} from "./ACPSessionConnection"; type AcpMeta = Record; +const FILES_MENTIONED_HEADER = "# Files mentioned by the user:\n"; +const REQUEST_MARKER = "\n## My request for Codex:\n"; + +export function visibleUserMessageText(text: string): string { + const normalized = text.trimStart(); + const requestIndex = normalized.indexOf(REQUEST_MARKER); + if (normalized.startsWith(FILES_MENTIONED_HEADER) && requestIndex !== -1) { + return normalized.slice(requestIndex + REQUEST_MARKER.length); + } + return text; +} + export function createCodexMessagePhaseMeta(phase: string | null | undefined): AcpMeta | undefined { if (!phase) { return undefined; diff --git a/src/ResponseItemHistoryFallback.ts b/src/ResponseItemHistoryFallback.ts index 7bcba3e0..aef47ca5 100644 --- a/src/ResponseItemHistoryFallback.ts +++ b/src/ResponseItemHistoryFallback.ts @@ -6,7 +6,12 @@ import { stripShellPrefix } from "./CommandUtils"; import type { CommandAction, Thread, ThreadItem } from "./app-server/v2"; import { createCommandActionEvent } from "./CodexToolCallMapper"; import { createTerminalOutputMeta, type TerminalOutputMode } from "./TerminalOutputMode"; -import { createAgentMessageChunk, createCodexMessagePhaseMeta } from "./ContentChunks"; +import { + createAgentMessageChunk, + createCodexMessagePhaseMeta, + createUserMessageChunk, + visibleUserMessageText, +} from "./ContentChunks"; type JsonRecord = Record; type AcpToolCallEvent = Extract; @@ -263,18 +268,11 @@ function createEventMsgUpdates(record: JsonRecord): UpdateSessionEvent[] | null } function createUserMessageEventUpdates(payload: JsonRecord): UpdateSessionEvent[] { - const blocks: ContentBlock[] = []; - const message = stringValue(payload["message"]); - if (message !== null && message.length > 0) { - blocks.push({ type: "text", text: message }); + const text = visibleUserMessageText(stringValue(payload["message"]) ?? ""); + if (text.length > 0) { + return [createUserMessageChunk({ type: "text", text })]; } - blocks.push(...imageBlocks(payload["images"])); - blocks.push(...imageBlocks(payload["local_images"])); - - return blocks.map((content) => ({ - sessionUpdate: "user_message_chunk", - content, - })); + return []; } function createAgentReasoningEventUpdates(payload: JsonRecord): UpdateSessionEvent[] { @@ -289,22 +287,6 @@ function createAgentReasoningEventUpdates(payload: JsonRecord): UpdateSessionEve }]; } -function imageBlocks(images: unknown): ContentBlock[] { - if (!Array.isArray(images)) { - return []; - } - - return images.flatMap((image): ContentBlock[] => { - if (typeof image === "string") { - return [{ type: "text", text: `[@image](${image})` }]; - } - - const record = asRecord(image); - const path = record ? stringValue(record["path"]) ?? stringValue(record["url"]) : null; - return path ? [{ type: "text", text: `[@image](${path})` }] : []; - }); -} - function contentBlocksFromResponseContent(content: unknown): ContentBlock[] { if (!Array.isArray(content)) { return []; diff --git a/src/__tests__/CodexACPAgent/data/load-session-history.json b/src/__tests__/CodexACPAgent/data/load-session-history.json index 69644617..b715c651 100644 --- a/src/__tests__/CodexACPAgent/data/load-session-history.json +++ b/src/__tests__/CodexACPAgent/data/load-session-history.json @@ -170,6 +170,41 @@ } ] } +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "image", + "mimeType": "image/png", + "data": "dGVzdCBpbWFnZQ==" + } + } + } + ] +} +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "image", + "mimeType": "image/png", + "data": "dGVzdCBpbWFnZQ==", + "uri": "file:///tmp/codex-acp-load-session-image.png" + } + } + } + ] +} { "method": "sessionUpdate", "args": [ diff --git a/src/__tests__/CodexACPAgent/data/load-session-response-item-history-fallback.json b/src/__tests__/CodexACPAgent/data/load-session-response-item-history-fallback.json index c78620f2..c58cdfff 100644 --- a/src/__tests__/CodexACPAgent/data/load-session-response-item-history-fallback.json +++ b/src/__tests__/CodexACPAgent/data/load-session-response-item-history-fallback.json @@ -145,6 +145,23 @@ } ] } +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-legacy", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "image", + "mimeType": "image/png", + "data": "dW5pcXVlLWZhbGxiYWNrLWltYWdl" + } + } + } + ] +} { "method": "sessionUpdate", "args": [ diff --git a/src/__tests__/CodexACPAgent/load-session.test.ts b/src/__tests__/CodexACPAgent/load-session.test.ts index 05440d65..e51ed3eb 100644 --- a/src/__tests__/CodexACPAgent/load-session.test.ts +++ b/src/__tests__/CodexACPAgent/load-session.test.ts @@ -1,9 +1,11 @@ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, onTestFinished, vi } from "vitest"; import type * as acp from "@agentclientprotocol/sdk"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import { createCodexMockTestFixture, createTestModel } from "../acp-test-utils"; +import type { UpdateSessionEvent } from "../../ACPSessionConnection"; import type { Model, Thread, ThreadGoal } from "../../app-server/v2"; describe("CodexACPAgent - loadSession", () => { @@ -191,6 +193,11 @@ describe("CodexACPAgent - loadSession", () => { }); it("should replay history during loadSession", async () => { + const localImageDirectory = await mkdtemp(join(tmpdir(), "codex-acp-load-session-")); + const localImagePath = join(localImageDirectory, "image.png"); + await writeFile(localImagePath, "test image"); + onTestFinished(() => rm(localImageDirectory, { recursive: true })); + const fixture = createCodexMockTestFixture(); const codexAcpAgent = fixture.getCodexAcpAgent(); const codexAcpClient = fixture.getCodexAcpClient(); @@ -273,8 +280,14 @@ describe("CodexACPAgent - loadSession", () => { id: "item-user-1", clientId: null, content: [ - { type: "text", text: "Hi", text_elements: [] }, + { + type: "text", + text: `\n# Files mentioned by the user:\n\n## image.png: ${localImagePath}\n\n## My request for Codex:\nHi`, + text_elements: [], + }, { type: "image", url: "https://example.com/image.png" }, + { type: "image", url: "data:image/png;base64,dGVzdCBpbWFnZQ==" }, + { type: "localImage", path: localImagePath }, ], }, { @@ -417,7 +430,11 @@ describe("CodexACPAgent - loadSession", () => { expect(codexAppServerClient.threadReadWithHistory).toHaveBeenCalledWith(thread.id); expect(codexAppServerClient.threadGoalGet).toHaveBeenCalledWith({ threadId: thread.id }); - await expect(fixture.getAcpConnectionDump([])).toMatchFileSnapshot( + const replay = fixture.getAcpConnectionDump([]).replaceAll( + pathToFileURL(localImagePath).href, + "file:///tmp/codex-acp-load-session-image.png", + ); + await expect(replay).toMatchFileSnapshot( "data/load-session-history.json" ); }); @@ -534,8 +551,8 @@ describe("CodexACPAgent - loadSession", () => { type: "event_msg", payload: { type: "user_message", - message: "List the files", - images: [], + message: "\n# Files mentioned by the user:\n\n## screenshot.png: embedded\n\n## My request for Codex:\nList the files", + images: ["data:image/png;base64,dW5pcXVlLWZhbGxiYWNrLWltYWdl"], local_images: [], text_elements: [], }, @@ -718,7 +735,14 @@ describe("CodexACPAgent - loadSession", () => { type: "userMessage", id: "item-user-1", clientId: null, - content: [{ type: "text", text: "List the files", text_elements: [] }], + content: [ + { + type: "text", + text: "List the files", + text_elements: [], + }, + { type: "image", url: "data:image/png;base64,dW5pcXVlLWZhbGxiYWNrLWltYWdl" }, + ], }, { type: "reasoning", @@ -772,6 +796,18 @@ describe("CodexACPAgent - loadSession", () => { mcpServers: [], }); + const replayedUserContent = fixture.getAcpConnectionEvents([]) + .filter(event => event.method === "sessionUpdate") + .map(event => event.args[0].update) + .filter((update): update is Extract => ( + update.sessionUpdate === "user_message_chunk" + )) + .map(update => update.content); + expect(replayedUserContent).toEqual([ + {type: "text", text: "List the files"}, + {type: "image", mimeType: "image/png", data: "dW5pcXVlLWZhbGxiYWNrLWltYWdl"}, + ]); + await expect(fixture.getAcpConnectionDump([])).toMatchFileSnapshot( "data/load-session-response-item-history-fallback.json", ); diff --git a/src/__tests__/CodexACPAgent/response-item-history-fallback.test.ts b/src/__tests__/CodexACPAgent/response-item-history-fallback.test.ts index aaeef097..f1036f3e 100644 --- a/src/__tests__/CodexACPAgent/response-item-history-fallback.test.ts +++ b/src/__tests__/CodexACPAgent/response-item-history-fallback.test.ts @@ -55,6 +55,26 @@ describe("ResponseItemHistoryFallback", () => { expect(thoughtTexts(updates)).toEqual(["Need to inspect the directory."]); }); + it("leaves user attachments to authoritative thread history", () => { + const updates = parseResponseItemHistoryFallback(jsonl([ + { + type: "event_msg", + payload: { + type: "user_message", + message: "\n# Files mentioned by the user:\n\n## screenshot.png: /tmp/screenshot.png\n\n## My request for Codex:\nInspect the screenshot", + images: ["data:image/png;base64,dGVzdA=="], + local_images: ["/tmp/screenshot.png"], + }, + }, + functionCall("call-missing", "ls"), + functionCallOutput("call-missing", "Chunk ID: missing\nProcess exited with code 0\nOutput:\nREADME.md\n"), + ]), "terminal_output"); + + expect(userMessageContents(updates)).toEqual([ + { type: "text", text: "Inspect the screenshot" }, + ]); + }); + it("preserves assistant message phase metadata from response items", () => { const updates = parseResponseItemHistoryFallback(jsonl([ { @@ -154,6 +174,14 @@ function thoughtTexts(updates: UpdateSessionEvent[] | null): string[] { .flatMap((update) => update.content.type === "text" ? [update.content.text] : []); } +function userMessageContents(updates: UpdateSessionEvent[] | null): unknown[] { + return (updates ?? []) + .filter((update): update is Extract => ( + update.sessionUpdate === "user_message_chunk" + )) + .map((update) => update.content); +} + function agentMessageMetas(updates: UpdateSessionEvent[] | null): unknown[] { return (updates ?? []) .filter((update): update is Extract => ( From cf2f7d52f287565661ad6ac9ebeb7abbc9175f55 Mon Sep 17 00:00:00 2001 From: Evgeniy Podivilov Date: Sun, 13 Sep 2026 17:45:55 +0100 Subject: [PATCH 2/2] fix(history): preserve attachment replay fallbacks --- src/CodexAcpServer.ts | 15 +++++++-- .../data/load-session-history.json | 33 +++++++++++++++++++ .../CodexACPAgent/load-session.test.ts | 2 ++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/CodexAcpServer.ts b/src/CodexAcpServer.ts index 64319cad..4e70af38 100644 --- a/src/CodexAcpServer.ts +++ b/src/CodexAcpServer.ts @@ -2379,8 +2379,14 @@ export class CodexAcpServer { return [{ type: "text", text: this.formatUriAsLink("image", input.url) }]; } case "localImage": { - const localPath = input.path.startsWith("file://") ? fileURLToPath(input.path) : input.path; - const uri = pathToFileURL(localPath).href; + let localPath: string; + let uri: string; + try { + localPath = input.path.startsWith("file://") ? fileURLToPath(input.path) : input.path; + uri = pathToFileURL(localPath).href; + } catch { + return [{ type: "text", text: this.formatUriAsLink(null, input.path) }]; + } const mimeType = localImageMimeType(localPath); const data = mimeType ? await readFile(localPath).catch(() => null) : null; if (mimeType && data) { @@ -2392,8 +2398,11 @@ export class CodexAcpServer { return [{ type: "text", text: `skill:${input.name} (${input.path})` }]; case "audio": case "localAudio": - case "mention": return []; + case "mention": { + const uri = input.path.startsWith("file://") ? input.path : pathToFileURL(input.path).href; + return [{ type: "resource_link", name: input.name, uri }]; + } } } diff --git a/src/__tests__/CodexACPAgent/data/load-session-history.json b/src/__tests__/CodexACPAgent/data/load-session-history.json index b715c651..44aeb720 100644 --- a/src/__tests__/CodexACPAgent/data/load-session-history.json +++ b/src/__tests__/CodexACPAgent/data/load-session-history.json @@ -205,6 +205,39 @@ } ] } +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "text", + "text": "[@invalid%2Fimage.png](file:///tmp/invalid%2Fimage.png)" + } + } + } + ] +} +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "resource_link", + "name": "notes.txt", + "uri": "file:///test/project/notes.txt" + } + } + } + ] +} { "method": "sessionUpdate", "args": [ diff --git a/src/__tests__/CodexACPAgent/load-session.test.ts b/src/__tests__/CodexACPAgent/load-session.test.ts index e51ed3eb..592b01cf 100644 --- a/src/__tests__/CodexACPAgent/load-session.test.ts +++ b/src/__tests__/CodexACPAgent/load-session.test.ts @@ -288,6 +288,8 @@ describe("CodexACPAgent - loadSession", () => { { type: "image", url: "https://example.com/image.png" }, { type: "image", url: "data:image/png;base64,dGVzdCBpbWFnZQ==" }, { type: "localImage", path: localImagePath }, + { type: "localImage", path: "file:///tmp/invalid%2Fimage.png" }, + { type: "mention", name: "notes.txt", path: "/test/project/notes.txt" }, ], }, {