diff --git a/apps/desktop/src/main/__tests__/session-status-presentation.test.ts b/apps/desktop/src/main/__tests__/session-status-presentation.test.ts new file mode 100644 index 0000000000..0d2fb84c40 --- /dev/null +++ b/apps/desktop/src/main/__tests__/session-status-presentation.test.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { deriveFailedTurnRecovery } from '../../renderer/session-status-presentation.js'; + +const outputFreeFailure = { + partialOutputRetained: false, + toolActivityCount: 0, + erroredToolCount: 0, +}; + +describe('failed turn recovery presentation', () => { + it('does not recommend a byte-identical retry after context overflow', () => { + assert.deepEqual( + deriveFailedTurnRecovery({ ...outputFreeFailure, errorClass: 'context_overflow' }, 'zh'), + { action: 'continue', label: '上下文仍超出限制，请减少附件或开启新任务' }, + ); + assert.deepEqual( + deriveFailedTurnRecovery({ ...outputFreeFailure, errorClass: 'context_overflow' }, 'en'), + { + action: 'continue', + label: 'Context is still too large; reduce attachments or start a new task', + }, + ); + }); + + it('keeps the generic retry fallback for an unknown output-free failure', () => { + assert.deepEqual( + deriveFailedTurnRecovery({ ...outputFreeFailure, errorClass: 'unknown_failure' }, 'en'), + { action: 'retry', label: 'No tools ran; retry directly' }, + ); + }); + + it('preserves higher-priority retained-output and tool guidance', () => { + assert.deepEqual( + deriveFailedTurnRecovery( + { ...outputFreeFailure, errorClass: 'context_overflow', partialOutputRetained: true }, + 'en', + ), + { action: 'continue', label: 'Partial output was retained; continue from here' }, + ); + assert.deepEqual( + deriveFailedTurnRecovery( + { ...outputFreeFailure, errorClass: 'context_overflow', toolActivityCount: 1 }, + 'en', + ), + { action: 'inspect_tool', label: 'Tool history was retained; inspect it before retrying' }, + ); + assert.deepEqual( + deriveFailedTurnRecovery( + { ...outputFreeFailure, errorClass: 'context_overflow', erroredToolCount: 1 }, + 'en', + ), + { action: 'inspect_tool', label: 'Inspect the tool result before retrying' }, + ); + }); +}); diff --git a/apps/desktop/src/renderer/locales/conversation-copy.ts b/apps/desktop/src/renderer/locales/conversation-copy.ts index 7f62272903..f73e4b3207 100644 --- a/apps/desktop/src/renderer/locales/conversation-copy.ts +++ b/apps/desktop/src/renderer/locales/conversation-copy.ts @@ -296,7 +296,7 @@ export interface DesktopConversationCopy { permission: string; restarted: string; sandboxBoundaryClosed: string; - recovery: Record<'safeResume' | 'stepCap' | 'toolError' | 'connection' | 'partial' | 'toolRecord' | 'retry' | 'sandboxBoundaryClosed', string>; + recovery: Record<'safeResume' | 'stepCap' | 'toolError' | 'connection' | 'partial' | 'toolRecord' | 'retry' | 'contextOverflow' | 'sandboxBoundaryClosed', string>; }; } @@ -566,7 +566,7 @@ const COPY = { reauth: { label: '上次连接测试鉴权失败', tooltip: '最近一次连接测试返回鉴权失败（401 / 403），密钥可能已过期或被吊销。这不会拦截发送，但若发送失败请到 设置 · 模型 重新登录。' }, testError: { label: '上次连接测试失败', tooltip: '最近一次连接测试因网络 / 超时 / 5xx 失败。这不会拦截发送，但若问题持续请到 设置 · 模型 检查 Base URL / 代理。' }, }, - turnError: { unknown: '未知错误', contextOverflow: '上下文窗口已超出限制', timeout: '请求超时', auth: '鉴权失败', providerBilling: '模型服务计费受限', rateLimit: '触发模型速率限制', network: '网络错误', provider: '模型服务返回错误', stepCap: '达到工具步骤上限', tool: '工具调用失败', permission: '等待权限确认', restarted: '本地应用重启，上一轮没有完成', sandboxBoundaryClosed: '本地应用重启，等待确认的「允许访问工作区以外的内容」请求已按拒绝关闭', recovery: { safeResume: '检查当前状态后，可尝试安全恢复', stepCap: '任务可能尚未完成，可以继续', toolError: '先检查工具结果，再决定是否重试', connection: '先检查模型连接或登录状态', partial: '已保留部分输出，可从这里继续', toolRecord: '工具记录已保留，重试前先看结果', retry: '没有执行工具，可直接重试', sandboxBoundaryClosed: '访问范围没有放开，重试本轮后可重新决定' } }, + turnError: { unknown: '未知错误', contextOverflow: '上下文窗口已超出限制', timeout: '请求超时', auth: '鉴权失败', providerBilling: '模型服务计费受限', rateLimit: '触发模型速率限制', network: '网络错误', provider: '模型服务返回错误', stepCap: '达到工具步骤上限', tool: '工具调用失败', permission: '等待权限确认', restarted: '本地应用重启，上一轮没有完成', sandboxBoundaryClosed: '本地应用重启，等待确认的「允许访问工作区以外的内容」请求已按拒绝关闭', recovery: { safeResume: '检查当前状态后，可尝试安全恢复', stepCap: '任务可能尚未完成，可以继续', toolError: '先检查工具结果，再决定是否重试', connection: '先检查模型连接或登录状态', partial: '已保留部分输出，可从这里继续', toolRecord: '工具记录已保留，重试前先看结果', retry: '没有执行工具，可直接重试', contextOverflow: '上下文仍超出限制，请减少附件或开启新任务', sandboxBoundaryClosed: '访问范围没有放开，重试本轮后可重新决定' } }, }, en: { actions: { stopFailedTitle: 'Failed to stop', stopFailedFallback: 'The task action failed. Try again later.', refreshSessionsFailedTitle: 'Failed to refresh tasks', refreshSessionsFailedFallback: 'The task list could not be refreshed. Try again later.', conversationErrorTitle: 'Task error', conversationErrorFallback: 'The task run failed. Try again later.', regenerateStartedTitle: 'Regeneration started', regenerateStartedDescription: 'Generating a new response', branchCreatedTitle: 'Branch created', branchCreatedDescription: (name) => `New task: ${name}`, revisionStartedTitle: 'Edit draft ready', revisionStartedDescription: 'The original task is kept; sending creates a new version', revisionReadyTitle: 'Ready to edit and resend', revisionReadyDescription: 'Rewound to before that message; edit and send when ready', revisionUnavailableTitle: 'This message cannot be edited yet', revisionAttachmentsUnsupported: 'Edit & resend does not yet support historical attachments. Copy the text into a new message instead.', revisionTransformedTextUnsupported: 'Edit & resend does not yet support messages sent with an explicit skill. Copy the text and select the skill again instead.', revisionDraftAttachmentConflict: 'The composer already has pending attachments. Send or remove them before editing a sent message.', revisionCommandUnsupported: 'You cannot run /compact, /side, or orchestration commands while editing a sent message. Cancel the edit first.', revisionAlreadyActive: 'Another message is already being edited. Send or cancel that edit first.', revisionCancelLabel: 'Cancel', revisionBannerTitle: 'Editing sent message', revisionBannerDetail: '· New version on send', revisionUnchanged: 'Nothing changed. Use Regenerate if you only want a new answer.', operationFailedTitle: 'Action failed', operationFailedFallback: 'The task action failed. Try again later.', attachmentFailedTitle: 'Failed to add attachment', tryAgain: 'Try again later.', modelReboundTitle: 'Switched to an available model', modelReboundDescription: (modelId) => `The previous connection is unavailable${modelId ? ` · ${modelId}` : ''}`, messageReadFailedTitle: 'Failed to load task', returnLatest: 'Return to latest' }, @@ -768,7 +768,7 @@ const COPY = { reauth: { label: 'Last connection test failed authentication', tooltip: 'The latest test returned 401 / 403. Sending is not blocked, but sign in again under Settings · Models if it fails.' }, testError: { label: 'Last connection test failed', tooltip: 'The latest test failed because of a network, timeout, or 5xx error. Sending is not blocked; check Base URL or proxy settings if it persists.' }, }, - turnError: { unknown: 'Unknown error', contextOverflow: 'Context window exceeded', timeout: 'Request timed out', auth: 'Authentication failed', providerBilling: 'Provider billing required', rateLimit: 'Model rate limit reached', network: 'Network error', provider: 'Model service error', stepCap: 'Tool-step limit reached', tool: 'Tool call failed', permission: 'Waiting for permission', restarted: 'The app restarted before the previous turn completed', sandboxBoundaryClosed: 'The app restarted, so the pending request to reach outside the workspace was closed as denied', recovery: { safeResume: 'Inspect the current state, then try safe recovery', stepCap: 'The task may be incomplete; continue from here', toolError: 'Inspect the tool result before retrying', connection: 'Check the model connection or sign-in status', partial: 'Partial output was retained; continue from here', toolRecord: 'Tool history was retained; inspect it before retrying', retry: 'No tools ran; retry directly', sandboxBoundaryClosed: 'Access was not widened; retry the turn to decide again' } }, + turnError: { unknown: 'Unknown error', contextOverflow: 'Context window exceeded', timeout: 'Request timed out', auth: 'Authentication failed', providerBilling: 'Provider billing required', rateLimit: 'Model rate limit reached', network: 'Network error', provider: 'Model service error', stepCap: 'Tool-step limit reached', tool: 'Tool call failed', permission: 'Waiting for permission', restarted: 'The app restarted before the previous turn completed', sandboxBoundaryClosed: 'The app restarted, so the pending request to reach outside the workspace was closed as denied', recovery: { safeResume: 'Inspect the current state, then try safe recovery', stepCap: 'The task may be incomplete; continue from here', toolError: 'Inspect the tool result before retrying', connection: 'Check the model connection or sign-in status', partial: 'Partial output was retained; continue from here', toolRecord: 'Tool history was retained; inspect it before retrying', retry: 'No tools ran; retry directly', contextOverflow: 'Context is still too large; reduce attachments or start a new task', sandboxBoundaryClosed: 'Access was not widened; retry the turn to decide again' } }, }, } satisfies UiCatalog; diff --git a/apps/desktop/src/renderer/session-status-presentation.ts b/apps/desktop/src/renderer/session-status-presentation.ts index 721eb22881..ee22da2894 100644 --- a/apps/desktop/src/renderer/session-status-presentation.ts +++ b/apps/desktop/src/renderer/session-status-presentation.ts @@ -180,5 +180,8 @@ export function deriveFailedTurnRecovery(input: FailedTurnRecoveryInput, locale: if (input.toolActivityCount > 0) { return { action: 'inspect_tool', label: copy.toolRecord }; } + if (lower === 'context_overflow') { + return { action: 'continue', label: copy.contextOverflow }; + } return { action: 'retry', label: copy.retry }; } diff --git a/docs/superpowers/plans/2026-08-21-issue-3372-reactive-image-overflow-recovery.md b/docs/superpowers/plans/2026-08-21-issue-3372-reactive-image-overflow-recovery.md new file mode 100644 index 0000000000..0e6e9c4ab3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-issue-3372-reactive-image-overflow-recovery.md @@ -0,0 +1,436 @@ +# Issue #3372 Reactive Image Overflow Recovery Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Recover once from provider-confirmed context overflow by omitting only hydrated historical image tool results, and give Desktop users honest guidance when overflow recovery is exhausted. + +**Architecture:** Derive eligible image tool-call IDs from `MidTurnCapacityCompactState.priorContentEvents`, then apply a pure provider-message projection keyed by those IDs. Invoke it first in reactive `ContextLength` recovery and reapply the stored ID map to later request projections; remove every proactive image-capacity estimate and omission path. At the Desktop presentation boundary, map the stable `context_overflow` error class to dedicated non-retry guidance. + +**Tech Stack:** TypeScript, Node.js test runner, AI SDK `ModelMessage`, Maka `RuntimeEvent`, Biome, npm workspaces. + +## Global Constraints + +- Node.js must be `>=22.19.0`; use `/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin` ahead of `/usr/local/bin` in `PATH`. +- Only a provider `ContextLength` classification may activate image omission. +- Eligibility comes only from historical `state.priorContentEvents` image `function_response` events. +- Never mutate user attachments, current-turn images, newly produced image results, persisted `RuntimeEvent`s, or stored artifacts. +- Keep one shared overflow retry; a second overflow terminates without generic provider retry. +- Do not add `PreparedRequestSegment.bytes` gating or multimodal token estimates. +- Desktop guidance is presentation-only: add no button or automatic retry behavior. + +--- + +## File Structure + +- Create `packages/runtime/src/provider-image-overflow-recovery.ts`: collect historical image identities and perform the ID-scoped provider-message projection. +- Create `packages/runtime/src/__tests__/provider-image-overflow-recovery.test.ts`: unit contract for eligibility, artifact copy, protocol preservation, and non-target preservation. +- Delete `packages/runtime/src/provider-image-capacity.ts` and `packages/runtime/src/__tests__/provider-image-capacity.test.ts`: remove the proactive estimator helper and its behavior-affirming tests. +- Modify `packages/runtime/src/ai-sdk-compaction.ts`: invoke the reactive rung, retain exact omitted IDs, reapply them, and restore the original text-capacity estimator. +- Modify `packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts`: cover reactive omission, exact replay, new-image preservation, and bounded retry. +- Modify `packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts`: replace proactive omission expectations with step-zero preservation. +- Restore `packages/core/src/usage-record-schema.ts`, `packages/core/src/usage-stats/types.ts`, and `packages/runtime/src/context-budget.ts` to their pre-WIP behavior by removing image-estimate diagnostics. +- Keep the `context_overflow` generic-retry exclusion in `packages/runtime/src/ai-sdk-backend.ts`. +- Create `apps/desktop/src/main/__tests__/session-status-presentation.test.ts`: pin dedicated context-overflow guidance and preserve the unknown-failure fallback. +- Modify `apps/desktop/src/renderer/session-status-presentation.ts`: classify `context_overflow` before the generic retry fallback. +- Modify `apps/desktop/src/renderer/locales/conversation-copy.ts`: add typed Chinese and English recovery copy. +- Delete unrelated `docs/archive/sun-gliang-public-pr-resume-analysis-2026-08-18.md` from this issue's working tree. + +--- + +### Task 1: ID-scoped historical image projection + +**Files:** +- Create: `packages/runtime/src/provider-image-overflow-recovery.ts` +- Create: `packages/runtime/src/__tests__/provider-image-overflow-recovery.test.ts` +- Delete: `packages/runtime/src/provider-image-capacity.ts` +- Delete: `packages/runtime/src/__tests__/provider-image-capacity.test.ts` + +**Interfaces:** +- Consumes: `readonly RuntimeEvent[]` and `readonly ModelMessage[]`. +- Produces: `collectHistoricalImageToolResults(events): Map`. +- Produces: `omitHistoricalImageToolResults(messages, eligible): HistoricalImageOmissionResult`. +- `HistoricalImageOmissionResult.omittedToolCallIds` is the exact ID set later stored in mid-turn state. + +- [ ] **Step 1: Replace the helper tests with failing ID-boundary tests** + +Write tests that import the new module and assert the intended API: + +```ts +const eligible = collectHistoricalImageToolResults(priorEvents); +const result = omitHistoricalImageToolResults( + [ + userImageMessage('USER_IMAGE'), + toolImageMessage('prior-image-call', 'PRIOR_IMAGE'), + toolImageMessage('new-image-call', 'NEW_IMAGE'), + ], + eligible, +); + +assert.deepEqual([...result.omittedToolCallIds], ['prior-image-call']); +assert.equal(prompt(result.messages).includes('USER_IMAGE'), true); +assert.equal(prompt(result.messages).includes('NEW_IMAGE'), true); +assert.equal(prompt(result.messages).includes('PRIOR_IMAGE'), false); +assert.match(prompt(result.messages), /screenshot\.png/); +assert.match(prompt(result.messages), /repeat the preceding Read tool call/i); +``` + +Add a second test proving text siblings and the `tool-result` envelope remain present, the input messages remain unchanged, and a second projection pass performs no additional rewrite. + +- [ ] **Step 2: Run the Runtime workspace suite and verify RED** + +Run: + +```bash +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npm --workspace @maka/runtime test +``` + +Expected: the Runtime workspace cleans and rebuilds before the new projection test fails because +`provider-image-overflow-recovery.ts` and its exports do not exist. + +- [ ] **Step 3: Implement the minimal pure projection** + +Create the module around these exact public contracts: + +```ts +export interface HistoricalImageToolResult { + toolName: string; + artifactLabel: string; +} + +export interface HistoricalImageOmissionResult { + messages: ModelMessage[]; + omittedParts: number; + omittedToolCallIds: Set; +} + +export function collectHistoricalImageToolResults( + events: readonly RuntimeEvent[], +): Map; + +export function omitHistoricalImageToolResults( + messages: readonly ModelMessage[], + eligible: ReadonlyMap, +): HistoricalImageOmissionResult; +``` + +Collection accepts only non-error `function_response` events whose result has +`kind: 'image'`, a non-empty image MIME type, and a recognized storage ref. The artifact label is +`relativePath` for session/workspace refs and `absolutePath` for external refs. + +Projection visits only nested `tool-result.output.type === 'content'` parts. For matching +`toolCallId`s, replace inline image `file` parts with: + +```ts +{ + type: 'text', + text: `[Image artifact "${artifactLabel}" omitted after provider context overflow. Repeat the preceding ${toolName} tool call if visual context is required.]`, +} +``` + +Do not inspect or rewrite direct message image/file parts. Clone only changed arrays and objects. + +- [ ] **Step 4: Run the unit test and verify GREEN** + +Run the Task 1 command again. Expected: all tests in +`provider-image-overflow-recovery.test.js` pass with zero failures. + +- [ ] **Step 5: Commit the helper contract** + +```bash +git add -- packages/runtime/src/provider-image-overflow-recovery.ts \ + packages/runtime/src/__tests__/provider-image-overflow-recovery.test.ts +git commit -m "fix(runtime): target historical image overflow projection" \ + -m "Generated-by: Codex" +``` + +--- + +### Task 2: Reactive recovery integration and proactive-path removal + +**Files:** +- Modify: `packages/runtime/src/ai-sdk-compaction.ts` +- Modify: `packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts` +- Modify: `packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts` +- Modify: `packages/runtime/src/ai-sdk-backend.ts` +- Modify: `packages/core/src/usage-record-schema.ts` +- Modify: `packages/core/src/usage-stats/types.ts` +- Modify: `packages/runtime/src/context-budget.ts` + +**Interfaces:** +- Consumes Task 1's two exported functions and `HistoricalImageToolResult`. +- Produces `MidTurnCapacityCompactState.omittedImageToolResults`, a map keyed by exact omitted tool-call IDs. + +- [ ] **Step 1: Write failing reactive and step-zero regressions** + +Change the reactive fixture so an overflow request contains three independent shapes: + +```ts +// Historical eligible image: prior-image-call. +imagePrior: true, +// Current-turn tool image with a different tool-call ID. +liveImageResult: true, +// User image on the current anchor. +currentImage: true, +``` + +Assert the rejected request contains all three images. Assert the retry removes only +`prior-image-call`, contains `screenshot.png`, preserves the user and live-tool images, does not +rerun completed tools, and does not invoke the summarizer. After the retry returns another tool +call, assert the successor request still omits `prior-image-call` while preserving the new result. + +Replace proactive tests with this regression: + +```ts +test('does not omit hydrated images before a selected-model step-zero provider verdict', async () => { + const fixture = buildFixture({ + contextWindow: 10_000, + currentImage: true, + imageBytes: 8_000, + withoutPriorTurns: true, + }); + await runFixtureTurn(fixture); + assert.equal(fixture.model.doStreamCalls.length > 0, true); + assert.equal(promptJson(fixture, 0).includes('"mediaType":"image/png"'), true); +}); +``` + +- [ ] **Step 2: Run the Runtime workspace suite and verify RED** + +Run: + +```bash +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npm --workspace @maka/runtime test +``` + +Expected: the Runtime workspace cleans and rebuilds before the new integration assertions fail +because the current WIP removes every image and performs proactive step-zero omission. + +- [ ] **Step 3: Integrate exact IDs into reactive recovery** + +In `recoverFromOverflowError`, derive and apply eligible prior images before the existing summary +rung: + +```ts +const eligibleImages = collectHistoricalImageToolResults(state.priorContentEvents); +const imageOmission = omitHistoricalImageToolResults(input.currentMessages, eligibleImages); +if (imageOmission.omittedParts > 0) { + state.omittedImageToolResults = new Map( + [...imageOmission.omittedToolCallIds].flatMap((toolCallId) => { + const image = eligibleImages.get(toolCallId); + return image ? [[toolCallId, image] as const] : []; + }), + ); + state.lastRequestPayloadChars = undefined; + state.lastRequestInputTokens = undefined; + return { messages: imageOmission.messages }; +} +``` + +Declare state as: + +```ts +omittedImageToolResults = new Map(); +``` + +Before the mid-turn capacity stage measures a later durable replay, apply the stored map to +`incomingMessages`. Reapply the same pure helper at the final projection boundary so later shaping +cannot resurrect a targeted result. Neither call emits estimate diagnostics. + +- [ ] **Step 4: Remove proactive image capacity behavior** + +Restore `lastRequestPayloadChars` as the sole signed-delta baseline and remove: + +- `lastRequestCapacityChars`; +- `midTurnRequestCapacityChars` and `inlineImageEstimatedTokensSaved`; +- all calls to `projectInlineImageCapacity`, `omitOldestInlineImage`, and broad + `omitAllInlineImages`; +- step-zero enforcement based on hydrated image presence; +- proactive omission loops before and after capacity reentry; +- `capacityOmittedImageParts` and `capacityImageEstimatedTokensSaved` from core types, schema, + merge logic, and tests. + +Keep this backend condition so the spent recovery does not fall into blind generic retries: + +```ts +failure.kind !== 'context_overflow' +``` + +- [ ] **Step 5: Run focused tests and verify GREEN** + +Run the Task 2 command again. Expected: both files pass with zero failures. + +- [ ] **Step 6: Commit the runtime integration** + +```bash +git add -- packages/core/src/usage-record-schema.ts packages/core/src/usage-stats/types.ts \ + packages/runtime/src/ai-sdk-backend.ts packages/runtime/src/ai-sdk-compaction.ts \ + packages/runtime/src/context-budget.ts \ + packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts \ + packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts +git commit -m "fix(runtime): recover overflow by omitting historical images" \ + -m "Generated-by: Codex" +``` + +--- + +### Task 3: Desktop context-overflow recovery guidance + +**Files:** +- Create: `apps/desktop/src/main/__tests__/session-status-presentation.test.ts` +- Modify: `apps/desktop/src/renderer/session-status-presentation.ts:156-184` +- Modify: `apps/desktop/src/renderer/locales/conversation-copy.ts:285-299,569,771` + +**Interfaces:** +- Consumes: `deriveFailedTurnRecovery(input, locale)` and stable Runtime error class `context_overflow`. +- Produces: `turnError.recovery.contextOverflow` in both locales and an existing `continue` presentation category. + +- [ ] **Step 1: Write the failing presentation test** + +Create `apps/desktop/src/main/__tests__/session-status-presentation.test.ts`: + +```ts +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { deriveFailedTurnRecovery } from '../../renderer/session-status-presentation.js'; + +const outputFreeFailure = { + partialOutputRetained: false, + toolActivityCount: 0, + erroredToolCount: 0, +}; + +describe('failed turn recovery presentation', () => { + it('does not recommend a byte-identical retry after context overflow', () => { + assert.deepEqual( + deriveFailedTurnRecovery({ ...outputFreeFailure, errorClass: 'context_overflow' }, 'zh'), + { action: 'continue', label: '上下文仍超出限制，请减少附件或开启新任务' }, + ); + assert.deepEqual( + deriveFailedTurnRecovery({ ...outputFreeFailure, errorClass: 'context_overflow' }, 'en'), + { action: 'continue', label: 'Context is still too large; reduce attachments or start a new task' }, + ); + }); + + it('keeps the generic retry fallback for an unknown output-free failure', () => { + assert.deepEqual( + deriveFailedTurnRecovery({ ...outputFreeFailure, errorClass: 'unknown_failure' }, 'en'), + { action: 'retry', label: 'No tools ran; retry directly' }, + ); + }); +}); +``` + +- [ ] **Step 2: Run the Desktop workspace test and verify RED** + +Run: + +```bash +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npm --workspace @maka/desktop test +``` + +Expected: the new context-overflow assertion fails because the function returns the generic +`retry` presentation and copy. + +- [ ] **Step 3: Add the minimal typed copy and branch** + +Add `contextOverflow` to the `turnError.recovery` record in +`apps/desktop/src/renderer/locales/conversation-copy.ts`, with these exact values: + +```ts +contextOverflow: '上下文仍超出限制，请减少附件或开启新任务' +contextOverflow: 'Context is still too large; reduce attachments or start a new task' +``` + +In `deriveFailedTurnRecovery`, immediately before the generic return, add: + +```ts +if (lower === 'context_overflow') { + return { action: 'continue', label: copy.contextOverflow }; +} +``` + +- [ ] **Step 4: Run the Desktop workspace test and verify GREEN** + +Run the Step 2 command again. Expected: all Desktop tests pass with zero failures. + +- [ ] **Step 5: Commit the Desktop guidance** + +```bash +git add -- apps/desktop/src/main/__tests__/session-status-presentation.test.ts \ + apps/desktop/src/renderer/session-status-presentation.ts \ + apps/desktop/src/renderer/locales/conversation-copy.ts +git commit -m "fix(desktop): clarify context overflow recovery" \ + -m "Generated-by: Codex" +``` + +--- + +### Task 4: Scope cleanup and full verification + +**Files:** +- Delete: `docs/archive/sun-gliang-public-pr-resume-analysis-2026-08-18.md` +- Verify all files changed since `57e08d83497d1d7ace7d6eff88e4e5267a0345b5`. + +**Interfaces:** +- Consumes the completed Task 1, Task 2, and Task 3 implementations. +- Produces a review-ready branch containing only issue #3372 design and implementation work. + +- [ ] **Step 1: Remove the unrelated untracked document** + +Delete only `docs/archive/sun-gliang-public-pr-resume-analysis-2026-08-18.md`; do not remove any +other archive content. + +- [ ] **Step 2: Run fresh full verification** + +```bash +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npm --workspace @maka/runtime test +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npm --workspace @maka/desktop test +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npm run lint +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npm run format:check +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npm run build +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npm run typecheck +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npx knip --workspace apps/desktop +PATH=/Users/sun/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin:$PATH \ + npx knip --workspace packages/ui +git diff --check 57e08d83497d1d7ace7d6eff88e4e5267a0345b5..HEAD +git diff --check HEAD +git status --short +``` + +Expected: all commands exit 0; Runtime and Desktop report zero failed tests; lint, format, and both +`knip` checks report no errors; both diff checks print nothing; status contains no uncommitted issue +implementation files. + +- [ ] **Step 3: Capture the user-visible recovery-copy evidence** + +Capture the failed-turn recovery row before and after the Desktop copy change, in the same locale +and viewport. Attach the images or a short recording to the pull request. Do not commit the +temporary evidence under `docs/`. + +Expected: reviewers can compare the former direct-retry guidance with the dedicated +context-overflow guidance without reconstructing the UI from the diff. + +- [ ] **Step 4: Review the final diff against the design** + +Confirm each invariant directly in the diff: + +```text +provider ContextLength is the only activation signal +eligibility comes from priorContentEvents +state stores exact tool-call IDs +user/current/new images remain unchanged +placeholder names the artifact and original tool +second overflow cannot enter generic retry +no proactive image-token estimator remains +Desktop context_overflow guidance never recommends a direct retry +``` diff --git a/docs/superpowers/specs/2026-08-21-issue-3372-reactive-image-overflow-recovery-design.md b/docs/superpowers/specs/2026-08-21-issue-3372-reactive-image-overflow-recovery-design.md new file mode 100644 index 0000000000..8ffbd4cb96 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-issue-3372-reactive-image-overflow-recovery-design.md @@ -0,0 +1,118 @@ +# Issue #3372 Reactive Image Overflow Recovery Design + +## Context + +Hydrated image tool results can cause a provider to reject a request with `ContextLength` even +when local text-oriented capacity estimates remain below the selected model's context window. +Client-side image-token estimates cannot reliably predict multimodal provider accounting, so the +provider rejection is the admission signal for this recovery path. + +The current work-in-progress adds proactive image weighting and can omit any inline image, +including a user's current attachment. This design replaces that behavior with a narrow reactive +projection that affects only historical image tool results. + +## Goals + +- Recover once from a provider `ContextLength` response by mechanically omitting hydrated images + from eligible historical tool results before attempting LLM summarization. +- Preserve user attachments, current-turn images, newly produced image results, persisted + `RuntimeEvent`s, and stored artifacts. +- Keep omitted historical results omitted on every later provider request in the same send, even + when durable replay hydrates them again. +- Preserve the tool-call/tool-result protocol pair and identify the stored artifact clearly enough + for the model to repeat the original tool call when visual context is needed. +- Terminate honestly if the single recovered request also overflows. +- Present context-overflow-specific Desktop recovery guidance instead of suggesting a + byte-identical direct retry. + +## Non-goals + +- Predicting provider multimodal token usage. +- Adding a `PreparedRequestSegment.bytes` admission gate. +- Changing the existing image transport-size budget. +- Mutating or deleting persisted history or artifact bytes. +- Adding a new Desktop recovery button, action, or automatic retry behavior. + +## Chosen Design + +### Eligibility authority + +`MidTurnCapacityCompactState.priorContentEvents` is the sole eligibility source. Build a map for +prior events whose content is a successful `function_response` and whose result is a stored image: + +- key: `content.id`, the stable tool-call ID; +- tool name: `content.name`; +- artifact label: the image storage ref, preferring its meaningful path field. + +Current-turn ledger events and provider-message shape alone never grant eligibility. This prevents +the projection from touching the current user attachment or a new image returned after recovery. + +### Provider-message projection + +A focused helper accepts provider `ModelMessage[]` plus the eligible tool-call map. It rewrites +only `tool-result` parts whose `toolCallId` is eligible and whose content output contains an inline +image file. It replaces the image file part with a text part while retaining the surrounding tool +result and all non-image siblings. + +The placeholder names the artifact and the original tool, and tells the model to repeat the +preceding tool call if visual context is required. The helper returns the rewritten messages and +the exact set of tool-call IDs it changed. It never rewrites direct user or assistant image parts. + +### Recovery and replay state + +On the first provider `ContextLength` failure: + +1. Derive eligible historical images from `state.priorContentEvents`. +2. Rewrite all eligible hydrated image results in `currentMessages` in one bounded pass. +3. If at least one result changed, store the returned tool-call IDs in + `state.omittedImageToolCallIds` and spend the existing single overflow retry on the rewritten + request. +4. If none changed, continue to the existing LLM-summary recovery rung. + +For every later request in the same send, reapply the same ID-scoped projection after durable +replay and before dispatch. Reapplying is idempotent. A later image with a different tool-call ID is +preserved. + +The retry latch remains shared: if the image-free retry also receives `ContextLength`, surface the +provider error without another image retry, summarization attempt, or generic provider retry. + +### Removal of proactive behavior + +Delete the work-in-progress image-weighted capacity estimate, step-zero image admission path, +oldest-image proactive omission loops, and their estimate-specific diagnostics. Restore the +existing text capacity estimator and step-zero behavior. Keep the backend guard that prevents a +`context_overflow` failure from falling into byte-identical generic provider retries after the +single recovery path is unavailable or spent. + +### Desktop recovery guidance + +`deriveFailedTurnRecovery` recognizes the stable Runtime error class `context_overflow` before its +generic no-tool retry fallback. It returns the existing `continue` presentation category with a +localized label explaining that the context is still too large and asking the user to reduce +attachments or start a new task. + +This is presentation-only: the existing renderer consumes the label but does not dispatch the +returned action, so the change adds no button or automatic recovery behavior. Other failure +classes keep their existing precedence and copy. + +## Testing + +Use red-green-refactor for each behavior: + +1. Unit projection test: an eligible historical image tool result is replaced with a named + placeholder while its tool-result envelope and text siblings remain intact. +2. Boundary test: direct user images, unlisted tool-call IDs, and later image results remain + unchanged. +3. Reactive integration test: the rejected request contains the hydrated historical image; the + single retry omits it without rerunning the tool or invoking the summarizer. +4. Replay integration test: successor provider requests keep the recorded historical ID omitted + while a new image tool-call ID remains eligible for normal hydration. +5. Termination test: a second context overflow surfaces as a real error without a generic retry + loop. +6. Regression test: selected-model step zero does not proactively remove user or historical + images based on a local image estimate. +7. Desktop presentation test: `context_overflow` returns the dedicated non-retry guidance in both + supported locales, while an unknown output-free failure still returns the generic retry copy. + +Run the focused tests during TDD, then the complete `@maka/runtime` and `@maka/desktop` test suites, +root build and typecheck, lint, format check, and `git diff --check` before completion. diff --git a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts index 4736e78485..8d86d9566d 100644 --- a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts +++ b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts @@ -120,9 +120,15 @@ interface MidTurnFixtureOptions { record?: (checkpoint: HistoryCompactCheckpoint) => void; /** Payload size for each text prior, or for the tool result in a tool-heavy prior. */ priorChars?: number; - priorShape?: 'text' | 'tool_heavy'; + /** Hydrated image byte length when it must be sized independently from text priors. */ + imageBytes?: number; + priorShape?: 'text' | 'tool_heavy' | 'image_tool'; + /** Put one image attachment on the durable current-turn user anchor. */ + currentImage?: boolean; /** First tool result is huge (finding C: prune must be able to rescue it). */ hugeFirstResult?: boolean; + /** Exact first Read result for capacity-ordering regressions. */ + firstResult?: string; /** The model finishes on the second request instead of running three steps. */ finalAtSecondCall?: boolean; /** Add a third tool step whose result outgrows even a rolled-forward fold (finding A). */ @@ -240,18 +246,19 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { }, }); const priorChars = options.priorChars ?? 120; + const imageBytes = options.imageBytes ?? priorChars; const priorEvents: RuntimeEvent[] = options.withoutPriorTurns ? [] - : options.priorShape === 'tool_heavy' + : options.priorShape === 'image_tool' ? [ - runtimeTextEvent('prior-user', 'turn-0', 'user', 'PRIOR_FACT inspect the artifact'), + runtimeTextEvent('prior-user', 'turn-0', 'user', 'PRIOR_IMAGE inspect the screenshot'), { ...runtimeTextEvent('prior-call', 'turn-0', 'model', ''), content: { kind: 'function_call' as const, - id: 'prior-tool-1', + id: 'prior-image-tool-1', name: 'Read', - args: { path: 'artifact.log' }, + args: { path: 'screenshot.png' }, }, }, { @@ -260,30 +267,85 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { author: 'tool' as const, content: { kind: 'function_response' as const, - id: 'prior-tool-1', + id: 'prior-image-tool-1', name: 'Read', - result: `OVERSIZED_TOOL_RESULT_${'r'.repeat(priorChars)}`, + result: { + kind: 'image' as const, + mimeType: 'image/png', + ref: { + kind: 'session_file' as const, + sessionId: 'session-1', + relativePath: 'screenshot.png', + }, + }, isError: false, }, }, - runtimeTextEvent('prior-model', 'turn-0', 'model', 'PRIOR_FACT inspection complete'), + runtimeTextEvent('prior-model', 'turn-0', 'model', 'PRIOR_IMAGE inspection complete'), ] - : [ - runtimeTextEvent( - 'prior-user', - 'turn-0', - 'user', - `PRIOR_FACT question ${'p'.repeat(priorChars)}`, - ), - runtimeTextEvent( - 'prior-model', - 'turn-0', - 'model', - `PRIOR_FACT answer ${'q'.repeat(priorChars)}`, - ), - ]; + : options.priorShape === 'tool_heavy' + ? [ + runtimeTextEvent('prior-user', 'turn-0', 'user', 'PRIOR_FACT inspect the artifact'), + { + ...runtimeTextEvent('prior-call', 'turn-0', 'model', ''), + content: { + kind: 'function_call' as const, + id: 'prior-tool-1', + name: 'Read', + args: { path: 'artifact.log' }, + }, + }, + { + ...runtimeTextEvent('prior-result', 'turn-0', 'model', ''), + role: 'tool' as const, + author: 'tool' as const, + content: { + kind: 'function_response' as const, + id: 'prior-tool-1', + name: 'Read', + result: `OVERSIZED_TOOL_RESULT_${'r'.repeat(priorChars)}`, + isError: false, + }, + }, + runtimeTextEvent('prior-model', 'turn-0', 'model', 'PRIOR_FACT inspection complete'), + ] + : [ + runtimeTextEvent( + 'prior-user', + 'turn-0', + 'user', + `PRIOR_FACT question ${'p'.repeat(priorChars)}`, + ), + runtimeTextEvent( + 'prior-model', + 'turn-0', + 'model', + `PRIOR_FACT answer ${'q'.repeat(priorChars)}`, + ), + ]; const anchor: RuntimeEvent = { ...runtimeTextEvent('anchor-1', 'turn-1', 'user', ANCHOR_TEXT), + ...(options.currentImage + ? { + content: { + kind: 'text' as const, + text: ANCHOR_TEXT, + attachments: [ + { + kind: 'image' as const, + name: 'current.png', + mimeType: 'image/png', + bytes: imageBytes, + ref: { + kind: 'session_file' as const, + sessionId: 'session-1', + relativePath: 'current.png', + }, + }, + ], + }, + } + : {}), ...(options.branch !== undefined ? { branch: options.branch } : {}), }; @@ -355,6 +417,15 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { apiKey: 'sk-test', modelId: 'mock-model-id', modelFactory: () => model, + ...(options.priorShape === 'image_tool' || options.currentImage + ? { + supportsVision: true, + readAttachmentBytes: async () => ({ + ok: true as const, + bytes: new Uint8Array(imageBytes), + }), + } + : {}), tools: [ { name: 'Read', @@ -363,7 +434,9 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { impl: async (args: { path: string }) => { toolExecutions.push(args.path); if (args.path === 'one.md') - return { body: options.hugeFirstResult ? HUGE_RESULT : RAW_SPAN_ONE }; + return { + body: options.firstResult ?? (options.hugeFirstResult ? HUGE_RESULT : RAW_SPAN_ONE), + }; if (args.path === 'three.md') return { body: ROLLING_TAIL }; return { body: RAW_SPAN_TWO }; }, @@ -1433,6 +1506,48 @@ describe('mid-turn capacity compaction flow plumbing', () => { }); describe('mid-turn capacity default-on safety guards (issue #882 PR 3)', () => { + test('does not omit hydrated images before a selected-model step-zero provider verdict', async () => { + const fixture = buildFixture({ + contextWindow: 10_000, + currentImage: true, + imageBytes: 8_000, + priorShape: 'image_tool', + }); + await runFixtureTurn(fixture); + + const firstMessages = fixture.model.doStreamCalls[0]?.prompt ?? []; + const firstPrompt = promptJson(fixture, 0); + assert.equal(fixture.model.doStreamCalls.length > 0, true); + assert.equal(firstPrompt.match(/"mediaType":"image\/png"/g)?.length, 2); + assert.equal( + firstMessages.some( + (message) => + message.role === 'user' && + Array.isArray(message.content) && + message.content.some((part) => part.type === 'file' && part.mediaType === 'image/png'), + ), + true, + ); + assert.equal( + firstMessages.some( + (message) => + message.role === 'tool' && + message.content.some( + (part) => + part.type === 'tool-result' && + part.toolCallId === 'prior-image-tool-1' && + part.output.type === 'content' && + part.output.value.some( + (outputPart) => outputPart.type === 'file' && outputPart.mediaType === 'image/png', + ), + ), + ), + true, + ); + assert.doesNotMatch(firstPrompt, /omitted after provider context overflow/); + assert.equal(fixture.summarizerCalls, 0); + }); + test('keeps the fallback capacity guard inert below its unknown-model bound', async () => { // The unknown model derives a 48,384-token capacity from the default // 32,000-token history budget plus its 16,384-token reserve. This small diff --git a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts index c2fad52423..97356b2be2 100644 --- a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts +++ b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts @@ -86,6 +86,7 @@ type CallKind = | 'gated' | 'done' | 'overflow' + | 'overflow503' | 'overflowPart' | 'overflowPartResponses' | 'toolThenOverflowPart' @@ -107,6 +108,12 @@ interface ReactiveFixtureOptions { midTurnEnabled?: boolean; withoutPriorTurns?: boolean; bigPriors?: boolean; + /** Add one hydrated historical image that fits the proactive capacity estimate. */ + imagePrior?: boolean; + /** Put one image attachment on the durable current-turn user anchor. */ + currentImage?: boolean; + /** Make Read return a newly produced image during this turn. */ + liveImageResult?: boolean; summarize?: () => | Promise | string @@ -229,6 +236,12 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture statusCode: 400, }); } + if (kind === 'overflow503') { + throw Object.assign(new Error(OVERFLOW_MESSAGE), { + name: 'AI_APICallError', + statusCode: 503, + }); + } if (kind === 'error500') { throw Object.assign(new Error('internal server error'), { name: 'AI_APICallError', @@ -375,21 +388,78 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture const priorChars = options.bigPriors ? 4_000 : 120; const priorEvents: RuntimeEvent[] = options.withoutPriorTurns ? [] - : [ - runtimeTextEvent( - 'prior-user', - 'turn-0', - 'user', - `PRIOR_FACT question ${'p'.repeat(priorChars)}`, - ), - runtimeTextEvent( - 'prior-model', - 'turn-0', - 'model', - `PRIOR_FACT answer ${'q'.repeat(priorChars)}`, - ), - ]; - const anchor = runtimeTextEvent('anchor-1', 'turn-1', 'user', ANCHOR_TEXT); + : options.imagePrior + ? [ + runtimeTextEvent('prior-user', 'turn-0', 'user', 'inspect the screenshot'), + { + ...runtimeTextEvent('prior-call', 'turn-0', 'model', ''), + content: { + kind: 'function_call' as const, + id: 'prior-image-call', + name: 'Read', + args: { path: 'screenshot.png' }, + }, + }, + { + ...runtimeTextEvent('prior-result', 'turn-0', 'model', ''), + role: 'tool' as const, + author: 'tool' as const, + content: { + kind: 'function_response' as const, + id: 'prior-image-call', + name: 'Read', + result: { + kind: 'image' as const, + mimeType: 'image/png', + ref: { + kind: 'session_file' as const, + sessionId: 'session-1', + relativePath: 'screenshot.png', + }, + }, + isError: false, + }, + }, + runtimeTextEvent('prior-model', 'turn-0', 'model', 'screenshot inspected'), + ] + : [ + runtimeTextEvent( + 'prior-user', + 'turn-0', + 'user', + `PRIOR_FACT question ${'p'.repeat(priorChars)}`, + ), + runtimeTextEvent( + 'prior-model', + 'turn-0', + 'model', + `PRIOR_FACT answer ${'q'.repeat(priorChars)}`, + ), + ]; + const anchor: RuntimeEvent = { + ...runtimeTextEvent('anchor-1', 'turn-1', 'user', ANCHOR_TEXT), + ...(options.currentImage + ? { + content: { + kind: 'text' as const, + text: ANCHOR_TEXT, + attachments: [ + { + kind: 'image' as const, + name: 'current.png', + mimeType: 'image/png', + bytes: 8, + ref: { + kind: 'session_file' as const, + sessionId: 'session-1', + relativePath: 'current.png', + }, + }, + ], + }, + } + : {}), + }; const ledger: RuntimeEvent[] = [anchor]; const ledgerCtx: InvocationContext = { @@ -462,6 +532,15 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture apiKey: 'sk-test', modelId: 'mock-model-id', modelFactory: () => model, + ...(options.imagePrior || options.currentImage || options.liveImageResult + ? { + supportsVision: true, + readAttachmentBytes: async () => ({ + ok: true as const, + bytes: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]), + }), + } + : {}), ...(options.maxSteps !== undefined ? { maxSteps: options.maxSteps } : {}), tools: [ { @@ -470,6 +549,17 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture parameters: z.object({ path: z.string() }), impl: async (args: { path: string }) => { toolExecutions.push(args.path); + if (options.liveImageResult) { + return { + kind: 'image' as const, + mimeType: 'image/png', + ref: { + kind: 'session_file' as const, + sessionId: 'session-1', + relativePath: 'live.png', + }, + }; + } return { body: args.path === 'big.md' ? BIG_RESULT : RAW_SPAN_ONE }; }, }, @@ -919,6 +1009,68 @@ describe('reactive overflow recovery in the streaming backend', () => { assert.equal(lastCall?.totalTokens, 250); }); + test('drops hydrated images before the single overflow retry without rerunning tools', async () => { + const fixture = buildReactiveFixture({ + script: ['tool', 'overflow', 'done'], + imagePrior: true, + currentImage: true, + liveImageResult: true, + }); + await runTurn(fixture); + + assert.equal(fixture.model.doStreamCalls.length, 3); + assert.equal(complete(fixture)?.stopReason, 'end_turn'); + assert.deepEqual(fixture.toolExecutions, ['one.md']); + assert.equal(fixture.summarizerCalls(), 0); + assert.equal(fixture.recorded.length, 0); + + const rejectedPrompt = JSON.stringify(fixture.model.doStreamCalls[1]?.prompt); + const retryPrompt = JSON.stringify(fixture.model.doStreamCalls[2]?.prompt); + assert.equal(rejectedPrompt.match(/"mediaType":"image\/png"/g)?.length, 3); + assert.equal(retryPrompt.match(/"mediaType":"image\/png"/g)?.length, 2); + assert.match( + retryPrompt, + /Image artifact \\"screenshot\.png\\" omitted after provider context overflow/, + ); + }); + + test('surfaces a retryable second overflow after the image-only retry without another loop', async () => { + const fixture = buildReactiveFixture({ + script: ['tool', 'overflow503', 'overflow503'], + imagePrior: true, + }); + await runTurn(fixture); + + assert.equal(fixture.model.doStreamCalls.length, 3); + assert.equal(complete(fixture)?.stopReason, 'error'); + assert.deepEqual(fixture.toolExecutions, ['one.md']); + assert.equal(fixture.summarizerCalls(), 0); + assert.equal(fixture.recorded.length, 0); + }); + + test('keeps only the historical image omitted after the overflow retry returns a tool call', async () => { + const fixture = buildReactiveFixture({ + script: ['tool', 'overflow', 'tool', 'done'], + imagePrior: true, + currentImage: true, + liveImageResult: true, + }); + await runTurn(fixture); + + assert.equal(fixture.model.doStreamCalls.length, 4); + assert.equal(complete(fixture)?.stopReason, 'end_turn'); + assert.equal(fixture.summarizerCalls(), 0); + const expectedImages = [2, 3]; + for (const [index, call] of fixture.model.doStreamCalls.slice(2).entries()) { + const prompt = JSON.stringify(call.prompt); + assert.equal(prompt.match(/"mediaType":"image\/png"/g)?.length, expectedImages[index]); + assert.match( + prompt, + /Image artifact \\"screenshot\.png\\" omitted after provider context overflow/, + ); + } + }); + test('keeps a step-0 overflow checkpoint projected after the retry returns a tool call', async () => { // The first request overflows before any step completes. Recovery folds // only prior history into a pre-turn checkpoint and retries with the diff --git a/packages/runtime/src/__tests__/provider-image-overflow-recovery.test.ts b/packages/runtime/src/__tests__/provider-image-overflow-recovery.test.ts new file mode 100644 index 0000000000..234eebd853 --- /dev/null +++ b/packages/runtime/src/__tests__/provider-image-overflow-recovery.test.ts @@ -0,0 +1,108 @@ +import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; + +import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { ModelMessage } from '../model-protocol.js'; +import { + collectHistoricalImageToolResults, + omitHistoricalImageToolResults, +} from '../provider-image-overflow-recovery.js'; + +function imageResultEvent(toolCallId: string, relativePath: string): RuntimeEvent { + return { + id: `event-${toolCallId}`, + sessionId: 'session-1', + invocationId: 'run-1', + runId: 'run-1', + turnId: 'turn-0', + seq: 1, + ts: 1, + role: 'tool', + author: 'tool', + partial: false, + content: { + kind: 'function_response', + id: toolCallId, + name: 'Read', + result: { + kind: 'image', + mimeType: 'image/png', + ref: { kind: 'session_file', sessionId: 'session-1', relativePath }, + }, + isError: false, + }, + } as unknown as RuntimeEvent; +} + +function imageFile(data: string): Record { + return { + type: 'file', + data: { type: 'data', data }, + mediaType: 'image/png', + }; +} + +function toolImageMessage(toolCallId: string, data: string): ModelMessage { + return { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId, + toolName: 'Read', + output: { + type: 'content', + value: [{ type: 'text', text: 'Image read successfully.' }, imageFile(data)], + }, + }, + ], + } as ModelMessage; +} + +function prompt(messages: readonly ModelMessage[]): string { + return JSON.stringify(messages); +} + +describe('provider image overflow recovery projection', () => { + test('omits only eligible historical tool-result images and names their artifact', () => { + const priorEvents = [imageResultEvent('prior-image-call', 'screenshots/screenshot.png')]; + const messages = [ + { + role: 'user', + content: [{ type: 'text', text: 'compare these' }, imageFile('USER_IMAGE')], + } as unknown as ModelMessage, + toolImageMessage('prior-image-call', 'PRIOR_IMAGE'), + toolImageMessage('new-image-call', 'NEW_IMAGE'), + ]; + + const eligible = collectHistoricalImageToolResults(priorEvents); + const result = omitHistoricalImageToolResults(messages, eligible); + const rendered = prompt(result.messages); + + assert.deepEqual([...result.omittedToolCallIds], ['prior-image-call']); + assert.equal(result.omittedParts, 1); + assert.equal(rendered.includes('USER_IMAGE'), true); + assert.equal(rendered.includes('NEW_IMAGE'), true); + assert.equal(rendered.includes('PRIOR_IMAGE'), false); + assert.match(rendered, /screenshots\/screenshot\.png/); + assert.match(rendered, /repeat the preceding Read tool call/i); + }); + + test('preserves the tool-result envelope and siblings without mutating its input', () => { + const messages = [toolImageMessage('prior-image-call', 'PRIOR_IMAGE')]; + const original = structuredClone(messages); + const eligible = collectHistoricalImageToolResults([ + imageResultEvent('prior-image-call', 'screenshot.png'), + ]); + + const first = omitHistoricalImageToolResults(messages, eligible); + const second = omitHistoricalImageToolResults(first.messages, eligible); + const firstJson = prompt(first.messages); + + assert.deepEqual(messages, original); + assert.match(firstJson, /"type":"tool-result"/); + assert.match(firstJson, /Image read successfully/); + assert.equal(second.omittedParts, 0); + assert.deepEqual(second.messages, first.messages); + }); +}); diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 33fbba7737..552a92d0f2 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -2574,6 +2574,7 @@ export class AiSdkBackend implements AgentBackend { incompleteStreamHasNoObservableOutput; if ( (failure.retryable || idleWatchdogRecovery || incompleteStreamRecovery) && + failure.kind !== 'context_overflow' && providerAttempt < MAX_PROVIDER_ATTEMPTS_PER_STEP && stepBudgetRemains && (attemptHasNoObservableOutput() || idleWatchdogRecovery || incompleteStreamRecovery) diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index f57d90aee8..4bbbf0f264 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -93,6 +93,11 @@ import { resolveContextBudgetCapacity, type ContextBudgetCapacity, } from './context-budget-policy.js'; +import { + collectHistoricalImageToolResults, + type HistoricalImageToolResult, + omitHistoricalImageToolResults, +} from './provider-image-overflow-recovery.js'; /** * Image byte allowance for one turn, accumulated across its provider steps. @@ -1322,10 +1327,15 @@ export class AiSdkCompaction { return async (options) => { const incomingMessages = options.messages; - const projectedMessages = projectAcceptedActiveCompactionMessages( + let projectedMessages = projectAcceptedActiveCompactionMessages( incomingMessages, acceptedProjection, ); + projectedMessages = + projectHistoricalImageOmissions( + projectedMessages ?? incomingMessages, + state.omittedImageToolResults, + ) ?? projectedMessages; const keepProjection = (): RequestProjection | undefined => projectedMessages ? { messages: projectedMessages } : undefined; // Step 0 is shaped by the pre_turn path; the mid-turn trigger only runs @@ -1791,6 +1801,20 @@ export class AiSdkCompaction { if (input.retryAlreadyUsed || !state) return undefined; if (this.modelAdapter.classifyError(input.error) !== 'ContextLength') return undefined; + const eligibleImages = collectHistoricalImageToolResults(state.priorContentEvents); + const imageOmission = omitHistoricalImageToolResults(input.currentMessages, eligibleImages); + if (imageOmission.omittedParts > 0) { + state.omittedImageToolResults = new Map( + [...imageOmission.omittedToolCallIds].flatMap((toolCallId) => { + const image = eligibleImages.get(toolCallId); + return image ? [[toolCallId, image] as const] : []; + }), + ); + state.lastRequestPayloadChars = undefined; + state.lastRequestInputTokens = undefined; + return { messages: imageOmission.messages }; + } + // The shrink baseline is the request the provider actually rejected. Its // single owner is the verdict owner's per-request payload measure // (state.lastRequestPayloadChars), recorded at the end of every @@ -1925,6 +1949,13 @@ export class AiSdkCompaction { } = input; return async (options) => { let result = await Promise.resolve(shaped(options)); + const omissionProjection = projectHistoricalImageOmissions( + result?.messages ?? options.messages, + state.omittedImageToolResults, + ); + if (omissionProjection) { + result = { ...(result ?? {}), messages: omissionProjection }; + } const finalPayloadChars = (): number => midTurnRequestPayloadChars( result?.messages ?? options.messages, @@ -2076,6 +2107,15 @@ interface AcceptedActiveCompactionProjection { semanticBlock?: SemanticCompactBlock; } +function projectHistoricalImageOmissions( + messages: readonly ModelMessage[], + omittedImageToolResults: ReadonlyMap, +): ModelMessage[] | undefined { + if (omittedImageToolResults.size === 0) return undefined; + const omission = omitHistoricalImageToolResults(messages, omittedImageToolResults); + return omission.omittedParts > 0 ? omission.messages : undefined; +} + function projectAcceptedActiveCompactionMessages( incomingMessages: readonly ModelMessage[], acceptedProjection: AcceptedActiveCompactionProjection | undefined, @@ -2160,12 +2200,9 @@ export function hasActiveToolResultPruneDiagnosticPatch( */ export class MidTurnCapacityCompactState { /** - * Chars of the final (system prompt + messages + active tool schema) - * payload of the LAST prepared request, recorded by the final-request - * estimate owner at the end of every request-projection pipeline run. All capacity estimates are signed - * deltas against this number, so they are anchored to the request the - * provider actually saw — a compacted projection, a pruned tail, or a - * same-turn tool-schema expansion all move the delta the same way. + * Raw serialized chars of the final provider request. Overflow recovery + * uses this as its shrink-reference baseline because it must compare the + * actual rejected projection with a candidate replacement. */ lastRequestPayloadChars: number | undefined; /** @@ -2203,6 +2240,8 @@ export class MidTurnCapacityCompactState { * trigger. Consumed by the capacity hook on its next invocation. */ forcedTriggerEstimate: number | undefined; + /** Exact historical image results omitted after a provider overflow. */ + omittedImageToolResults = new Map(); /** * The capacity hook's most recent shaping failure. The owner reads it (for * the same step only) to pick the terminal detail and diagnostic reason @@ -2229,7 +2268,8 @@ export class MidTurnCapacityCompactState { * (sent through the separate `system` field), the (projected) messages, and * the serialized schemas of the active tool subset. The capacity trigger and * the final-request estimate owner both measure with this ONE function, so - * their deltas against `lastRequestPayloadChars` are commensurable and + * their raw payload comparisons against `lastRequestPayloadChars` are + * commensurable and * same-turn tool-schema growth (a `load_tools` activation) is counted like * any other payload growth. The system prompt is constant between adjacent * requests — signed deltas cancel it — but the cold-start estimate (no usable diff --git a/packages/runtime/src/provider-image-overflow-recovery.ts b/packages/runtime/src/provider-image-overflow-recovery.ts new file mode 100644 index 0000000000..f06e959979 --- /dev/null +++ b/packages/runtime/src/provider-image-overflow-recovery.ts @@ -0,0 +1,130 @@ +import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { ModelMessage } from './model-protocol.js'; + +export interface HistoricalImageToolResult { + toolName: string; + artifactLabel: string; +} + +export interface HistoricalImageOmissionResult { + messages: ModelMessage[]; + omittedParts: number; + omittedToolCallIds: Set; +} + +type UnknownRecord = Record; + +function isRecord(value: unknown): value is UnknownRecord { + return value !== null && typeof value === 'object'; +} + +function storageRefLabel(value: unknown): string | undefined { + if (!isRecord(value) || typeof value.kind !== 'string') return undefined; + if ( + (value.kind === 'session_file' || value.kind === 'workspace_file') && + typeof value.relativePath === 'string' && + value.relativePath.length > 0 + ) { + return value.relativePath; + } + if ( + value.kind === 'external_file' && + typeof value.absolutePath === 'string' && + value.absolutePath.length > 0 + ) { + return value.absolutePath; + } + return undefined; +} + +export function collectHistoricalImageToolResults( + events: readonly RuntimeEvent[], +): Map { + const collected = new Map(); + for (const event of events) { + const content = event.content; + if (content?.kind !== 'function_response' || content.isError === true) continue; + const result = content.result; + if ( + !isRecord(result) || + result.kind !== 'image' || + typeof result.mimeType !== 'string' || + result.mimeType.length === 0 + ) { + continue; + } + const artifactLabel = storageRefLabel(result.ref); + if (!artifactLabel) continue; + collected.set(content.id, { toolName: content.name, artifactLabel }); + } + return collected; +} + +function isInlineImageFilePart(value: unknown): value is UnknownRecord { + if ( + !isRecord(value) || + value.type !== 'file' || + typeof value.mediaType !== 'string' || + !value.mediaType.toLowerCase().startsWith('image/') || + !isRecord(value.data) + ) { + return false; + } + return value.data.type === 'data'; +} + +function omissionText(image: HistoricalImageToolResult): UnknownRecord { + return { + type: 'text', + text: `[Image artifact ${JSON.stringify(image.artifactLabel)} omitted after provider context overflow. Repeat the preceding ${image.toolName} tool call if visual context is required.]`, + }; +} + +function rewriteToolResultPart( + part: unknown, + eligible: ReadonlyMap, + omittedToolCallIds: Set, +): { part: unknown; omittedParts: number } { + if ( + !isRecord(part) || + part.type !== 'tool-result' || + typeof part.toolCallId !== 'string' || + !isRecord(part.output) || + part.output.type !== 'content' || + !Array.isArray(part.output.value) + ) { + return { part, omittedParts: 0 }; + } + const image = eligible.get(part.toolCallId); + if (!image) return { part, omittedParts: 0 }; + + let omittedParts = 0; + const value = part.output.value.map((contentPart) => { + if (!isInlineImageFilePart(contentPart)) return contentPart; + omittedParts += 1; + return omissionText(image); + }); + if (omittedParts === 0) return { part, omittedParts: 0 }; + omittedToolCallIds.add(part.toolCallId); + return { part: { ...part, output: { ...part.output, value } }, omittedParts }; +} + +export function omitHistoricalImageToolResults( + messages: readonly ModelMessage[], + eligible: ReadonlyMap, +): HistoricalImageOmissionResult { + const omittedToolCallIds = new Set(); + let omittedParts = 0; + const rewritten = messages.map((message) => { + if (!Array.isArray(message.content)) return message; + let changed = false; + const content = (message.content as unknown[]).map((part) => { + const result = rewriteToolResultPart(part, eligible, omittedToolCallIds); + if (result.part !== part) changed = true; + omittedParts += result.omittedParts; + return result.part; + }); + return changed ? ({ ...message, content } as ModelMessage) : message; + }); + return { messages: rewritten, omittedParts, omittedToolCallIds }; +}