diff --git a/packages/cli/src/__tests__/runtime-host-run-command.test.ts b/packages/cli/src/__tests__/runtime-host-run-command.test.ts index ea104295f3..38e1b04454 100644 --- a/packages/cli/src/__tests__/runtime-host-run-command.test.ts +++ b/packages/cli/src/__tests__/runtime-host-run-command.test.ts @@ -2,10 +2,17 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; import type { SessionEvent } from '@maka/core/events'; import type { SessionSummary, StoredMessage } from '@maka/core/session'; +import { + createRuntimeHostSessionProjectionSeed, + RuntimeHostSessionProjector, +} from '@maka/runtime-host/adapter'; import { LOCAL_RUNTIME_HOST_PROFILE, type RuntimeHostConnection } from '@maka/runtime-host/client'; -import type { - InteractionPendingSnapshot, - SessionCatalogProjection, +import { + SESSION_CONTINUITY_SCHEMA_VERSION, + type InteractionPendingSnapshot, + type SessionCatalogProjection, + type SessionContinuitySnapshot, + type SubscriptionFrame, } from '@maka/runtime-host/protocol'; import { resolveRuntimeHostCliTarget } from '../runtime-host-cli-context.js'; import { createRuntimeHostRunContext, runRuntimeHostTextCli } from '../runtime-host-run-command.js'; @@ -200,6 +207,133 @@ describe('Runtime Host maka run adapter', () => { ]); }); + test('returns exit code 1 when an ordinary Host Turn fails', async () => { + const stderr: string[] = []; + const fixture = runFixture({ turnEvents: failedEvents('turn-1', 'provider_failure') }); + const exitCode = await runFixtureCommand(fixture, ['fail once'], undefined, (text) => + stderr.push(text), + ); + + assert.equal(exitCode, 1); + assert.equal(stderr.join(''), 'maka run: Turn failed\n'); + }); + + test('returns exit code 1 when a same-step sibling succeeds after a sandbox failure', async () => { + const stdout: string[] = []; + const stderr: string[] = []; + const fixture = runFixture({ + turnEvents: projectedSameStepSandboxFailureEvents('turn-1'), + }); + const exitCode = await runFixtureCommand( + fixture, + ['run parallel tools'], + (text) => stdout.push(text), + (text) => stderr.push(text), + ); + + assert.equal(exitCode, 1); + assert.equal(stdout.join(''), ''); + assert.equal( + stderr.join(''), + 'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n', + ); + }); + + test('returns exit code 1 when a denied boundary request follows a sandbox failure', async () => { + const stdout: string[] = []; + const stderr: string[] = []; + const fixture = runFixture({ + turnEvents: sandboxBoundaryEvents( + 'turn-1', + 'step-1', + 'step-2', + 'Boundary was not widened', + 'request_sandbox_boundary', + ), + }); + const exitCode = await runFixtureCommand( + fixture, + ['request inaccessible work'], + (text) => stdout.push(text), + (text) => stderr.push(text), + ); + + assert.equal(exitCode, 1); + assert.equal(stdout.join(''), ''); + assert.equal( + stderr.join(''), + 'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n', + ); + }); + + test('returns exit code 0 when the final Graph Turn completes', async () => { + const stdout: string[] = []; + const fixture = runFixture({ graph: true }); + const exitCode = await runFixtureCommand(fixture, ['delegate once', '--graph'], (text) => + stdout.push(text), + ); + + assert.equal(exitCode, 0); + assert.equal(stdout.join(''), 'Final graph answer\n'); + }); + + test('returns exit code 0 when a root Graph boundary failure recovers', async () => { + const stdout: string[] = []; + const fixture = runFixture({ + graph: true, + turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'), + }); + const exitCode = await runFixtureCommand(fixture, ['recover once', '--graph'], (text) => + stdout.push(text), + ); + + assert.equal(exitCode, 0); + assert.equal(stdout.join(''), 'Final graph answer\n'); + }); + + test('returns exit code 1 when a same-step Graph sibling succeeds after a sandbox failure', async () => { + const fixture = runFixture({ + graph: true, + finalMessages: sandboxBoundaryMessages('step-1', 'step-1'), + }); + const exitCode = await runFixtureCommand(fixture, ['run parallel tools', '--graph']); + + assert.equal(exitCode, 1); + }); + + test('returns exit code 1 when a denied Graph boundary request follows a sandbox failure', async () => { + const stdout: string[] = []; + const fixture = runFixture({ + graph: true, + finalMessages: sandboxBoundaryMessages('step-1', 'step-2', 'request_sandbox_boundary'), + }); + const exitCode = await runFixtureCommand( + fixture, + ['request inaccessible work', '--graph'], + (text) => stdout.push(text), + ); + + assert.equal(exitCode, 1); + assert.equal(stdout.join(''), ''); + }); + + test('returns exit code 1 when the final Graph Turn fails', async () => { + const stderr: string[] = []; + const fixture = runFixture({ + graph: true, + finalMessages: failedGraphMessages('provider_failure'), + }); + const exitCode = await runFixtureCommand( + fixture, + ['delegate once', '--graph'], + undefined, + (text) => stderr.push(text), + ); + + assert.equal(exitCode, 1); + assert.equal(stderr.join(''), 'maka run: Agent Graph final Turn failed\n'); + }); + test('waits for Host-started graph supervisor Turns before returning', async () => { const observed: MakaRunOutcome[] = []; const fixture = runFixture({ observed, graph: true, graphProjectionRace: true }); @@ -246,6 +380,87 @@ describe('Runtime Host maka run adapter', () => { assert.equal(observed.at(-1)?.finalOutput, 'Final graph answer'); }); + test('reports a recovered sandbox boundary from live and durable Turns', async () => { + const live = await observeFixtureOutcome({ + turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'), + }); + const durable = await observeFixtureOutcome({ + graph: true, + finalMessages: sandboxBoundaryMessages('step-1', 'step-2'), + }); + + assert.equal(live.sandboxBoundary, 'recovered'); + assert.equal(durable.sandboxBoundary, 'recovered'); + }); + + test('leaves sandbox failures unresolved when their provider steps are unavailable', async () => { + const live = await observeFixtureOutcome({ + turnEvents: sandboxBoundaryEvents('turn-1', undefined, undefined, 'Incomplete answer'), + }); + const durable = await observeFixtureOutcome({ + graph: true, + finalMessages: sandboxBoundaryMessages(undefined, undefined), + }); + + assert.equal(live.sandboxBoundary, 'unresolved'); + assert.equal(durable.sandboxBoundary, 'unresolved'); + }); + + test('returns a recovered boundary to unresolved after a later sandbox failure', async () => { + const outcome = await observeFixtureOutcome({ + turnEvents: sandboxFailureAfterRecoveryEvents('turn-1'), + }); + + assert.equal(outcome.sandboxBoundary, 'unresolved'); + }); + + test('classifies live and durable Turn cancellations as aborted', async () => { + const live = await observeFixtureOutcome({ turnEvents: abortedEvents('turn-1') }); + const durable = await observeFixtureOutcome({ + graph: true, + finalMessages: abortedGraphMessages(), + }); + + assert.equal(live.status, 'failed'); + assert.equal(live.failure?.class, 'aborted'); + assert.equal(durable.status, 'failed'); + assert.equal(durable.failure?.class, 'aborted'); + }); + + test('classifies live and durable step-cap failures equally', async () => { + const live = await observeFixtureOutcome({ + turnEvents: completionEvents('turn-1', 'step_limit'), + }); + const durable = await observeFixtureOutcome({ + graph: true, + finalMessages: failedGraphMessages('tool_step_cap_reached'), + }); + + assert.equal(live.status, 'failed'); + assert.equal(live.failure?.class, 'tool_step_cap_reached'); + assert.equal(durable.status, 'failed'); + assert.equal(durable.failure?.class, 'tool_step_cap_reached'); + }); + + test('classifies a standalone context-budget completion as failed', async () => { + const outcome = await observeFixtureOutcome({ + turnEvents: completionEvents('turn-1', 'context_budget_exhausted'), + }); + + assert.equal(outcome.status, 'failed'); + assert.equal(outcome.failure?.class, 'context_budget_exhausted'); + }); + + test('uses the latest durable terminal state for a Graph Turn', async () => { + const outcome = await observeFixtureOutcome({ + graph: true, + finalMessages: failedThenCompletedGraphMessages(), + }); + + assert.equal(outcome.status, 'completed'); + assert.equal(outcome.finalOutput, 'Final graph answer'); + }); + test('waits for the exact final Graph wake after an earlier wake already settled', async () => { const observed: MakaRunOutcome[] = []; const fixture = runFixture({ observed, graph: true, graphMultiWakeRace: true }); @@ -736,26 +951,25 @@ function runFixture(input: { throw new Error(`Unexpected operation: ${operation}`); }, } as unknown as RuntimeHostConnection; + const createContext = (contextInput: MakaRunContextInput) => + createRuntimeHostRunContext(connection, connectionCatalog(), contextInput, { + createDriver: () => driver, + }); let create = () => - createRuntimeHostRunContext( - connection, - connectionCatalog(), - { - workspaceRoot: '/data', - cwd: '/workspace', - ...(input.graph ? { enableAgentGraph: true } : {}), - ...(input.maxSteps ? { maxSteps: input.maxSteps } : {}), - ...(input.sessionCwdOverride ? { sessionCwdOverride: input.sessionCwdOverride } : {}), - ...(input.observed - ? { - runOutcomeObserver: (result: MakaRunOutcome) => { - input.observed?.push(result); - }, - } - : {}), - }, - { createDriver: () => driver }, - ); + createContext({ + workspaceRoot: '/data', + cwd: '/workspace', + ...(input.graph ? { enableAgentGraph: true } : {}), + ...(input.maxSteps ? { maxSteps: input.maxSteps } : {}), + ...(input.sessionCwdOverride ? { sessionCwdOverride: input.sessionCwdOverride } : {}), + ...(input.observed + ? { + runOutcomeObserver: (result: MakaRunOutcome) => { + input.observed?.push(result); + }, + } + : {}), + }); return { get context() { const context = create(); @@ -768,6 +982,7 @@ function runFixture(input: { exactTurnStops, preparedMaxSteps, sandboxResponses, + createContext, publishPendingInteraction(pending: InteractionPendingSnapshot) { for (const listener of pendingInteractionListeners) listener(structuredClone(pending)); }, @@ -777,6 +992,65 @@ function runFixture(input: { }; } +async function observeFixtureOutcome( + input: Parameters[0], +): Promise { + const observed: MakaRunOutcome[] = []; + const fixture = runFixture({ ...input, observed }); + const session = await fixture.context.runtime.createSession({ + cwd: '/workspace', + llmConnectionSlug: 'openai-main', + model: 'gpt-5', + permissionMode: 'ask', + }); + await collect( + fixture.context.runtime.sendMessage(session.id, { + turnId: 'turn-1', + text: 'observe outcome', + ...(input.graph + ? { turnOrchestration: { mode: 'graph' as const, source: 'host_api' as const } } + : {}), + }), + ); + if (input.graph) await fixture.context.agentGraph?.waitForCompletion(session.id); + const outcome = observed.at(-1); + assert.ok(outcome); + return outcome; +} + +function runFixtureCommand( + fixture: ReturnType, + argv: readonly string[], + writeStdout: (text: string) => void = () => {}, + writeStderr: (text: string) => void = () => {}, +): Promise { + return runRuntimeHostTextCli( + argv, + { ...publicCommandEnvironment(), writeStdout, writeStderr }, + { + connect: async () => ({ + connection: readinessConnection(), + catalog: connectionCatalog(), + profile: LOCAL_RUNTIME_HOST_PROFILE, + close: async () => {}, + }), + createContext: (_connection, _catalog, input) => fixture.createContext(input), + }, + ); +} + +function publicCommandEnvironment() { + return { + workspaceRoot: () => '/runtime-host-data', + processCwd: () => process.cwd(), + stdinIsTTY: () => true, + readStdin: async () => '', + writeStderr: () => {}, + onSigint: () => () => {}, + newId: () => 'turn-1', + }; +} + function connectionCatalog() { return { revision: 1, @@ -977,6 +1251,76 @@ function graphMessages(includeTerminal = true): StoredMessage[] { return messages; } +function sandboxBoundaryMessages( + failureStepId: string | undefined, + successStepId: string | undefined, + successToolName = 'Read', +): StoredMessage[] { + const sameStep = failureStepId !== undefined && failureStepId === successStepId; + return [ + ...graphMessages(false), + ...(failureStepId === undefined ? [] : [storedToolCall('turn-2', 'tool-1', failureStepId, 5)]), + ...(sameStep ? [storedToolCall('turn-2', 'tool-2', successStepId, 6, successToolName)] : []), + sandboxFailureToolResult('turn-2', 7), + ...(successStepId === undefined || sameStep + ? [] + : [storedToolCall('turn-2', 'tool-2', successStepId, 8, successToolName)]), + successfulToolResult('turn-2', 9), + { + type: 'turn_state', + id: 'state-turn-2', + turnId: 'turn-2', + ts: 10, + status: 'completed', + partialOutputRetained: true, + }, + ]; +} + +function abortedGraphMessages(): StoredMessage[] { + return [ + ...graphMessages(false), + { + type: 'turn_state', + id: 'state-turn-2', + turnId: 'turn-2', + ts: 5, + status: 'aborted', + abortSource: 'user_interrupt', + partialOutputRetained: true, + }, + ]; +} + +function failedGraphMessages(errorClass: string): StoredMessage[] { + return [ + ...graphMessages(false), + { + type: 'turn_state', + id: 'state-turn-2', + turnId: 'turn-2', + ts: 5, + status: 'failed', + errorClass, + partialOutputRetained: true, + }, + ]; +} + +function failedThenCompletedGraphMessages(): StoredMessage[] { + return [ + ...failedGraphMessages('provider_failure'), + { + type: 'turn_state', + id: 'completed-state-turn-2', + turnId: 'turn-2', + ts: 6, + status: 'completed', + partialOutputRetained: true, + }, + ]; +} + function multiWakeGraphMessages(includeFinalTerminal: boolean): StoredMessage[] { const messages = [ ...graphMessages(), @@ -1015,16 +1359,242 @@ function multiWakeGraphMessages(includeFinalTerminal: boolean): StoredMessage[] return messages; } -async function* eventsFor(turnId: string, text: string): AsyncIterable { +async function* eventsFor(turnId: string, text: string, ts = 1): AsyncIterable { yield { type: 'text_complete', id: `${turnId}-text`, turnId, messageId: `${turnId}-message`, - ts: 1, + ts, text, }; - yield { type: 'complete', id: `${turnId}-complete`, turnId, ts: 2, stopReason: 'end_turn' }; + yield { type: 'complete', id: `${turnId}-complete`, turnId, ts: ts + 1, stopReason: 'end_turn' }; +} + +async function* sandboxBoundaryEvents( + turnId: string, + failureStepId: string | undefined, + successStepId: string | undefined, + text: string, + successToolName = 'Read', +): AsyncIterable { + const sameStep = failureStepId !== undefined && failureStepId === successStepId; + if (failureStepId !== undefined) yield toolStart(turnId, 'tool-1', failureStepId, 1); + if (sameStep) yield toolStart(turnId, 'tool-2', successStepId, 2, successToolName); + yield sandboxFailureToolResult(turnId, 3); + if (successStepId !== undefined && !sameStep) { + yield toolStart(turnId, 'tool-2', successStepId, 4, successToolName); + } + yield successfulToolResult(turnId, 5); + yield* eventsFor(turnId, text, 6); +} + +async function* projectedSameStepSandboxFailureEvents(turnId: string): AsyncIterable { + yield toolStart(turnId, 'tool-1', 'step-1', 1); + yield toolStart(turnId, 'tool-2', 'step-1', 2); + const initial = continuitySnapshot(turnId); + const projector = new RuntimeHostSessionProjector( + initial, + createRuntimeHostSessionProjectionSeed([], initial), + () => 10, + ); + yield* projector.accept({ + kind: 'subscription.session_event', + hostEpoch: 'host-1', + subscriptionId: 'subscription-1', + sequence: 1, + sessionId: 'session-created', + runId: 'run-1', + event: { + type: 'tool_result', + id: 'tool-1-result', + turnId, + ts: 3, + toolUseId: 'tool-1', + status: 'errored', + sandboxFailureReason: 'sandbox_boundary_required', + }, + } satisfies SubscriptionFrame).events; + yield successfulToolResult(turnId, 4); + yield* eventsFor(turnId, 'Incomplete answer', 5); +} + +function continuitySnapshot( + turnId: string, + overrides: Partial = {}, +): SessionContinuitySnapshot { + return { + schemaVersion: SESSION_CONTINUITY_SCHEMA_VERSION, + session: { + sessionId: 'session-created', + metadataRevision: 1, + status: 'running', + createdAt: 1, + lastUsedAt: 1, + isArchived: false, + }, + projectionRevision: 1, + rootTurn: { + sessionId: 'session-created', + turnId, + runId: 'run-1', + status: 'running', + }, + goal: null, + queue: { hostEpoch: 'host-1', queueRevision: 0, steering: [], followup: [] }, + interactions: { pending: [] }, + ...overrides, + }; +} + +async function* sandboxFailureAfterRecoveryEvents(turnId: string): AsyncIterable { + yield toolStart(turnId, 'tool-1', 'step-1', 1); + yield sandboxFailureToolResult(turnId, 2); + yield toolStart(turnId, 'tool-2', 'step-2', 3); + yield successfulToolResult(turnId, 4); + yield toolStart(turnId, 'tool-3', 'step-3', 5); + yield sandboxFailureToolResult(turnId, 6, 'tool-3'); + yield* eventsFor(turnId, 'Incomplete answer', 7); +} + +async function* abortedEvents(turnId: string): AsyncIterable { + yield { + type: 'abort', + id: `${turnId}-abort`, + turnId, + ts: 1, + reason: 'user_stop', + }; + yield { + type: 'complete', + id: `${turnId}-complete`, + turnId, + ts: 2, + stopReason: 'user_stop', + }; +} + +async function* failedEvents(turnId: string, reason: string): AsyncIterable { + yield { + type: 'text_complete', + id: `${turnId}-text`, + turnId, + messageId: `${turnId}-message`, + ts: 1, + text: 'Partial answer', + }; + yield { + type: 'error', + id: `${turnId}-error`, + turnId, + ts: 2, + recoverable: false, + reason, + message: 'Turn failed', + }; + yield { + type: 'complete', + id: `${turnId}-complete`, + turnId, + ts: 3, + stopReason: 'error', + }; +} + +async function* completionEvents( + turnId: string, + stopReason: Extract['stopReason'], +): AsyncIterable { + yield { + type: 'complete', + id: `${turnId}-complete`, + turnId, + ts: 1, + stopReason, + }; +} + +type SharedToolResult = Extract & + Extract; + +function toolStart( + turnId: string, + toolUseId: string, + stepId: string, + ts: number, + toolName = 'Read', +): Extract { + return { + type: 'tool_start', + id: `${turnId}-${toolUseId}-start`, + turnId, + ts, + toolUseId, + toolName, + args: {}, + stepId, + }; +} + +function storedToolCall( + turnId: string, + toolUseId: string, + stepId: string, + ts: number, + toolName = 'Read', +): Extract { + return { + type: 'tool_call', + id: toolUseId, + turnId, + ts, + toolName, + args: {}, + stepId, + }; +} + +function sandboxFailureToolResult( + turnId: string, + ts: number, + toolUseId = 'tool-1', +): SharedToolResult { + return { + type: 'tool_result', + id: `${turnId}-${toolUseId}-sandbox-failure`, + turnId, + ts, + toolUseId, + isError: true, + content: sandboxFailureContent(), + }; +} + +function successfulToolResult(turnId: string, ts: number): SharedToolResult { + return { + type: 'tool_result', + id: `${turnId}-tool-success`, + turnId, + ts, + toolUseId: 'tool-2', + isError: false, + content: { kind: 'text', text: 'ok' }, + }; +} + +function sandboxFailureContent() { + return { + kind: 'text' as const, + text: 'Write requires an approved sandbox boundary expansion.', + sandboxFailure: { + reason: 'sandbox_boundary_required' as const, + requiredExpansion: { + filesystem: { + entries: [{ path: '/outside', access: 'write' as const, scope: 'subtree' as const }], + }, + }, + }, + }; } async function* eventsAfterPendingNotification( diff --git a/packages/cli/src/cli-core.ts b/packages/cli/src/cli-core.ts index ad2865e677..a87ee4d57e 100644 --- a/packages/cli/src/cli-core.ts +++ b/packages/cli/src/cli-core.ts @@ -1,7 +1,6 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { formatMakaResumeHint } from './cli-invocation.js'; import { resolveMakaDataRoots } from './workspace-root.js'; import { parseRuntimeHostCommand, type RuntimeHostCliCommand } from './runtime-host-cli.js'; import { resolveCliUiLocale } from './cli-ui-locale.js'; @@ -172,13 +171,6 @@ function helpText(cliCommand: string): string { ].join('\n'); } -export function formatResumeHint( - sessionId: string | null, - cliCommand: string = RELEASE_MAKA_CLI_LAUNCH_OPTIONS.cliCommand, -): string | null { - return formatMakaResumeHint(cliCommand, sessionId); -} - export async function runMakaCli( argv: string[] = process.argv.slice(2), options: MakaCliLaunchOptions = RELEASE_MAKA_CLI_LAUNCH_OPTIONS, diff --git a/packages/cli/src/run-command-core.ts b/packages/cli/src/run-command-core.ts index 90e88f6d9f..e0279c7957 100644 --- a/packages/cli/src/run-command-core.ts +++ b/packages/cli/src/run-command-core.ts @@ -260,7 +260,7 @@ export async function runMakaTextCliCore( } let outcome: MakaRunOutcome | undefined; - let streamBoundaryFailure = false; + let unclassifiedBoundaryFailure = false; const boundaryFailureInvocationIds = new Set(); let context: MakaRunContext; try { @@ -293,8 +293,10 @@ export async function runMakaTextCliCore( runOutcomeObserver: (result) => { if (result.sandboxBoundary === 'recovered') { boundaryFailureInvocationIds.delete(result.outcomeId); + unclassifiedBoundaryFailure = false; } else if (result.sandboxBoundary === 'unresolved') { boundaryFailureInvocationIds.add(result.outcomeId); + unclassifiedBoundaryFailure = false; } outcome = result; }, @@ -377,7 +379,7 @@ export async function runMakaTextCliCore( : {}), })) { if (event.type === 'sandbox_boundary_request') { - streamBoundaryFailure = true; + unclassifiedBoundaryFailure = true; deps.writeStderr( 'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n', ); @@ -388,7 +390,7 @@ export async function runMakaTextCliCore( } const sandboxFailureReason = sessionEventSandboxBoundaryFailureReason(event); if (sandboxFailureReason) { - streamBoundaryFailure = true; + unclassifiedBoundaryFailure = true; deps.writeStderr( sandboxFailureReason === 'requires_bypass' ? 'maka run: sandbox bypass requires an explicit --yolo\n' @@ -420,10 +422,7 @@ export async function runMakaTextCliCore( return 1; } if (streamFailed) return 1; - if ( - (streamBoundaryFailure && outcome?.sandboxBoundary !== 'recovered') || - boundaryFailureInvocationIds.size > 0 - ) { + if (unclassifiedBoundaryFailure || boundaryFailureInvocationIds.size > 0) { return 1; } if (!outcome) { diff --git a/packages/cli/src/runtime-host-run-command.ts b/packages/cli/src/runtime-host-run-command.ts index 44b557dc27..bf5121f817 100644 --- a/packages/cli/src/runtime-host-run-command.ts +++ b/packages/cli/src/runtime-host-run-command.ts @@ -1,4 +1,4 @@ -import { type SessionEvent } from '@maka/core/events'; +import { failureClassFromCompleteStopReason, type SessionEvent } from '@maka/core/events'; import { findProjectByIdentity } from '@maka/core/project'; import { type StoredMessage } from '@maka/core/session'; import type { CreateSessionInput, UserMessageInput } from '@maka/core/runtime-inputs'; @@ -408,7 +408,7 @@ class RuntimeHostRunRuntime implements MakaRunRuntime { } async *#observeTurn(turn: MakaPreparedSessionTurn): AsyncIterable { - const accumulator = new TurnOutcomeAccumulator(turn.runId ?? turn.turnId); + const classifier = new TurnOutcomeClassifier(turn.runId ?? turn.turnId); const events = turn.events[Symbol.asyncIterator](); for (;;) { const next = await this.#interactions.race(events.next()); @@ -417,11 +417,11 @@ class RuntimeHostRunRuntime implements MakaRunRuntime { if (event.type === 'user_question_request' || event.type === 'sandbox_boundary_request') { continue; } - accumulator.accept(event); + classifier.accept(observationFromSessionEvent(event)); yield event; } await this.#interactions.settle(); - await this.#observer?.(accumulator.finish()); + await this.#observer?.(classifier.outcome('fail')); } async #stopTurn(turn: { sessionId: string; turnId: string; runId: string }): Promise { @@ -505,60 +505,219 @@ function runtimeHostSessionSummaries(items: readonly SessionCatalogItem[]): Sess return items.flatMap((item) => ('kind' in item ? [] : [runtimeHostSessionSummary(item)])); } -class TurnOutcomeAccumulator { +type TurnOutcomeObservation = + | { readonly kind: 'output'; readonly text: string } + | { + readonly kind: 'terminal'; + readonly update: 'replace' | 'if_unset'; + readonly status: 'completed'; + } + | { + readonly kind: 'terminal'; + readonly update: 'replace' | 'if_unset'; + readonly status: 'failed'; + readonly failure: NonNullable; + } + | { + readonly kind: 'tool_call'; + readonly toolUseId: string; + readonly stepId: string | undefined; + readonly toolName: string; + } + | { + readonly kind: 'tool_result'; + readonly toolUseId: string; + readonly outcome: 'sandbox_failure' | 'success'; + }; + +type TerminalOutcomeObservation = Extract; +type SandboxBoundaryState = + | { readonly status: 'none' } + | { readonly status: 'unresolved'; readonly failedStepId: string | undefined } + | { readonly status: 'recovered' }; + +class TurnOutcomeClassifier { readonly #outcomeId: string; + readonly #callByToolUseId = new Map< + string, + { readonly stepId: string | undefined; readonly toolName: string } + >(); #finalOutput: string | undefined; - #failure: { class: string; message: string } | undefined; - #completed = false; - #unresolvedBoundary = false; - #recoveredBoundary = false; + #terminal: TerminalOutcomeObservation | undefined; + #sandboxBoundary: SandboxBoundaryState = { status: 'none' }; constructor(outcomeId: string) { this.#outcomeId = outcomeId; } - accept(event: SessionEvent): void { - if (event.type === 'text_complete' && event.text.trim().length > 0) { - this.#finalOutput = event.text; - } else if (event.type === 'error') { - this.#failure = { class: event.reason ?? 'runtime_error', message: event.message }; - } else if (event.type === 'abort') { - this.#failure = { class: 'aborted', message: 'Turn was cancelled' }; - } else if (event.type === 'complete') { - this.#completed = true; - } - if (event.type !== 'tool_result') return; - if (event.isError && event.content.kind === 'text' && event.content.sandboxFailure) { - this.#unresolvedBoundary = true; - return; - } - if (!event.isError && this.#unresolvedBoundary) { - this.#unresolvedBoundary = false; - this.#recoveredBoundary = true; + accept(observation: TurnOutcomeObservation | undefined): void { + switch (observation?.kind) { + case undefined: + return; + case 'output': + this.#finalOutput = observation.text; + return; + case 'terminal': + if (observation.update === 'replace' || this.#terminal === undefined) { + this.#terminal = observation; + } + return; + case 'tool_call': + this.#callByToolUseId.set(observation.toolUseId, { + stepId: observation.stepId, + toolName: observation.toolName, + }); + return; + case 'tool_result': { + const call = this.#callByToolUseId.get(observation.toolUseId); + if (observation.outcome === 'sandbox_failure') { + this.#sandboxBoundary = { status: 'unresolved', failedStepId: call?.stepId }; + return; + } + if ( + observation.outcome === 'success' && + call?.toolName !== 'request_sandbox_boundary' && + this.#sandboxBoundary.status === 'unresolved' && + call?.stepId !== undefined && + this.#sandboxBoundary.failedStepId !== undefined && + call.stepId !== this.#sandboxBoundary.failedStepId + ) { + this.#sandboxBoundary = { status: 'recovered' }; + } + return; + } } } - finish(): MakaRunOutcome { - const completed = this.#completed && !this.#failure; + outcome(incomplete: 'fail'): MakaRunOutcome; + outcome(incomplete: 'pending'): MakaRunOutcome | undefined; + outcome(incomplete: 'fail' | 'pending'): MakaRunOutcome | undefined { + const terminal = this.#terminal; + if (!terminal && incomplete === 'pending') return undefined; + const completed = terminal?.status === 'completed'; + const failure = + terminal?.status === 'failed' + ? terminal.failure + : { + class: 'missing_terminal_event', + message: 'Turn ended unexpectedly', + }; return { outcomeId: this.#outcomeId, status: completed ? 'completed' : 'failed', ...(completed && this.#finalOutput !== undefined ? { finalOutput: this.#finalOutput } : {}), - ...(!completed - ? { - failure: this.#failure ?? { - class: 'missing_terminal_event', - message: 'Turn ended unexpectedly', - }, - } - : {}), - sandboxBoundary: this.#unresolvedBoundary - ? 'unresolved' - : this.#recoveredBoundary - ? 'recovered' - : 'none', + ...(!completed ? { failure } : {}), + sandboxBoundary: this.#sandboxBoundary.status, + }; + } +} + +function observationFromSessionEvent(event: SessionEvent): TurnOutcomeObservation | undefined { + if (event.type === 'text_complete' && event.text.trim().length > 0) { + return { kind: 'output', text: event.text }; + } + if (event.type === 'error') { + return { + kind: 'terminal', + update: 'replace', + status: 'failed', + failure: { class: event.reason ?? event.code ?? 'runtime_error', message: event.message }, + }; + } + if (event.type === 'abort') { + return { + kind: 'terminal', + update: 'replace', + status: 'failed', + failure: { class: 'aborted', message: 'Turn was cancelled' }, + }; + } + if (event.type === 'complete') { + return observationFromCompleteEvent(event); + } + if (event.type === 'tool_start') { + return { + kind: 'tool_call', + toolUseId: event.toolUseId, + stepId: event.stepId, + toolName: event.toolName, }; } + return event.type === 'tool_result' ? observationFromToolResult(event) : undefined; +} + +function observationFromStoredMessage(message: StoredMessage): TurnOutcomeObservation | undefined { + if (message.type === 'assistant' && message.text.trim().length > 0) { + return { kind: 'output', text: message.text }; + } + if (message.type === 'turn_state' && message.status === 'completed') { + return { kind: 'terminal', update: 'replace', status: 'completed' }; + } + if (message.type === 'turn_state' && message.status === 'aborted') { + return { + kind: 'terminal', + update: 'replace', + status: 'failed', + failure: { class: 'aborted', message: 'Turn was cancelled' }, + }; + } + if (message.type === 'turn_state' && message.status === 'failed') { + return { + kind: 'terminal', + update: 'replace', + status: 'failed', + failure: { + class: message.errorClass ?? 'runtime_error', + message: 'Agent Graph final Turn failed', + }, + }; + } + if (message.type === 'tool_call') { + return { + kind: 'tool_call', + toolUseId: message.id, + stepId: message.stepId, + toolName: message.toolName, + }; + } + return message.type === 'tool_result' ? observationFromToolResult(message) : undefined; +} + +function observationFromCompleteEvent( + event: Extract, +): TerminalOutcomeObservation { + if (event.stopReason === 'user_stop') { + return { + kind: 'terminal', + update: 'if_unset', + status: 'failed', + failure: { class: 'aborted', message: 'Turn was cancelled' }, + }; + } + const failureClass = failureClassFromCompleteStopReason(event.stopReason); + return failureClass + ? { + kind: 'terminal', + update: 'if_unset', + status: 'failed', + failure: { class: failureClass }, + } + : { kind: 'terminal', update: 'if_unset', status: 'completed' }; +} + +function observationFromToolResult( + result: Pick, 'content' | 'isError' | 'toolUseId'>, +): TurnOutcomeObservation | undefined { + if (result.isError && result.content.kind === 'text' && result.content.sandboxFailure) { + return { + kind: 'tool_result', + toolUseId: result.toolUseId, + outcome: 'sandbox_failure', + }; + } + return result.isError + ? undefined + : { kind: 'tool_result', toolUseId: result.toolUseId, outcome: 'success' }; } function graphSupervisorTurnIds(messages: readonly StoredMessage[]): Set { @@ -587,36 +746,11 @@ function outcomeFromStoredTurn( messages: readonly StoredMessage[], turnId: string, ): MakaRunOutcome | undefined { - const turnMessages = messages.filter((message) => message.turnId === turnId); - const finalOutput = [...turnMessages] - .reverse() - .find( - (message): message is Extract => - message.type === 'assistant' && message.text.trim().length > 0, - )?.text; - const storedTerminal = [...turnMessages] - .reverse() - .find( - (message): message is Extract => - message.type === 'turn_state' && message.status !== 'running', - ); - const status = storedTerminal?.status; - if (!status) return undefined; - const completed = status === 'completed'; - return { - outcomeId: turnId, - status: completed ? 'completed' : 'failed', - ...(completed && finalOutput !== undefined ? { finalOutput } : {}), - ...(!completed - ? { - failure: { - class: storedTerminal?.errorClass ?? storedTerminal?.abortSource ?? status, - message: status === 'aborted' ? 'Turn was cancelled' : 'Agent Graph final Turn failed', - }, - } - : {}), - sandboxBoundary: storedSandboxBoundaryOutcome(turnMessages), - }; + const classifier = new TurnOutcomeClassifier(turnId); + for (const message of messages) { + if (message.turnId === turnId) classifier.accept(observationFromStoredMessage(message)); + } + return classifier.outcome('pending'); } class NonInteractiveInteractionController { @@ -692,27 +826,6 @@ class NonInteractiveInteractionController { } } -function storedSandboxBoundaryOutcome( - messages: readonly StoredMessage[], -): MakaRunOutcome['sandboxBoundary'] { - let unresolved = false; - let recovered = false; - for (const message of messages) { - if ( - message.type === 'tool_result' && - message.isError && - message.content.kind === 'text' && - message.content.sandboxFailure - ) { - unresolved = true; - } else if (message.type === 'tool_result' && !message.isError && unresolved) { - unresolved = false; - recovered = true; - } - } - return unresolved ? 'unresolved' : recovered ? 'recovered' : 'none'; -} - function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/packages/cli/src/sandbox-boundary-failure.ts b/packages/cli/src/sandbox-boundary-failure.ts index 97e39b4bf2..1c3a1b8385 100644 --- a/packages/cli/src/sandbox-boundary-failure.ts +++ b/packages/cli/src/sandbox-boundary-failure.ts @@ -1,7 +1,6 @@ import type { SessionEvent } from '@maka/core/events'; -import type { InvocationResult } from '@maka/runtime/invocation-context'; -export type SandboxBoundaryFailureReason = 'sandbox_boundary_required' | 'requires_bypass'; +type SandboxBoundaryFailureReason = 'sandbox_boundary_required' | 'requires_bypass'; export function sessionEventSandboxBoundaryFailureReason( event: SessionEvent, @@ -17,42 +16,6 @@ export function sessionEventSandboxBoundaryFailureReason( return normalizeSandboxBoundaryFailureReason(event.content.sandboxFailure.reason); } -export function invocationHasSandboxBoundaryFailure(result: InvocationResult): boolean { - return result.events.some( - (event) => runtimeEventSandboxBoundaryFailureReason(event) !== undefined, - ); -} - -export function invocationRecoveredSandboxBoundaryFailure( - result: InvocationResult | undefined, -): boolean { - if (!invocationCompletedWithOutput(result)) return false; - let unresolvedBoundaryFailure = false; - let recoveredBoundaryFailure = false; - for (const event of result.events) { - if (event.content?.kind !== 'function_response') continue; - if (event.content.isError && runtimeEventSandboxBoundaryFailureReason(event)) { - unresolvedBoundaryFailure = true; - continue; - } - if (unresolvedBoundaryFailure && !event.content.isError) { - unresolvedBoundaryFailure = false; - recoveredBoundaryFailure = true; - } - } - return recoveredBoundaryFailure && !unresolvedBoundaryFailure; -} - -function runtimeEventSandboxBoundaryFailureReason( - event: InvocationResult['events'][number], -): SandboxBoundaryFailureReason | undefined { - if (event.content?.kind !== 'function_response' || !isRecord(event.content.result)) { - return undefined; - } - const failure = event.content.result.sandboxFailure; - return isRecord(failure) ? normalizeSandboxBoundaryFailureReason(failure.reason) : undefined; -} - function normalizeSandboxBoundaryFailureReason( reason: unknown, ): SandboxBoundaryFailureReason | undefined { @@ -60,13 +23,3 @@ function normalizeSandboxBoundaryFailureReason( ? reason : undefined; } - -function invocationCompletedWithOutput( - result: InvocationResult | undefined, -): result is InvocationResult & { status: 'completed'; finalOutput: string } { - return result?.status === 'completed' && typeof result.finalOutput === 'string'; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} diff --git a/packages/cli/src/session-recap.ts b/packages/cli/src/session-recap.ts index 1459a2adae..8489dc851c 100644 --- a/packages/cli/src/session-recap.ts +++ b/packages/cli/src/session-recap.ts @@ -1,8 +1,3 @@ -import { cleanSessionRecapText, SESSION_RECAP_INSTRUCTION } from '@maka/runtime/session-recap'; - -export const RECAP_INSTRUCTION = SESSION_RECAP_INSTRUCTION; -export const cleanRecapText = cleanSessionRecapText; - /** Idle gap (ms) after which the first normal prompt on return triggers an automatic recap. */ export const AUTO_RECAP_IDLE_MS = 180_000; /** Minimum main-turn count (user-prompted turns) before an automatic recap may fire. */ @@ -10,11 +5,6 @@ export const AUTO_RECAP_MIN_TURNS = 3; /** Raw-output size (bytes) above which an automatic recap is not surfaced in the transcript (still persisted). */ export const AUTO_RECAP_DISPLAY_LIMIT_BYTES = 500; -/** - * Cleans a raw model recap response: collapses whitespace, strips a leading - * `Recap:` / `Summary:` / `回顾:`-style label, strips one layer of wrapping - * quotes, and truncates to 1200 characters (with an ellipsis) if needed. - */ export interface ShouldAutoRecapInput { /** Milliseconds since the last recorded user activity. */ idleMs: number; diff --git a/packages/cli/src/tui-diff.ts b/packages/cli/src/tui-diff.ts index 97f39090a1..db668b73ed 100644 --- a/packages/cli/src/tui-diff.ts +++ b/packages/cli/src/tui-diff.ts @@ -9,7 +9,7 @@ import { ansi } from './tui-ansi.js'; * the transcript at all. Hunk headers stay: they are the only line-number * anchor in the terminal rendering. */ -export function colorDiffRow(row: UnifiedDiffRow): string { +function colorDiffRow(row: UnifiedDiffRow): string { switch (row.kind) { case 'add': return ansi.green(row.text); diff --git a/packages/runtime-host/src/__tests__/protocol.test.ts b/packages/runtime-host/src/__tests__/protocol.test.ts index a1a7b3810a..15418d091e 100644 --- a/packages/runtime-host/src/__tests__/protocol.test.ts +++ b/packages/runtime-host/src/__tests__/protocol.test.ts @@ -98,6 +98,10 @@ describe('Runtime Host bootstrap protocol', () => { assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 25); }); + test('publishes a new compatibility epoch for sandbox failure results', () => { + assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 29); + }); + test('selects the highest mutually supported protocol and rejects a gap', () => { assert.equal(negotiateProtocol({ min: 0, max: 0 }, { min: 0, max: 0 }), 0); assert.equal(negotiateProtocol({ min: 1, max: 3 }, { min: 2, max: 4 }), 3); @@ -253,6 +257,12 @@ describe('Runtime Host bootstrap protocol', () => { }, { ...identity, type: 'tool_progress', chunk: 'working' }, { ...identity, type: 'tool_result', status: 'completed', durationMs: 3 }, + { + ...identity, + type: 'tool_result', + status: 'errored', + sandboxFailureReason: 'sandbox_boundary_required', + }, { ...identity, type: 'tool_result_preview', @@ -288,6 +298,18 @@ describe('Runtime Host bootstrap protocol', () => { status: 'errored', error: 'raw provider error', }, + { + ...identity, + type: 'tool_result', + status: 'errored', + sandboxFailureReason: 'raw provider error', + }, + { + ...identity, + type: 'tool_result', + status: 'completed', + sandboxFailureReason: 'requires_bypass', + }, { ...identity, type: 'tool_result_preview', diff --git a/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts b/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts index 9dc5e30a56..382dbf7aa9 100644 --- a/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts @@ -1808,6 +1808,49 @@ test('tool_result clears retained tool_result_preview so a later open does not s coordinator.close(); }); +test('publishes only the minimal sandbox failure reason from a tool result', async () => { + const coordinator = new SessionContinuityCoordinator( + HOST_EPOCH, + async () => canonical(), + new SessionAdmissionGate(), + ); + const sink = new RecordingSink(); + const connection = coordinator.attachConnection('connection-1', sink); + const opened = await open(coordinator, 'connection-1'); + connection.activate(opened.subscriptionId); + + await coordinator.acceptRuntimeEvent(SESSION_ID, 'run-1', { + type: 'tool_result', + id: 'result-1', + turnId: 'turn-1', + ts: 2, + toolUseId: 'tool-1', + isError: true, + content: { + kind: 'text', + text: 'sensitive tool output', + sandboxFailure: { reason: 'sandbox_boundary_required' }, + }, + }); + await waitFor(() => sink.frames.length === 1); + + const [frame] = sink.frames; + assert.equal(frame?.kind, 'subscription.session_event'); + if (frame?.kind !== 'subscription.session_event') return; + assert.deepEqual(frame.event, { + type: 'tool_result', + id: 'result-1', + turnId: 'turn-1', + ts: 2, + toolUseId: 'tool-1', + status: 'errored', + sandboxFailureReason: 'sandbox_boundary_required', + }); + + connection.abort(opened.subscriptionId); + coordinator.close(); +}); + class RecordingSink implements SessionContinuityFrameSink { readonly frames: SubscriptionFrame[] = []; diff --git a/packages/runtime-host/src/adapter/session-projector.ts b/packages/runtime-host/src/adapter/session-projector.ts index 6a37dc2457..192229e552 100644 --- a/packages/runtime-host/src/adapter/session-projector.ts +++ b/packages/runtime-host/src/adapter/session-projector.ts @@ -506,7 +506,13 @@ function projectToolEvent( type: 'tool_result', ...base, isError: event.status === 'errored', - content: { kind: 'text', text: '' }, + content: { + kind: 'text', + text: '', + ...(event.sandboxFailureReason + ? { sandboxFailure: { reason: event.sandboxFailureReason } } + : {}), + }, ...(event.operationId ? { operationId: event.operationId } : {}), ...(event.durationMs === undefined ? {} : { durationMs: event.durationMs }), }; diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index 1de8a73624..895fe9f3ca 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -72,7 +72,9 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 31 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 32 as const; +// 32: Live tool results may carry the bounded sandbox failure reason. Older +// Clients reject that closed-frame addition, so mixed peers must not connect. // 31: `claude-subscription` leaves `OAUTH_LOGIN_PROVIDERS` and the // `oauth.account.usage.fetch` operation is removed with the provider that // needed its client identity. An older peer still offers both. diff --git a/packages/runtime-host/src/protocol/session-continuity.ts b/packages/runtime-host/src/protocol/session-continuity.ts index cb713f3c88..b2a10c3c16 100644 --- a/packages/runtime-host/src/protocol/session-continuity.ts +++ b/packages/runtime-host/src/protocol/session-continuity.ts @@ -1,5 +1,5 @@ import { TOOL_ACTIVITY_KINDS, TOOL_OUTPUT_DELTA_MAX_CHARS } from '@maka/core/events'; -import type { ToolResultPreviewContent } from '@maka/core/events'; +import type { SandboxBoundaryFailureSignal, ToolResultPreviewContent } from '@maka/core/events'; import { decodeToolResultPreviewContent } from '@maka/core/tool-result-preview'; import type { ToolActivityKind } from '@maka/core/events'; import type { SessionStatus } from '@maka/core/session'; @@ -162,6 +162,7 @@ export type SessionToolEvent = type: 'tool_result'; operationId?: string; status: 'completed' | 'errored'; + sandboxFailureReason?: SandboxBoundaryFailureSignal['reason']; durationMs?: number; }) | (SessionToolEventIdentity & { @@ -801,6 +802,7 @@ function decodeSessionToolEvent(value: unknown): SessionToolEvent { 'toolUseId', 'operationId', 'status', + 'sandboxFailureReason', 'durationMs', ]; assertAllowedKeys(record, 'Session tool result event', allowed); @@ -815,6 +817,9 @@ function decodeSessionToolEvent(value: unknown): SessionToolEvent { if (record.status !== 'completed' && record.status !== 'errored') { throw invalidProtocolFrame('Invalid Session tool result status'); } + if (record.status === 'completed' && record.sandboxFailureReason !== undefined) { + throw invalidProtocolFrame('Completed Session tool result cannot carry a sandbox failure'); + } return { type: record.type, ...identity, @@ -822,6 +827,9 @@ function decodeSessionToolEvent(value: unknown): SessionToolEvent { ? {} : { operationId: requireEntityId(record.operationId, 'operationId') }), status: record.status, + ...(record.sandboxFailureReason === undefined + ? {} + : { sandboxFailureReason: requireSandboxFailureReason(record.sandboxFailureReason) }), ...(record.durationMs === undefined ? {} : { @@ -858,6 +866,11 @@ function decodeSessionToolEvent(value: unknown): SessionToolEvent { throw invalidProtocolFrame('Invalid Session tool event type'); } +function requireSandboxFailureReason(value: unknown): SandboxBoundaryFailureSignal['reason'] { + if (value === 'sandbox_boundary_required' || value === 'requires_bypass') return value; + throw invalidProtocolFrame('Invalid Session tool result sandbox failure reason'); +} + function decodeSessionContinuityIdentity(value: unknown): SessionContinuityIdentity { const record = requireRecord(value, 'Session continuity identity'); assertAllowedKeys(record, 'Session continuity identity', [ diff --git a/packages/runtime-host/src/server/session-continuity-coordinator.ts b/packages/runtime-host/src/server/session-continuity-coordinator.ts index 02d50c741b..e7096ffba7 100644 --- a/packages/runtime-host/src/server/session-continuity-coordinator.ts +++ b/packages/runtime-host/src/server/session-continuity-coordinator.ts @@ -1956,6 +1956,9 @@ function projectToolEvent( ...identity, ...(event.operationId === undefined ? {} : { operationId: event.operationId }), status: event.isError ? 'errored' : 'completed', + ...(event.isError && event.content.kind === 'text' && event.content.sandboxFailure + ? { sandboxFailureReason: event.content.sandboxFailure.reason } + : {}), ...(event.durationMs === undefined ? {} : { durationMs: event.durationMs }), }; case 'tool_result_preview':