From ccd1bcefe8eca2c512232b5e3a1e6d236b80e47f Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:45:47 +0300 Subject: [PATCH 001/155] docs(hooks): note that the Bash matcher does not cover git tools --- src/hooks/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hooks/README.md b/src/hooks/README.md index 8db87f9a7..8adb91dc8 100644 --- a/src/hooks/README.md +++ b/src/hooks/README.md @@ -30,6 +30,8 @@ User and plugin commands that run before and after a tool call, in the Claude Co `matcher` is a regular expression tested against the whole tool name (empty or `*` = every tool). It is tested against our name (`os.shell.run`) and the Claude name from [hook-matcher.ts](hook-matcher.ts): `Bash`, `Read`, `Write`, `Edit` (also `os.fs.patch`), `MultiEdit`, `Glob`, `Grep`, `LS`, `WebFetch`, `WebSearch`, `Task` (`fusion.delegate`), and `mcp____` for `mcp..`. +`Bash` matches only `os.shell.run`. The built-in git tools (`os.git.*`) have no Claude name, so a `Bash` hook does not see them. A hook that should see git calls too needs its own matcher: `Bash|os\.git\..*`, written in `hooks.json` as `"matcher": "Bash|os\\.git\\..*"`. + ## Input and decisions stdin is JSON: `session_id`, `cwd`, `hook_event_name`, `tool_name` (Claude name when there is one), `atomic_tool_name`, `claude_tool_name`, `tool_input` (our arguments plus `file_path`, `command`, `old_string`, `new_string`), and for PostToolUse `tool_response` (`status`, `output`, `truncated`). From d07d964eff328e902f3c9d0008de5c363dd2d590 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:50:22 +0300 Subject: [PATCH 002/155] Retry with a shorter reply when a provider time limit ends a long one (ATO-1398) --- src/agent/agent-contract.ts | 14 ++ src/agent/agent-loop.test.ts | 107 ++++++++++++++ src/agent/agent-loop.ts | 5 + src/agent/docs/recovery.md | 4 +- .../turn/mid-reply-timeout-recovery.test.ts | 136 ++++++++++++++++++ src/agent/turn/mid-reply-timeout-recovery.ts | 67 +++++++++ src/agent/turn/truncation-recovery.ts | 2 +- src/agent/turn/turn-recovery.test.ts | 2 +- src/agent/turn/turn-recovery.ts | 57 ++++++++ src/agent/turn/turn-state.ts | 3 + src/cli/trace-formatter.ts | 1 + src/llm/reliability/mid-reply-timeout.test.ts | 41 ++++++ src/llm/reliability/mid-reply-timeout.ts | 45 ++++++ src/tracing/trace/trace-event.ts | 14 ++ src/tracing/trace/trace-recorder.ts | 13 ++ src/tui/agent-event-reducer.test.ts | 42 ++++++ src/tui/agent-event-reducer.ts | 10 ++ src/tui/issue-report/trace-redaction.ts | 1 + src/tui/provider-outage-state.ts | 24 ++-- 19 files changed, 576 insertions(+), 12 deletions(-) create mode 100644 src/agent/turn/mid-reply-timeout-recovery.test.ts create mode 100644 src/agent/turn/mid-reply-timeout-recovery.ts create mode 100644 src/llm/reliability/mid-reply-timeout.test.ts create mode 100644 src/llm/reliability/mid-reply-timeout.ts diff --git a/src/agent/agent-contract.ts b/src/agent/agent-contract.ts index a889bd376..0f77c83a2 100644 --- a/src/agent/agent-contract.ts +++ b/src/agent/agent-contract.ts @@ -806,6 +806,20 @@ export type AgentLoopEvent = budget: number; reason: string; } + | { + /** + * A provider time limit ended a long reply to step `stepIndex` and + * the step is being retried with a notice asking for a shorter + * one. Not an outage: the provider is answering. `attempt` counts + * within the step, `budget` is the ceiling after which the turn + * fails. + */ + type: "reply_time_limit_retried"; + stepIndex: number; + attempt: number; + budget: number; + reason: string; + } | { /** * The completion for step `stepIndex` came back with nothing in diff --git a/src/agent/agent-loop.test.ts b/src/agent/agent-loop.test.ts index b4d946c93..14b4ce041 100644 --- a/src/agent/agent-loop.test.ts +++ b/src/agent/agent-loop.test.ts @@ -1578,6 +1578,113 @@ describe("AgentLoop end-to-end with mock LLM", () => { expect(waits[0]?.reason).not.toContain("Tried"); }); + it("does not count a slow compaction as a long reply (ATO-1398)", async () => { + let now = 1_000_000; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + const controller = new AbortController(); + const events: string[] = []; + let compactions = 0; + const loop = new AgentLoop({ + registry: buildDefaultToolRegistry(), + slotManager: new SlotManager(2), + grammar: 'root ::= "ok"', + llmComplete: async () => { + throw new OpenAiHttpError( + "openai provider 504: A Timeout Occurred", + 504, + "https://api.fake.test/v1/chat/completions", + false, + null, + "fake", + undefined, + { streamError: "A Timeout Occurred" }, + ); + }, + compaction: { + open() {}, + close() {}, + beforeStep: (input) => { + compactions += 1; + if (compactions === 1) now += 70_000; + return Promise.resolve(input.session); + }, + }, + onEvent: (event) => { + events.push(event.type); + if (event.type === "provider_waiting") controller.abort(); + }, + toolDescriptors: TOOLS, + capabilities: CAPS, + skillCatalog: SKILLS, + }); + try { + await loop + .runTurn(createEmptySessionState({ id: "s-slow-compaction", workingDir }), { + userMessage: "build a big page", + maxSteps: 5, + taskMaxSteps: 5, + signal: controller.signal, + }) + .catch(() => undefined); + expect(events).toContain("provider_waiting"); + expect(events).not.toContain("reply_time_limit_retried"); + } finally { + clock.mockRestore(); + } + }); + + it("asks for a shorter reply when a time limit ended a long one (ATO-1398)", async () => { + let now = 1_000_000; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + const registry = buildDefaultToolRegistry(); + const requests: string[] = []; + let calls = 0; + const loop = new AgentLoop({ + registry, + slotManager: new SlotManager(2), + grammar: 'root ::= "ok"', + llmComplete: async (request) => { + calls += 1; + requests.push(JSON.stringify(request)); + if (calls === 1) { + now += 340_000; + throw new OpenAiHttpError( + "openai provider 504: A Timeout Occurred", + 504, + "https://api.fake.test/v1/chat/completions", + false, + null, + "fake", + undefined, + { streamError: "A Timeout Occurred" }, + ); + } + return makeCompletion( + JSON.stringify({ tool: "reply", args: { text: "recovered" } }), + ); + }, + toolDescriptors: TOOLS, + capabilities: CAPS, + skillCatalog: SKILLS, + }); + try { + const result = await loop.runTurn( + createEmptySessionState({ id: "s-mid-reply-timeout", workingDir }), + { + userMessage: "build a big page", + maxSteps: 5, + taskMaxSteps: 5, + signal: new AbortController().signal, + }, + ); + expect(result.reason).toBe("reply"); + expect(requests[0]).not.toContain("write it in parts"); + expect(requests[1]).toContain("write it in parts"); + } finally { + clock.mockRestore(); + } + }); + it("does not wait out our own request deadline (issue #490)", async () => { // A first-token timeout is `status === null` — there is no HTTP // response to carry a status — so it used to classify as an outage diff --git a/src/agent/agent-loop.ts b/src/agent/agent-loop.ts index 14b5b7fb6..773e0f9b4 100644 --- a/src/agent/agent-loop.ts +++ b/src/agent/agent-loop.ts @@ -384,6 +384,9 @@ export class AgentLoop { // summary step gets at least its own five minutes — it is // reserved whichever ceiling bit, and llama-server keeps decoding // the abandoned request until it notices the closed connection. + // Reset right before the model call: a slow compaction ahead of it + // must not count as a long reply. + let requestStartedAt = Date.now(); const requestDeadline = createRequestDeadline( options.signal, finalizationStep @@ -642,6 +645,7 @@ export class AgentLoop { }); } } + requestStartedAt = Date.now(); const outcome = await executeStep(stepContext, stepDeps); requestDeadline.dispose(); const durationMs = Date.now() - started; @@ -1037,6 +1041,7 @@ export class AgentLoop { state, options, stepIndex: i, finalizationStep, noticeForThisStep, effectiveTransport, requestDeadline, resumesStoppedTask, taskStartedAt, durationCeilingMs, providerWaitCfg, recoveryStepAvailable, + requestElapsedMs: Date.now() - requestStartedAt, ...(stepModelMode !== undefined ? { modelMode: stepModelMode } : {}), ...(contextExhausted !== undefined ? { contextExhausted } : {}), }, this.deps, turn); diff --git a/src/agent/docs/recovery.md b/src/agent/docs/recovery.md index 52d330c84..2af76009e 100644 --- a/src/agent/docs/recovery.md +++ b/src/agent/docs/recovery.md @@ -9,6 +9,8 @@ Completion parse/validation recovery, transport retry, request-size repair and p A reply cut short (on llama-server, `stop_type: "limit"` without a context overflow counts as the reply cap) is retried by truncation repair with a notice to think less and write a large file in parts. Cloud keeps the cap (two retries per step); local raises it two-fold once, unless the reply ended in a loop. A local reply whose tail repeats one block exactly (at least 256 characters and 8 copies) is aborted mid-stream and retried the same way, at the same cap. A filled window repacks once per step; six retries over back-to-back cut steps end the turn. +A provider time limit that ends a long reply mid-stream (a 408, 504 or 524 or a timeout message reported inside the stream, on a request that ran at least a minute) is not waited out: replaying the same request is cut at the same place. The step is retried with a notice to think less and write a large file in parts, twice at most, and the next timeout ends the turn. It reports `reply_time_limit_retried`, not `provider_waiting`, because the provider is answering. A timeout seconds into a stream, or with `agent.providerWait.enabled` off, keeps the ordinary outage path. + Before that retry, an answer cut at a reply cap (not the window, not a loop) is [continued](../step/answer-continuation.ts) once inside the step: the partial `reply` text is quoted back with a request for the rest only, and the two are joined. A continuation cut again ends the step with what arrived plus a line saying it was cut; one that fails any other way throws the original truncation, and the turn-level retry above runs as before. Never more than one continuation per step. A local thinking profile under the untouched default `localModels.completionMaxTokens` runs under a [lifted cap](../step/thinking-reply-cap.ts) of 32,768 on grammar links, since its think block spends the same budget as the answer; a configured value or `ATOMIC_AGENT_LLAMA_MAX_TOKENS` is kept as is. A request the model server refuses as too large for its context window (llama.cpp `exceed_context_size_error`, an OpenAI-style "maximum context length" 400/413) is compacted once per turn and the same step retried ([overflow compaction](../turn/overflow-compaction.ts)). The compaction plans against the window the server named (llama.cpp's body `n_ctx`, scaled by the agent's estimate over the server's `n_prompt_tokens` when the estimate undercounted), else most of the refused prompt's estimate; that window also drives the rest of the turn's automatic compaction, and only a window the server named is remembered (under the provider that refused) for later turns. A second refusal, or a compaction that frees no room, makes the turn's one `loop_failed` carry a plain "no longer fits this model's context window" error (the server's sentence stays on the cause), unless the turn was stopped or timed out or the refusal blames the reply reserve rather than the conversation. Each attempt and outcome is logged and a successful one emits `prompt_repacked`. @@ -35,6 +37,6 @@ Before that repair completion, call JSON that does not parse gets one mechanical ## Sources and tests -- [Loop](../agent-loop.ts), [detector](../loop-detector.ts), [parse recovery](../turn/parse-failure-recovery.ts), [empty recovery](../turn/empty-completion-recovery.ts), [size repair](../turn/size-rejection-recovery.ts), [overflow compaction](../turn/overflow-compaction.ts), [truncation repair](../turn/truncation-recovery.ts). +- [Loop](../agent-loop.ts), [detector](../loop-detector.ts), [parse recovery](../turn/parse-failure-recovery.ts), [empty recovery](../turn/empty-completion-recovery.ts), [size repair](../turn/size-rejection-recovery.ts), [overflow compaction](../turn/overflow-compaction.ts), [truncation repair](../turn/truncation-recovery.ts), [reply time limit](../turn/mid-reply-timeout-recovery.ts). - [Claims](../policies/claim-evidence.ts), [links](../policies/link-evidence.ts), [read coverage](../progress/read-coverage.ts), [review stall](../review-stall.ts). - [Loop tests](../agent-loop.test.ts), [detector tests](../loop-detector.test.ts), [step tests](../step-executor.test.ts), [repair cap tests](../step/step-parsing.test.ts). diff --git a/src/agent/turn/mid-reply-timeout-recovery.test.ts b/src/agent/turn/mid-reply-timeout-recovery.test.ts new file mode 100644 index 000000000..e9b718e95 --- /dev/null +++ b/src/agent/turn/mid-reply-timeout-recovery.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "vitest"; +import { CancelledError, TransportError } from "../../llm/index.js"; +import { OpenAiHttpError } from "../../llm/provider/openai/openai-http.js"; +import { createEmptySessionState } from "../../session/session-state.js"; +import type { AgentLoopEvent } from "../agent-contract.js"; +import { MID_REPLY_TIMEOUT_MIN_ELAPSED_MS, composeMidReplyTimeoutNotice } from "./mid-reply-timeout-recovery.js"; +import { recoverTurnStep, type TurnRecoveryContext } from "./turn-recovery.js"; +import { createTurnLoopState } from "./turn-state.js"; + +const URL = "https://vendor/v1/chat/completions"; +const LONG = MID_REPLY_TIMEOUT_MIN_ELAPSED_MS; + +function context(overrides: Partial = {}): TurnRecoveryContext { + return { + state: createEmptySessionState({ id: "mid-reply", workingDir: "/work" }), + options: { maxSteps: 4, signal: new AbortController().signal }, + stepIndex: 2, + finalizationStep: false, + noticeForThisStep: undefined, + effectiveTransport: "native_tools", + requestDeadline: { dispose() {}, fired: () => false }, + resumesStoppedTask: false, + taskStartedAt: 0, + durationCeilingMs: 10_000_000, + providerWaitCfg: { enabled: true, maxWaitMs: 300_000 }, + recoveryStepAvailable: () => true, + requestElapsedMs: LONG, + ...overrides, + }; +} + +function midReply(status: number, message: string): TransportError { + return new TransportError(`"openrouter" reported an error in the middle of its reply (${status}): ${message}`, status, URL, { + cause: new OpenAiHttpError(`openai provider ${status}: ${message}`, status, URL, false, null, "openrouter", undefined, { streamError: message }), + }); +} +const timedOut = () => midReply(504, "A Timeout Occurred"); + +describe("recovery from a provider time limit that ends a long reply", () => { + it("retries with a notice to work in smaller pieces instead of waiting", () => { + const turn = createTurnLoopState(); + const events: AgentLoopEvent[] = []; + const decision = recoverTurnStep(timedOut(), context({ noticeForThisStep: "earlier notice" }), { onEvent: (event) => events.push(event) }, turn); + expect(decision).toEqual({ kind: "retry_same" }); + expect(turn.pendingNotice).toContain("earlier notice"); + expect(turn.pendingNotice).toContain("write it in parts"); + expect(turn.midReplyTimeoutRetry).toEqual({ stepIndex: 2, attempts: 1 }); + expect(turn.awaitingRecovery).toBe(false); + expect(turn.outageAttempts).toBe(0); + expect(events).toEqual([expect.objectContaining({ type: "reply_time_limit_retried", stepIndex: 2, attempt: 1, budget: 2 })]); + }); + + it("closes an earlier outage wait before retrying", () => { + const turn = createTurnLoopState(); + turn.awaitingRecovery = true; + turn.outageWaitedMs = 2_000; + turn.outageAttempts = 1; + const events: AgentLoopEvent[] = []; + recoverTurnStep(timedOut(), context(), { onEvent: (event) => events.push(event) }, turn); + expect(events.map((event) => event.type)).toEqual(["provider_recovered", "reply_time_limit_retried"]); + expect(events[0]).toMatchObject({ waitedMs: 2_000 }); + expect(turn.awaitingRecovery).toBe(false); + expect(turn.outageWaitedMs).toBe(0); + expect(turn.outageAttempts).toBe(0); + }); + + it("does not stack the notice on the second retry", () => { + const turn = createTurnLoopState(); + recoverTurnStep(timedOut(), context(), {}, turn); + const carried = turn.pendingNotice; + const decision = recoverTurnStep(timedOut(), context({ noticeForThisStep: carried }), {}, turn); + expect(decision).toEqual({ kind: "retry_same" }); + expect(turn.pendingNotice).toBe(carried); + expect(turn.midReplyTimeoutRetry?.attempts).toBe(2); + }); + + it("ends the turn on the third timeout of one step and says why", () => { + const turn = createTurnLoopState(); + recoverTurnStep(timedOut(), context(), {}, turn); + recoverTurnStep(timedOut(), context(), {}, turn); + const decision = recoverTurnStep(timedOut(), context(), {}, turn); + expect(decision).toMatchObject({ kind: "failure", category: "transport", cancelled: false }); + if (decision.kind !== "failure") throw new Error("expected a failure"); + expect(decision.runError.message).toContain('"openrouter"'); + expect(decision.runError.message).toContain("3 times"); + }); + + it("gives a later step its own budget", () => { + const turn = createTurnLoopState(); + recoverTurnStep(timedOut(), context(), {}, turn); + recoverTurnStep(timedOut(), context(), {}, turn); + expect(recoverTurnStep(timedOut(), context({ stepIndex: 3 }), {}, turn)).toEqual({ kind: "retry_same" }); + expect(turn.midReplyTimeoutRetry).toEqual({ stepIndex: 3, attempts: 1 }); + }); + + it("keeps a request exactly at the threshold and leaves a shorter one to the outage wait", () => { + expect(recoverTurnStep(timedOut(), context({ requestElapsedMs: LONG }), {}, createTurnLoopState())).toEqual({ kind: "retry_same" }); + const turn = createTurnLoopState(); + expect(recoverTurnStep(timedOut(), context({ requestElapsedMs: LONG - 1 }), {}, turn).kind).toBe("wait"); + expect(turn.pendingNotice).toBeUndefined(); + expect(turn.midReplyTimeoutRetry).toBeNull(); + }); + + it("leaves an unmeasured request to the outage wait", () => { + expect(recoverTurnStep(timedOut(), context({ requestElapsedMs: undefined }), {}, createTurnLoopState()).kind).toBe("wait"); + }); + + it("leaves other long failures to the outage wait", () => { + expect(recoverTurnStep(midReply(502, "Network connection lost"), context(), {}, createTurnLoopState()).kind).toBe("wait"); + const onOpen = new TransportError("gateway timeout", 504, URL); + expect(recoverTurnStep(onOpen, context(), {}, createTurnLoopState()).kind).toBe("wait"); + }); + + it("keeps the failure when provider waiting is off", () => { + const turn = createTurnLoopState(); + const decision = recoverTurnStep(timedOut(), context({ providerWaitCfg: { enabled: false, maxWaitMs: 300_000 } }), {}, turn); + expect(decision.kind).toBe("failure"); + expect(turn.midReplyTimeoutRetry).toBeNull(); + }); + + it("does not spend a retry on a stopped turn or the summary step", () => { + const controller = new AbortController(); + controller.abort(); + const stopped = createTurnLoopState(); + expect(recoverTurnStep(timedOut(), context({ options: { maxSteps: 4, signal: controller.signal } }), {}, stopped)).toMatchObject({ kind: "failure", cancelled: true }); + expect(stopped.midReplyTimeoutRetry).toBeNull(); + expect(recoverTurnStep(new CancelledError(), context(), {}, createTurnLoopState())).toMatchObject({ cancelled: true }); + const summary = createTurnLoopState(); + expect(recoverTurnStep(timedOut(), context({ finalizationStep: true }), {}, summary).kind).toBe("stop"); + expect(summary.midReplyTimeoutRetry).toBeNull(); + }); + + it("composes onto an empty notice", () => { + expect(composeMidReplyTimeoutNotice("")).toBe(composeMidReplyTimeoutNotice(undefined)); + }); +}); diff --git a/src/agent/turn/mid-reply-timeout-recovery.ts b/src/agent/turn/mid-reply-timeout-recovery.ts new file mode 100644 index 000000000..957ed09ff --- /dev/null +++ b/src/agent/turn/mid-reply-timeout-recovery.ts @@ -0,0 +1,67 @@ +import { TransportError } from "../../llm/index.js"; +import { readMidReplyTimeout } from "../../llm/reliability/mid-reply-timeout.js"; +import { WRITE_IN_PARTS } from "./truncation-recovery.js"; + +/** + * How the agent loop recovers when a provider's time limit ends a long + * reply (ATO-1398). + * + * Replaying the same request is pointless: a reply too long to finish + * inside the limit is cut at the same place again, and each attempt runs + * for minutes that `providerWait.maxWaitMs` does not count. So the retry + * changes the request: a notice asking for a much shorter reply. After + * the budget the turn ends, saying why. + * + * Only a request that ran for a while qualifies. A 504 reported seconds + * into a stream is an ordinary outage and keeps the outage wait. + */ + +/** A request that ran at least this long before the time limit ended it. */ +export const MID_REPLY_TIMEOUT_MIN_ELAPSED_MS = 60_000; + +/** Retries one step may take; the next timeout on that step ends the turn. */ +export const MID_REPLY_TIMEOUT_STEP_BUDGET = 2; + +const NOTICE = + "Your previous reply to this step ran too long: the provider stopped it on a time limit before it finished, and none of it was kept. " + + `Think less: keep your reasoning brief and make this reply much shorter. ${WRITE_IN_PARTS}`; + +export type MidReplyTimeoutPlan = + | { kind: "retry"; attempts: number } + | { kind: "give_up"; error: TransportError }; + +export function planMidReplyTimeoutRetry(input: { + error: unknown; + /** How long the failed request ran; absent means it was not measured. */ + requestElapsedMs: number | undefined; + /** Retries this step index already took. */ + stepAttempts: number; +}): MidReplyTimeoutPlan | null { + if ( + input.requestElapsedMs === undefined || + input.requestElapsedMs < MID_REPLY_TIMEOUT_MIN_ELAPSED_MS + ) { + return null; + } + const timeout = readMidReplyTimeout(input.error); + if (timeout === null) return null; + if (input.stepAttempts < MID_REPLY_TIMEOUT_STEP_BUDGET) { + return { kind: "retry", attempts: input.stepAttempts + 1 }; + } + return { + kind: "give_up", + error: new TransportError( + `"${timeout.provider}" stopped this step's reply on a time limit ${input.stepAttempts + 1} times, so the step was ended. ` + + "Try a faster model, or ask for the work in smaller steps.", + timeout.status, + timeout.url, + { cause: input.error }, + ), + }; +} + +/** Fold the notice into the step's one-shot notice, once per retried step. */ +export function composeMidReplyTimeoutNotice(existing: string | undefined): string { + if (existing === undefined || existing.length === 0) return NOTICE; + return existing.includes(NOTICE) ? existing : `${existing}\n\n${NOTICE}`; +} diff --git a/src/agent/turn/truncation-recovery.ts b/src/agent/turn/truncation-recovery.ts index 8bab99cd5..d286ab474 100644 --- a/src/agent/turn/truncation-recovery.ts +++ b/src/agent/turn/truncation-recovery.ts @@ -144,7 +144,7 @@ export function truncationRetryMaxTokens( } /** Asked of every cap-bound retry: the way out of a long reply. */ -const WRITE_IN_PARTS = +export const WRITE_IN_PARTS = 'If you are writing a large file, write it in parts: create it with the first part, then add the rest with os.fs.write mode "append" in the next steps.'; /** diff --git a/src/agent/turn/turn-recovery.test.ts b/src/agent/turn/turn-recovery.test.ts index 5f82d2e39..03105ca4f 100644 --- a/src/agent/turn/turn-recovery.test.ts +++ b/src/agent/turn/turn-recovery.test.ts @@ -46,7 +46,7 @@ describe("turn recovery decisions and shared state", () => { const second = createTurnLoopState(); first.truncationRetry = { stepIndex: 1, attempts: 1, original: new Error("cut") }; first.reason = "cancelled"; - expect(Object.keys(second)).toHaveLength(21); + expect(Object.keys(second)).toHaveLength(22); expect(second.truncationRetry).toBeNull(); expect(second.reason).toBe("max_steps"); expect(second.lastBoundaryIndex).toBe(-1); diff --git a/src/agent/turn/turn-recovery.ts b/src/agent/turn/turn-recovery.ts index 78f8fdf67..00188fc6d 100644 --- a/src/agent/turn/turn-recovery.ts +++ b/src/agent/turn/turn-recovery.ts @@ -11,6 +11,7 @@ import { readProviderErrorVerdict } from "../../llm/reliability/provider-error-v import { classifyFailureCause } from "../../llm/reliability/classify-failure-cause.js"; import { classifyProviderWaitCause } from "../../llm/reliability/provider-wait-cause.js"; import { composeSizeRejectionNotice, planSizeRejectionRepack } from "./size-rejection-recovery.js"; +import { MID_REPLY_TIMEOUT_STEP_BUDGET, composeMidReplyTimeoutNotice, planMidReplyTimeoutRetry } from "./mid-reply-timeout-recovery.js"; import { composeTruncationNotice, planTruncationRetry, truncationRetryMaxTokens, type TruncationRetry, type TruncationRetryPlan } from "./truncation-recovery.js"; import { PARSE_RECOVERY_BUDGET, composeParseFailureNotice, isRecoverableParseFailure } from "./parse-failure-recovery.js"; import { EMPTY_COMPLETION_RECOVERY_BUDGET, composeEmptyCompletionNotice, isRecoverableEmptyCompletion, repeatedEmptyCompletionError } from "./empty-completion-recovery.js"; @@ -39,6 +40,8 @@ export interface TurnRecoveryContext { recoveryStepAvailable: (stepIndex: number) => boolean; /** The model mode serving the step; absent (legacy wiring) reads as `local`. */ modelMode?: "local" | "cloud"; + /** How long the failed step's request ran. */ + requestElapsedMs?: number; /** * The plain error the turn ends with when it fails on a context-size * refusal after the loop's overflow compaction (ATO-1028). Replaces the @@ -506,6 +509,60 @@ export function recoverTurnStep( return { kind: "local_out_of_memory", contextWindow: outOfMemory.contextWindow, cause: runError }; } } + // A provider's time limit ended a long reply. Waiting and replaying + // the same request hits the same limit, so the retry asks for less. + // Bounded per step; a request that ran only briefly falls through to + // the outage wait below. + if (category === "transport" && !cancelled && !creditRefused && providerWaitCfg.enabled) { + const stepAttempts = + turn.midReplyTimeoutRetry?.stepIndex === i ? turn.midReplyTimeoutRetry.attempts : 0; + const plan = planMidReplyTimeoutRetry({ + error: err, + requestElapsedMs: context.requestElapsedMs, + stepAttempts, + }); + if (plan && turn.awaitingRecovery) { + // An earlier outage wait on this step is over: the provider has + // just been answering for a minute or more. + deps.onEvent?.({ type: "provider_recovered", waitedMs: turn.outageWaitedMs }); + turn.awaitingRecovery = false; + turn.outageWaitedMs = 0; + turn.outageAttempts = 0; + } + if (plan?.kind === "give_up") { + return reportTurnFailure( + plan.error, + { category, cancelled: false, creditRefused: false }, + { state, stepIndex: i }, + deps, + turn, + ); + } + if (plan?.kind === "retry") { + turn.midReplyTimeoutRetry = { stepIndex: i, attempts: plan.attempts }; + turn.pendingNotice = composeMidReplyTimeoutNotice(noticeForThisStep); + const waitedOn = readFailingLink(err); + // Its own event, not `provider_waiting`: the provider is answering, + // so nothing here may read as an outage. + deps.onEvent?.({ + type: "reply_time_limit_retried", + stepIndex: i, + attempt: plan.attempts, + budget: MID_REPLY_TIMEOUT_STEP_BUDGET, + reason: runError.message, + }); + deps.logger?.warn("provider time limit ended a long reply; retrying the step with a notice", { + sessionId: state.id, + stepIndex: i, + attempt: plan.attempts, + requestElapsedMs: context.requestElapsedMs, + error: runError.message, + ...(waitedOn !== undefined ? { providerId: waitedOn } : {}), + }); + runError = null; + return { kind: "retry_same" }; + } + } // The provider is not answering. Park the turn instead of // killing it: nothing of this step has been committed (a // completion failure throws before any tool is dispatched — diff --git a/src/agent/turn/turn-state.ts b/src/agent/turn/turn-state.ts index 33ba816c2..e84eecb3b 100644 --- a/src/agent/turn/turn-state.ts +++ b/src/agent/turn/turn-state.ts @@ -68,6 +68,8 @@ export interface TurnLoopState { * a step that completes without a cut resets it. */ truncationRetries: number; + /** Retries of one step after a provider's time limit ended its reply. */ + midReplyTimeoutRetry: { stepIndex: number; attempts: number } | null; /** The step index already retried after a request-size refusal. */ sizeRepackRetry: { stepIndex: number } | null; /** The loop's own estimate of the last prompt built, for the repack fallback. */ @@ -119,6 +121,7 @@ export function createTurnLoopState(): TurnLoopState { oomRestarts: 0, truncationRetry: null, truncationRetries: 0, + midReplyTimeoutRetry: null, sizeRepackRetry: null, lastPromptTokens: 0, lastBoundaryIndex: -1, diff --git a/src/cli/trace-formatter.ts b/src/cli/trace-formatter.ts index 9c184012f..b6f2ebb25 100644 --- a/src/cli/trace-formatter.ts +++ b/src/cli/trace-formatter.ts @@ -97,6 +97,7 @@ function formatTraceEvent(event: TraceEvent, raw: boolean): string { event.elapsedMs / 1000, )}s`; case "parse_failure_recovered": + case "reply_time_limit_retried": return `${head} step=${event.stepIndex} attempt=${event.attempt}/${event.budget} reason=${truncate(event.reason, 120, raw)}`; case "empty_completion_recovered": return `${head} step=${event.stepIndex} attempt=${event.attempt}/${event.budget}`; diff --git a/src/llm/reliability/mid-reply-timeout.test.ts b/src/llm/reliability/mid-reply-timeout.test.ts new file mode 100644 index 000000000..8344b0e3c --- /dev/null +++ b/src/llm/reliability/mid-reply-timeout.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { OpenAiHttpError } from "../provider/openai/openai-http.js"; +import { TransportError } from "./llm-failures.js"; +import { readMidReplyTimeout } from "./mid-reply-timeout.js"; + +const URL = "https://vendor/v1/chat/completions"; +const inStream = (status: number | null, streamError: string) => + new OpenAiHttpError(`openai provider ${status}: ${streamError}`, status, URL, false, null, "openrouter", undefined, { streamError }); + +describe("readMidReplyTimeout", () => { + it.each([504, 408, 524])("reads a %i reported inside the stream", (status) => { + expect(readMidReplyTimeout(inStream(status, "A Timeout Occurred"))).toEqual({ provider: "openrouter", status, url: URL }); + }); + + it("reads a timeout message that carries no status", () => { + expect(readMidReplyTimeout(inStream(null, "Upstream idle timeout"))?.status).toBeNull(); + }); + + it("reads a time-limit status whose message does not say timeout", () => { + expect(readMidReplyTimeout(inStream(504, "Upstream error"))?.status).toBe(504); + }); + + it("reads timeout words on any other status", () => { + expect(readMidReplyTimeout(inStream(502, "A Timeout Occurred"))?.status).toBe(502); + }); + + it("finds it under the step executor's TransportError", () => { + const wrapped = new TransportError("wrapped", 504, URL, { cause: inStream(504, "A Timeout Occurred") }); + expect(readMidReplyTimeout(wrapped)?.provider).toBe("openrouter"); + }); + + it.each([ + ["a dropped connection", inStream(502, "Network connection lost")], + ["a reply ended with an error", inStream(504, "the provider ended the completion with an error")], + ["a 504 before any reply", new OpenAiHttpError("openai provider 504", 504, URL, false, null, "openrouter")], + ["an unrelated failure", new Error("A Timeout Occurred")], + ["a non-error", undefined], + ])("ignores %s", (_name, err) => { + expect(readMidReplyTimeout(err)).toBeNull(); + }); +}); diff --git a/src/llm/reliability/mid-reply-timeout.ts b/src/llm/reliability/mid-reply-timeout.ts new file mode 100644 index 000000000..dd9ea52fd --- /dev/null +++ b/src/llm/reliability/mid-reply-timeout.ts @@ -0,0 +1,45 @@ +import { + isErrorFinishMessage, + OpenAiHttpError, +} from "../provider/openai/openai-http.js"; + +/** Depth cap on the `cause` walk; longer is a cycle. */ +const MAX_CAUSE_DEPTH = 5; + +/** Timeout answers a gateway reports inside a stream: 524 is Cloudflare's. */ +const TIMEOUT_STATUSES = new Set([408, 504, 524]); + +const TIMEOUT_WORDS = /\btime(?:d)?[ -]?out\b/i; + +export interface MidReplyTimeout { + /** The name to show for the provider that timed out. */ + provider: string; + status: number | null; + url: string; +} + +/** + * Did a provider end an open stream by reporting a time limit? Only an + * error reported inside the stream counts: a 504 on the open means no + * reply was started, and a dropped connection says nothing about a limit. + */ +export function readMidReplyTimeout(err: unknown): MidReplyTimeout | null { + let current: unknown = err; + for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth += 1) { + if (typeof current !== "object" || current === null) return null; + if (current instanceof OpenAiHttpError && current.streamError !== undefined) { + if (isErrorFinishMessage(current.streamError)) return null; + const timedOut = + (current.status !== null && TIMEOUT_STATUSES.has(current.status)) || + TIMEOUT_WORDS.test(current.streamError); + if (!timedOut) return null; + return { + provider: current.providerLabel || "The provider", + status: current.status, + url: current.url, + }; + } + current = (current as { cause?: unknown }).cause; + } + return null; +} diff --git a/src/tracing/trace/trace-event.ts b/src/tracing/trace/trace-event.ts index e95e2eac8..1952be892 100644 --- a/src/tracing/trace/trace-event.ts +++ b/src/tracing/trace/trace-event.ts @@ -43,6 +43,7 @@ export type TraceEvent = | TraceCompletionTruncated | TracePromptRepacked | TraceParseFailureRecovered + | TraceReplyTimeLimitRetried | TraceEmptyCompletionRecovered | TraceLessonDeprecated | TraceVoteApplied @@ -347,6 +348,19 @@ export interface TraceParseFailureRecovered extends TraceEventBase { reason: string; } +/** + * A provider time limit ended a long reply and the step was retried with + * a notice asking for a shorter one. + */ +export interface TraceReplyTimeLimitRetried extends TraceEventBase { + type: "reply_time_limit_retried"; + turnIndex: number; + stepIndex: number; + attempt: number; + budget: number; + reason: string; +} + /** * A completion came back with nothing in any channel and the turn spent * another step on it instead of ending. Distinct from diff --git a/src/tracing/trace/trace-recorder.ts b/src/tracing/trace/trace-recorder.ts index cec4bd240..af5cb2008 100644 --- a/src/tracing/trace/trace-recorder.ts +++ b/src/tracing/trace/trace-recorder.ts @@ -619,6 +619,19 @@ export function createTraceRecorder( reason: event.reason, }); return; + case "reply_time_limit_retried": + push({ + type: "reply_time_limit_retried", + seq: nextSeq(), + sessionId, + ts: now(), + turnIndex: currentTurnIndex, + stepIndex: event.stepIndex, + attempt: event.attempt, + budget: event.budget, + reason: event.reason, + }); + return; case "empty_completion_recovered": push({ type: "empty_completion_recovered", diff --git a/src/tui/agent-event-reducer.test.ts b/src/tui/agent-event-reducer.test.ts index f11903a01..6fbf63687 100644 --- a/src/tui/agent-event-reducer.test.ts +++ b/src/tui/agent-event-reducer.test.ts @@ -1671,6 +1671,48 @@ describe("truncated completion", () => { }); }); +describe("provider time limit on a long reply (ATO-1398)", () => { + const retried: TuiAction = { + type: "agent_event", + event: { + type: "reply_time_limit_retried", + stepIndex: 1, + attempt: 1, + budget: 2, + reason: "A Timeout Occurred", + } as never, + }; + + it("says what happened without calling it an outage", () => { + const next = reduceTuiState(createInitialTuiState(fakeSession()), retried); + expect(next.feed.at(-1)?.line).toContain("(1/2)"); + expect(next.providerOutage).toBeNull(); + }); + + it("drops the lost reply so the retry is not spliced onto it", () => { + const streaming = { + ...createInitialTuiState(fakeSession()), + reasoning: [{ id: "r", timestamp: 1, stepIndex: 1, text: "lost thinking" }], + streamingAssistantText: "lost text", + }; + const next = reduceTuiState(streaming, retried); + expect(next.reasoning).toEqual([]); + expect(next.streamingAssistantText).toBeNull(); + }); + + it("does not report a wait when the turn gives up", () => { + const next = reduceTuiState( + reduceTuiState(createInitialTuiState(fakeSession()), retried), + { + type: "agent_event", + event: { type: "loop_failed", error: new Error("stopped on a time limit 3 times"), category: "transport" }, + } as TuiAction, + ); + expect(next.messages.at(-1)?.text).not.toContain("Stopped waiting"); + expect(next.providerOutage).toBeNull(); + }); +}); + describe("parse-failure recovery", () => { const recovered = (over: Record = {}): TuiAction => ({ type: "agent_event", diff --git a/src/tui/agent-event-reducer.ts b/src/tui/agent-event-reducer.ts index 24f160f9e..02b886c10 100644 --- a/src/tui/agent-event-reducer.ts +++ b/src/tui/agent-event-reducer.ts @@ -52,6 +52,7 @@ import { upsertReasoning, } from "./reducer-helpers.js"; import { + discardDeadAttempt, parkProviderOutage, retryProviderOutage, } from "./provider-outage-state.js"; @@ -1039,6 +1040,15 @@ function reduceAgentEvent(state: TuiState, event: AgentLoopEvent): TuiState { color: "yellow", }); } + case "reply_time_limit_retried": + // A feed line and no outage state: the provider is answering. The + // retry streams from the start, so the lost reply is dropped here. + return appendFeed(discardDeadAttempt(state, event.stepIndex), { + kind: "runtime_info", + stepIndex: event.stepIndex, + line: `» the provider's time limit ended this reply and none of it was kept. Asking for a shorter one (${event.attempt}/${event.budget})`, + color: "yellow", + }); case "empty_completion_recovered": // Same reason as above, and more so: an empty completion produces // literally nothing, so without this line the feed shows a step diff --git a/src/tui/issue-report/trace-redaction.ts b/src/tui/issue-report/trace-redaction.ts index f28b5d7ad..837d1aed3 100644 --- a/src/tui/issue-report/trace-redaction.ts +++ b/src/tui/issue-report/trace-redaction.ts @@ -134,6 +134,7 @@ const ERROR_LEVEL_EVENTS: ReadonlySet = new Set([ "loop_detected", "provider_waiting", "provider_recovered", + "reply_time_limit_retried", "trace_truncated", "turn_finished", "step_finished", diff --git a/src/tui/provider-outage-state.ts b/src/tui/provider-outage-state.ts index 6e3793b69..7e6c91f2e 100644 --- a/src/tui/provider-outage-state.ts +++ b/src/tui/provider-outage-state.ts @@ -17,6 +17,20 @@ import type { TuiState } from "./tui-state.js"; type ProviderOutage = NonNullable; +/** Drop what a failed attempt left streaming for a step about to be replayed. */ +export function discardDeadAttempt(state: TuiState, stepIndex: number): TuiState { + const last = state.reasoning.at(-1); + return { + ...state, + reasoning: + last && last.stepIndex === stepIndex + ? state.reasoning.slice(0, -1) + : state.reasoning, + streamingAssistantText: null, + streamingToolCalls: [], + }; +} + /** Park the turn: the backoff sleep has begun. */ export function parkProviderOutage( state: TuiState, @@ -65,16 +79,8 @@ export function retryProviderOutage( ): TuiState { const outage = state.providerOutage; if (!outage || outage.givenUp) return state; - const last = state.reasoning.at(-1); - const reasoning = - last && last.stepIndex === stepIndex - ? state.reasoning.slice(0, -1) - : state.reasoning; return { - ...state, + ...discardDeadAttempt(state, stepIndex), providerOutage: { ...outage, phase: "retrying", sinceTs: now }, - reasoning, - streamingAssistantText: null, - streamingToolCalls: [], }; } From 5147e3d3105d2166bcb90440988eb03c53cd090f Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:53:12 +0300 Subject: [PATCH 003/155] fix(prompt): use the session's working dir in every model mode The stable prefix took working_dir from the bootstrap capabilities and replaced it with session.workingDir only when a workspace was prepared, which happens in cloud mode alone. After a live folder switch in the desktop app, a local-mode provider kept seeing the old folder. Always render session.workingDir. Local mode still prepares no cloud workspace, and the prefix changes only when the folder changes. Fixes #1121 --- src/agent/step/step-inference.test.ts | 24 ++++++++++++++++++++++++ src/prompt/build-prompt.ts | 7 ++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/agent/step/step-inference.test.ts b/src/agent/step/step-inference.test.ts index 0e2c1f0c4..cc6ba6861 100644 --- a/src/agent/step/step-inference.test.ts +++ b/src/agent/step/step-inference.test.ts @@ -255,3 +255,27 @@ describe("compact prompt decision (ATO-1116)", () => { }); }); +describe("working_dir after a live folder switch (issue #1121)", () => { + it("local mode renders the session's folder, not the bootstrap one, without preparing a workspace", () => { + const { ctx, deps } = fixture(); + let prepared = 0; + const localDeps: StepDependencies = { + ...deps, + modelMode: { mode: "local", source: "provider", providerId: "primary", modelId: null }, + prepareWorkspace: () => { prepared += 1; throw new Error("local mode must not prepare a workspace"); }, + }; + const before = prepareStepInference(ctx, localDeps); + expect(before.prompt.stablePrefix).toContain("working_dir: /fixture"); + + // The desktop folder picker moves the session; bootstrap capabilities still say /fixture. + const switched: StepContext = { ...ctx, workspace: undefined, session: { ...ctx.session, workingDir: "/switched" } }; + const after = prepareStepInference(switched, localDeps); + expect(after.prompt.stablePrefix).toContain("working_dir: /switched"); + expect(after.prompt.stablePrefix).not.toContain("working_dir: /fixture"); + expect(prepared).toBe(0); + + // Same folder again: the stable prefix is byte-identical, so the cache survives. + const again = prepareStepInference(switched, localDeps); + expect(again.prompt.stablePrefix).toBe(after.prompt.stablePrefix); + }); +}); diff --git a/src/prompt/build-prompt.ts b/src/prompt/build-prompt.ts index 8d6f1f6a5..9a53dfa15 100644 --- a/src/prompt/build-prompt.ts +++ b/src/prompt/build-prompt.ts @@ -203,7 +203,12 @@ export function buildPrompt(input: BuildPromptInput): BuiltPrompt { const effectiveCatalog = workspaceCatalog ?? upstreamCatalog; const stablePrefix = buildStablePrefix({ toolDescriptors: input.toolDescriptors, - capabilities: input.workspace ? { ...input.capabilities, workingDir: input.session.workingDir } : input.capabilities, + // The session's folder, in every model mode: bootstrap capabilities + // carry the folder the runtime started in, and a live folder switch + // only moves `session.workingDir`. Local mode prepares no workspace, + // so gating on it left the old folder in the prompt (issue #1121). + // The prefix still moves only when the folder does. + capabilities: { ...input.capabilities, workingDir: input.session.workingDir }, skillCatalog: effectiveCatalog?.entries ?? input.skillCatalog, ...(effectiveCatalog ? { skillCatalogDropped: effectiveCatalog.dropped } : input.skillCatalogDropped !== undefined ? { skillCatalogDropped: input.skillCatalogDropped } From f22a2050f955fb12ac939fa13ff101e5288fda9e Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:57:25 +0300 Subject: [PATCH 004/155] fix(llm): stop sending temperature to Claude models that dropped it Claude Sonnet 5.5, Haiku 5.5 and the other new Claude models answer a request that carries `temperature` with a 400, and every turn sent 0.2. - model-params: Claude Opus 4.7+, Sonnet 5+, Haiku 5+ and every Fable and Mythos model get the `forbidden` temperature rule, read from the family and version in the id (direct, router-prefixed and Bedrock spellings). Older Claude models keep the free rule. - allowed-temperature: read a 400 that refuses the field itself ("`temperature` is deprecated for this model", "unsupported parameter") next to the existing "only N is allowed" lesson. - openai-provider: the one resend on a temperature lesson now also covers a refused field: the model is remembered as taking no temperature and the resend goes out without one, so an id the static rule does not know recovers after one 400. - A learned refusal also removes a temperature set through extraBody or model params, in turns and image descriptions. Linear: ATO-1397 --- .../openai/allowed-temperature.test.ts | 44 ++++++++++ .../provider/openai/allowed-temperature.ts | 56 +++++++++++++ src/llm/provider/openai/model-params.test.ts | 58 +++++++++++++ src/llm/provider/openai/model-params.ts | 84 +++++++++++++++++-- .../openai-build-body-temperature.test.ts | 37 ++++++++ src/llm/provider/openai/openai-build-body.ts | 16 +++- .../provider/openai/openai-describe-image.ts | 8 +- src/llm/provider/openai/openai-provider.ts | 53 +++++++----- .../openai/openai-temperature-retry.test.ts | 56 +++++++++++++ 9 files changed, 379 insertions(+), 33 deletions(-) diff --git a/src/llm/provider/openai/allowed-temperature.test.ts b/src/llm/provider/openai/allowed-temperature.test.ts index 3e5b5f51b..ceac0ed72 100644 --- a/src/llm/provider/openai/allowed-temperature.test.ts +++ b/src/llm/provider/openai/allowed-temperature.test.ts @@ -3,6 +3,8 @@ import { describe, expect, it } from "vitest"; import { allowedTemperatureFromError, parseAllowedTemperature, + parseTemperatureRefused, + temperatureLessonFromError, } from "./allowed-temperature.js"; import { OpenAiHttpError } from "./openai-http.js"; @@ -40,3 +42,45 @@ describe("allowedTemperatureFromError", () => { expect(allowedTemperatureFromError("only 1 is allowed")).toBeUndefined(); }); }); + +const ANTHROPIC_DEPRECATED_400 = + '{"type":"error","error":{"type":"invalid_request_error","message":"`temperature` is deprecated for this model."}}'; + +describe("parseTemperatureRefused (ATO-1397)", () => { + it("reads a refusal of the field itself", () => { + expect(parseTemperatureRefused(ANTHROPIC_DEPRECATED_400)).toBe(true); + expect( + parseTemperatureRefused( + "Unsupported parameter: 'temperature' is not supported with this model.", + ), + ).toBe(true); + expect(parseTemperatureRefused("unknown parameter: temperature")).toBe(true); + }); + + it("leaves a named value, a range error and other fields alone", () => { + expect(parseTemperatureRefused(MOONSHOT_400)).toBe(false); + expect( + parseTemperatureRefused("temperature: Input should be less than or equal to 1"), + ).toBe(false); + expect(parseTemperatureRefused("top_k is deprecated for this model")).toBe(false); + expect(parseTemperatureRefused("invalid max_tokens")).toBe(false); + }); +}); + +describe("temperatureLessonFromError", () => { + const err = (status: number, body: string) => + new OpenAiHttpError(`openai provider ${status}: ${body}`, status, "u"); + + it("names the one value, or says the field is refused, only on a 400", () => { + expect(temperatureLessonFromError(err(400, MOONSHOT_400))).toEqual({ + kind: "fixed", + value: 1, + }); + expect(temperatureLessonFromError(err(400, ANTHROPIC_DEPRECATED_400))).toEqual({ + kind: "forbidden", + }); + expect(temperatureLessonFromError(err(500, ANTHROPIC_DEPRECATED_400))).toBeUndefined(); + expect(temperatureLessonFromError(err(400, "invalid max_tokens"))).toBeUndefined(); + expect(temperatureLessonFromError("temperature is deprecated")).toBeUndefined(); + }); +}); diff --git a/src/llm/provider/openai/allowed-temperature.ts b/src/llm/provider/openai/allowed-temperature.ts index dfafd6920..5899e7e95 100644 --- a/src/llm/provider/openai/allowed-temperature.ts +++ b/src/llm/provider/openai/allowed-temperature.ts @@ -11,6 +11,11 @@ * Deliberately narrow: only a 400, only the "only N is allowed" phrasing, * and only when "temperature" is said close to it. Any other 400 is left * alone and fails the way it always did. + * + * The same file reads the other temperature lesson a 400 can carry: that + * the model takes no temperature at all (`parseTemperatureRefused`). + * Anthropic's newer Claude models answer the field as deprecated; the + * static rule covers the ids we know, this covers the next one. */ const ONLY_ALLOWED_RE = /only\s+([0-9]+(?:\.[0-9]+)?)\s+is\s+allowed/i; @@ -37,3 +42,54 @@ export function allowedTemperatureFromError(err: unknown): number | undefined { if ((err as { status?: unknown }).status !== 400) return undefined; return parseAllowedTemperature(err.message); } + +/** + * Words a 400 uses to refuse a field outright, as opposed to refusing its + * value: Anthropic's newer Claude models call `temperature` deprecated + * (ATO-1397), OpenAI's reasoning models call it an unsupported parameter, + * strict validators call it unknown or not permitted. + */ +const FIELD_REFUSED_RE = + /\b(?:deprecated|no longer supported|not supported|unsupported|not allowed|not permitted|unknown (?:parameter|field)|unrecognized|removed)\b/gi; + +/** + * How far from the refusal word "temperature" may be. Tighter than for + * the named value: validators that echo the request body back would + * otherwise put "temperature" near a refusal of some other field. + */ +const REFUSAL_NEAR_CHARS = 80; + +/** + * Whether a 400 body refuses `temperature` as a field. A body that names + * the one allowed value is a different lesson (`parseAllowedTemperature`) + * and reads as `false` here; so does a refusal that says nothing about + * temperature near it, or a range error ("must be at most 1"). + */ +export function parseTemperatureRefused(text: string): boolean { + if (parseAllowedTemperature(text) !== undefined) return false; + for (const match of text.matchAll(FIELD_REFUSED_RE)) { + const at = match.index ?? 0; + const from = Math.max(0, at - REFUSAL_NEAR_CHARS); + const to = at + match[0].length + REFUSAL_NEAR_CHARS; + if (/temperature/i.test(text.slice(from, to))) return true; + } + return false; +} + +/** + * What a failed send teaches about the model's temperature: the one value + * it takes, that it takes none, or nothing (any other error). + */ +export type TemperatureLesson = + | { readonly kind: "fixed"; readonly value: number } + | { readonly kind: "forbidden" }; + +export function temperatureLessonFromError( + err: unknown, +): TemperatureLesson | undefined { + const value = allowedTemperatureFromError(err); + if (value !== undefined) return { kind: "fixed", value }; + if (!(err instanceof Error)) return undefined; + if ((err as { status?: unknown }).status !== 400) return undefined; + return parseTemperatureRefused(err.message) ? { kind: "forbidden" } : undefined; +} diff --git a/src/llm/provider/openai/model-params.test.ts b/src/llm/provider/openai/model-params.test.ts index 2a7e336ca..8a4b25de0 100644 --- a/src/llm/provider/openai/model-params.test.ts +++ b/src/llm/provider/openai/model-params.test.ts @@ -4,6 +4,7 @@ import { forgetLearnedModelParams, modelParamProfile, rememberAllowedTemperature, + rememberTemperatureForbidden, resolveTemperature, } from "./model-params.js"; @@ -67,6 +68,60 @@ describe("modelParamProfile", () => { }); }); +describe("modelParamProfile — Claude models without sampling parameters (ATO-1397)", () => { + it.each([ + "claude-sonnet-5-5", + "claude-haiku-5-5", + "claude-opus-5-5", + "claude-opus-5", + "claude-sonnet-5", + "claude-fable-5", + "claude-fable-5-1", + "claude-mythos-5-1", + "claude-opus-4-8", + "claude-opus-4-7", + "anthropic/claude-sonnet-5", + "anthropic/claude-opus-4.8", + "anthropic/claude-opus-5-fast", + "anthropic/claude-sonnet-5.5", + "anthropic.claude-haiku-5-5", + "claude-sonnet-5-20260101", + "Claude-Sonnet-5-5", + // An unknown future version falls on the side of the line that omits. + "claude-sonnet-6", + ])("forbids temperature but keeps max_tokens for %s", (model) => { + expect(modelParamProfile(model)).toEqual({ + temperature: { kind: "forbidden" }, + capField: "max_tokens", + }); + }); + + it.each([ + "claude-opus-4-6", + "claude-opus-4-1-20250805", + "claude-opus-4", + "claude-sonnet-4-6", + "claude-sonnet-4-5-20250929", + "claude-haiku-4-5", + "anthropic/claude-haiku-4.5", + "claude-3-5-sonnet-20241022", + ])("leaves temperature free for the older %s", (model) => { + expect(modelParamProfile(model).temperature).toEqual({ kind: "free" }); + }); + + it("learns at run time that a model takes no temperature, for that id only", () => { + expect(rememberTemperatureForbidden("vendor/new-model")).toBe(true); + expect(rememberTemperatureForbidden("vendor/new-model")).toBe(false); + expect(modelParamProfile("vendor/new-model")).toEqual({ + temperature: { kind: "forbidden", learned: true }, + capField: "max_tokens", + }); + expect(modelParamProfile("vendor/other-model").temperature).toEqual({ + kind: "free", + }); + }); +}); + describe("resolveTemperature", () => { it("sends the caller's value, else the fallback, when free", () => { const profile = modelParamProfile("gpt-4o"); @@ -77,6 +132,9 @@ describe("resolveTemperature", () => { it("sends nothing when forbidden", () => { expect(resolveTemperature(modelParamProfile("gpt-5"), 0.7, 0.2)).toBeUndefined(); + expect( + resolveTemperature(modelParamProfile("claude-sonnet-5-5"), 0.7, 0.2), + ).toBeUndefined(); }); it("sends the one allowed value when fixed, whatever was asked", () => { diff --git a/src/llm/provider/openai/model-params.ts b/src/llm/provider/openai/model-params.ts index 41414478b..2ffb04cd1 100644 --- a/src/llm/provider/openai/model-params.ts +++ b/src/llm/provider/openai/model-params.ts @@ -12,8 +12,14 @@ * - Moonshot's Kimi K3.1 family accepts exactly one temperature, 1, and * answers anything else with a 400 ("only 1 is allowed for this * model"). Kimi K2.x takes any value and keeps the historical body. + * - Anthropic's newer Claude models (Opus 4.7 and later, Sonnet 5 and + * later, Haiku 5 and later, every Fable and Mythos) no longer take + * sampling parameters and answer `temperature` with a 400 (ATO-1397). + * Older Claude models (Opus 4.6, Sonnet 4.6, Haiku 4.5, …) keep it. * - A model can also be taught its one temperature at run time, from - * the 400 that named it (`rememberAllowedTemperature`). + * the 400 that named it (`rememberAllowedTemperature`), or that it + * takes none at all, from a 400 that refused the field + * (`rememberTemperatureForbidden`). * - Everything else keeps the historical body byte for byte. * * The id is read after any vendor prefix (`openai/o3` on OpenRouter is @@ -26,14 +32,15 @@ * What a model accepts as `temperature`: * - `free`: any value; the caller's, else its default. * - `forbidden`: the field must not be sent at all. - * - `fixed`: only this value. `learned` marks one taught by a 400 at - * run time rather than known up front; that one also beats an - * operator's `extraBody` / `params`, since the service has just said - * the operator's value is refused. + * - `fixed`: only this value. + * + * `learned` marks a rule taught by a 400 at run time rather than known up + * front; that one also beats an operator's `extraBody` / `params`, since + * the service has just said the operator's value is refused. */ export type TemperatureRule = | { readonly kind: "free" } - | { readonly kind: "forbidden" } + | { readonly kind: "forbidden"; readonly learned?: true } | { readonly kind: "fixed"; readonly value: number; readonly learned?: true }; export interface ModelParamProfile { @@ -55,10 +62,50 @@ const OPENAI_REASONING_MODEL_RE = /^(?:o[1-9]|gpt-5)(?![a-z])/i; */ const KIMI_K31_MODEL_RE = /^kimi-k3\.1(?![0-9])/i; +/** + * A Claude model id in Anthropic's naming: family, major version and an + * optional minor version (`claude-sonnet-5-5`, `claude-opus-4.8`, + * `anthropic/claude-fable-5`, Bedrock's `anthropic.claude-haiku-5-5`). The + * minor is one or two digits so a date suffix + * (`claude-sonnet-5-20260101`) is not read as one. The old naming + * (`claude-3-5-sonnet`) does not match and keeps the free rule. + */ +const CLAUDE_MODEL_RE = + /(?:^|[^a-z0-9])claude-(opus|sonnet|haiku|fable|mythos)-(\d+)(?:[-.](\d{1,2}))?(?![0-9])/i; + +/** + * Whether a Claude model rejects `temperature`. Anthropic removed the + * sampling parameters from Opus 4.7 on, from the Sonnet and Haiku 5 + * generations on, and from every Fable and Mythos model; the 5.5 models + * answer a non-default value with a 400. Unknown future versions fall on + * the "rejects" side of the line, and the run-time lesson covers a model + * this rule gets wrong. + */ +function claudeRejectsTemperature(bare: string): boolean { + const match = CLAUDE_MODEL_RE.exec(bare); + if (!match) return false; + const family = match[1]!.toLowerCase(); + const major = Number(match[2]); + const minor = match[3] === undefined ? 0 : Number(match[3]); + switch (family) { + case "fable": + case "mythos": + return true; + case "opus": + return major > 4 || (major === 4 && minor >= 7); + default: + // sonnet, haiku + return major >= 5; + } +} + const FREE: TemperatureRule = { kind: "free" }; -/** Temperatures services told us at run time, keyed by model id. */ -const learnedTemperatures = new Map(); +/** + * Temperatures services told us at run time, keyed by model id: the one + * value a model accepts, or `forbidden` when it refused the field. + */ +const learnedTemperatures = new Map(); export function modelParamProfile(modelId: string): ModelParamProfile { const bare = modelId.includes("/") @@ -71,6 +118,12 @@ export function modelParamProfile(modelId: string): ModelParamProfile { }; } const learned = learnedTemperatures.get(modelId.trim()); + if (learned === "forbidden") { + return { + temperature: { kind: "forbidden", learned: true }, + capField: "max_tokens", + }; + } if (learned !== undefined) { return { temperature: { kind: "fixed", value: learned, learned: true }, @@ -80,6 +133,9 @@ export function modelParamProfile(modelId: string): ModelParamProfile { if (KIMI_K31_MODEL_RE.test(bare)) { return { temperature: { kind: "fixed", value: 1 }, capField: "max_tokens" }; } + if (claudeRejectsTemperature(bare)) { + return { temperature: { kind: "forbidden" }, capField: "max_tokens" }; + } return { temperature: FREE, capField: "max_tokens" }; } @@ -118,6 +174,18 @@ export function rememberAllowedTemperature( return true; } +/** + * Remember, for the rest of the process, that a service refused + * `temperature` for this model altogether. Returns whether anything + * changed, like `rememberAllowedTemperature`. + */ +export function rememberTemperatureForbidden(modelId: string): boolean { + const key = modelId.trim(); + if (learnedTemperatures.get(key) === "forbidden") return false; + learnedTemperatures.set(key, "forbidden"); + return true; +} + /** Test hook: forget everything learned at run time. */ export function forgetLearnedModelParams(): void { learnedTemperatures.clear(); diff --git a/src/llm/provider/openai/openai-build-body-temperature.test.ts b/src/llm/provider/openai/openai-build-body-temperature.test.ts index 3dbf7b2c4..f1786ebe1 100644 --- a/src/llm/provider/openai/openai-build-body-temperature.test.ts +++ b/src/llm/provider/openai/openai-build-body-temperature.test.ts @@ -4,6 +4,7 @@ import { buildOpenAiChatBody } from "./openai-build-body.js"; import { forgetLearnedModelParams, rememberAllowedTemperature, + rememberTemperatureForbidden, } from "./model-params.js"; afterEach(() => { @@ -55,3 +56,39 @@ describe("buildOpenAiChatBody — temperature per model (ATO-1214)", () => { expect(body.temperature).toBe(1); }); }); + +describe("buildOpenAiChatBody — Claude models without temperature (ATO-1397)", () => { + it.each(["claude-sonnet-5-5", "claude-haiku-5-5", "claude-opus-5-5", "claude-fable-5-1"])( + "sends no temperature for %s, whatever the caller asked, and keeps max_tokens", + (model) => { + const body = buildOpenAiChatBody( + { prompt: "hi", temperature: 0.7, maxTokens: 512 }, + model, + false, + ); + expect(body).not.toHaveProperty("temperature"); + expect(body.max_tokens).toBe(512); + }, + ); + + it("keeps the historical 0.2 default for claude-sonnet-4-6", () => { + expect( + buildOpenAiChatBody({ prompt: "hi" }, "claude-sonnet-4-6", false).temperature, + ).toBe(0.2); + }); + + it("drops a temperature from extraBody and model params once the service refused it", () => { + rememberTemperatureForbidden("vendor/strict-model"); + const body = buildOpenAiChatBody( + { prompt: "hi", temperature: 0.3 }, + "vendor/strict-model", + false, + { temperature: 0.5 }, + undefined, + undefined, + undefined, + { modelParams: { temperature: 0.4 } }, + ); + expect(body).not.toHaveProperty("temperature"); + }); +}); diff --git a/src/llm/provider/openai/openai-build-body.ts b/src/llm/provider/openai/openai-build-body.ts index 39b4404fd..7eb00906c 100644 --- a/src/llm/provider/openai/openai-build-body.ts +++ b/src/llm/provider/openai/openai-build-body.ts @@ -116,8 +116,9 @@ export function buildOpenAiChatBody( ]; // OpenAI's reasoning models reject the field outright (`Unsupported // parameter: 'temperature'`), so for them it is not sent at all — not - // even a caller's own value. Kimi K3.1 accepts only 1, so it gets 1 - // whatever the caller asked. See `model-params.ts`. + // even a caller's own value; nor is it for the Claude models that + // dropped sampling parameters (ATO-1397). Kimi K3.1 accepts only 1, so + // it gets 1 whatever the caller asked. See `model-params.ts`. const temperature = resolveTemperature(profile, filtered.temperature, 0.2); const body: Record = { model: defaultChatModel, @@ -255,10 +256,17 @@ export function buildOpenAiChatBody( else delete merged[key]; } // A temperature the service itself named in a 400 beats the operator's - // passthrough: that value is exactly what was just refused. A rule - // known up front (Kimi K3.1) stays overridable like any other field. + // passthrough: that value is exactly what was just refused. So does a + // 400 that refused the field outright: it goes, whoever set it. A rule + // known up front (Kimi K3.1, Claude 5.x) stays overridable like any + // other field. if (profile.temperature.kind === "fixed" && profile.temperature.learned) { merged.temperature = profile.temperature.value; + } else if ( + profile.temperature.kind === "forbidden" && + profile.temperature.learned + ) { + delete merged.temperature; } return checkedBody(merged, request); } diff --git a/src/llm/provider/openai/openai-describe-image.ts b/src/llm/provider/openai/openai-describe-image.ts index 2855058cb..be5e095a7 100644 --- a/src/llm/provider/openai/openai-describe-image.ts +++ b/src/llm/provider/openai/openai-describe-image.ts @@ -38,7 +38,8 @@ export async function describeImageViaOpenAi( userContent.push({ type: "text", text: request.prompt }); // Same per-model rules as a turn (`model-params.ts`): OpenAI's // reasoning models get no temperature and `max_completion_tokens`, - // Kimi K3.1 gets its one allowed temperature. + // the newer Claude models get no temperature, Kimi K3.1 gets its one + // allowed temperature. const profile = modelParamProfile(defaultChatModel); const temperature = resolveTemperature(profile, request.temperature, 0.1); const base: Record = { @@ -99,6 +100,11 @@ function withPassthrough( // `buildOpenAiChatBody`. if (profile.temperature.kind === "fixed" && profile.temperature.learned) { merged.temperature = profile.temperature.value; + } else if ( + profile.temperature.kind === "forbidden" && + profile.temperature.learned + ) { + delete merged.temperature; } return merged; } diff --git a/src/llm/provider/openai/openai-provider.ts b/src/llm/provider/openai/openai-provider.ts index ee9b3bc1e..d6c22ba99 100644 --- a/src/llm/provider/openai/openai-provider.ts +++ b/src/llm/provider/openai/openai-provider.ts @@ -30,8 +30,11 @@ import { type OpenAiBodyOptions, } from "./openai-build-body.js"; import { isNativeShapeRejection } from "./openai-native-messages.js"; -import { allowedTemperatureFromError } from "./allowed-temperature.js"; -import { rememberAllowedTemperature } from "./model-params.js"; +import { temperatureLessonFromError } from "./allowed-temperature.js"; +import { + rememberAllowedTemperature, + rememberTemperatureForbidden, +} from "./model-params.js"; import { OPENAI_DEFAULT_REQUEST_TIMEOUT_MS } from "../../transport-deadlines.js"; import { buildOpenAiHeaders, @@ -306,8 +309,8 @@ export class OpenAiProvider implements LlmProvider { ), ); // One resend per completion when the service names the only - // temperature it takes (`allowed-temperature.ts`); the rebuilt body - // carries the lesson. + // temperature it takes, or refuses the field altogether + // (`allowed-temperature.ts`); the rebuilt body carries the lesson. let temperatureRetried = false; const sendOnce = async ( shape: "native" | "flat", @@ -315,7 +318,7 @@ export class OpenAiProvider implements LlmProvider { try { return await send(shape); } catch (err) { - if (temperatureRetried || !this.learnAllowedTemperature(request, err)) { + if (temperatureRetried || !this.learnTemperature(request, err)) { throw err; } temperatureRetried = true; @@ -397,27 +400,37 @@ export class OpenAiProvider implements LlmProvider { } /** - * Whether a failed send is a 400 naming the one temperature this model - * accepts (Moonshot's "only 1 is allowed"). The value is remembered for - * the model for the rest of the process, so the next body built — the - * resend, and every later turn — carries it. The caller allows one - * such resend per completion. + * Whether a failed send is a 400 that teaches this model's temperature: + * the one value it accepts (Moonshot's "only 1 is allowed"), or that it + * takes none (Anthropic's "temperature is deprecated for this model", + * ATO-1397). The lesson is remembered for the model for the rest of the + * process, so the next body built — the resend, and every later turn — + * carries it. The caller allows one such resend per completion. */ - private learnAllowedTemperature( + private learnTemperature( request: CompletionRequest, err: unknown, ): boolean { if (request.signal?.aborted) return false; - const value = allowedTemperatureFromError(err); - if (value === undefined) return false; - if (!rememberAllowedTemperature(this.defaultChatModel!, value)) return true; + const lesson = temperatureLessonFromError(err); + if (lesson === undefined) return false; + const model = this.defaultChatModel!; + if (lesson.kind === "forbidden") { + if (!rememberTemperatureForbidden(model)) return true; + this.http.logger?.warn( + `llm: "${this.id}" refused temperature for this model; resending once without it and keeping that for this model`, + { provider: this.id, model, status: 400, temperature: null }, + ); + return true; + } + if (!rememberAllowedTemperature(model, lesson.value)) return true; this.http.logger?.warn( - `llm: "${this.id}" accepts only temperature ${value} for this model; resending once and keeping it for this model`, + `llm: "${this.id}" accepts only temperature ${lesson.value} for this model; resending once and keeping it for this model`, { provider: this.id, - model: this.defaultChatModel, + model, status: 400, - temperature: value, + temperature: lesson.value, }, ); return true; @@ -596,14 +609,14 @@ export class OpenAiProvider implements LlmProvider { // stopped. `signal.reason` is abort-shaped by construction. if (request.signal?.aborted) throw cancellationError(request.signal, err); - // The service named the one temperature it takes: resend once - // with it. Same window as the flat resend below — a 400 comes + // The service named the one temperature it takes, or refused the + // field: resend once with the lesson. Same window as the flat resend below — a 400 comes // from the open, before any output — and only once per // completion, so a second rejection falls through. if ( !committed && !temperatureRetried && - this.learnAllowedTemperature(request, err) + this.learnTemperature(request, err) ) { temperatureRetried = true; body = this.buildBody(request, true, shape); diff --git a/src/llm/provider/openai/openai-temperature-retry.test.ts b/src/llm/provider/openai/openai-temperature-retry.test.ts index c23351d65..633a1760a 100644 --- a/src/llm/provider/openai/openai-temperature-retry.test.ts +++ b/src/llm/provider/openai/openai-temperature-retry.test.ts @@ -136,3 +136,59 @@ describe("OpenAiProvider — named-temperature retry (ATO-1217)", () => { expect(fetchImpl).toHaveBeenCalledTimes(2); }); }); + +const DEPRECATED_400 = JSON.stringify({ + type: "error", + error: { + type: "invalid_request_error", + message: "`temperature` is deprecated for this model.", + }, +}); + +describe("OpenAiProvider — refused-temperature retry (ATO-1397)", () => { + it("resends a unary completion once without temperature", async () => { + const { fetchImpl, bodies } = scripted([ + () => json(DEPRECATED_400, 400), + okUnary, + ]); + const result = await provider(fetchImpl).complete({ prompt: "hi" }); + + expect(result.content).toBe("ok"); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(bodies[0]!.temperature).toBe(0.2); + expect(bodies[1]).not.toHaveProperty("temperature"); + }); + + it("resends a streamed completion once and keeps omitting it for the model", async () => { + const { fetchImpl, bodies } = scripted([ + () => json(DEPRECATED_400, 400), + okStream, + okStream, + ]); + const p = provider(fetchImpl); + const result = await drain(p.completeStream({ prompt: "hi" })); + + expect(result.content).toBe("ok"); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(bodies[1]).not.toHaveProperty("temperature"); + + await drain(p.completeStream({ prompt: "again" })); + expect(fetchImpl).toHaveBeenCalledTimes(3); + expect(bodies[2]).not.toHaveProperty("temperature"); + }); + + it("sends no temperature to claude-sonnet-5-5 in the first place", async () => { + const { fetchImpl, bodies } = scripted([okUnary]); + const p = new OpenAiProvider({ + id: "anthropic", + baseUrl: "https://api.anthropic.com", + apiKey: "", + defaultChatModel: "claude-sonnet-5-5", + fetchImpl: fetchImpl as typeof fetch, + }); + await p.complete({ prompt: "hi", temperature: 0.7 }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(bodies[0]).not.toHaveProperty("temperature"); + }); +}); From bcca1a37d929fc508fcc8630d1aba78f56e206d0 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:00:32 +0300 Subject: [PATCH 005/155] fix(llm): hand DeepSeek its tool calls' reasoning_content back DeepSeek's thinking mode answers the next request of a tool loop with a 400 unless the earlier assistant tool-call messages carry their `reasoning_content`. Cloud bodies never carried it: the cloud context journal drops reasoning, and the ATO-887 hand-back was limited to local template servers. - build-cloud-prompt: the current, un-replied macro-turn's call reasoning is put back on the outgoing turns from the transcript. The persisted journal and the text form are unchanged, and a provider that does not hand reasoning back sends the same body as before. - openai-native-messages: a `reasoningRequired` option hands reasoning back without the local token cap and, once any call carries reasoning, gives every assistant tool-call message the field (empty where the transcript kept none). - openai-provider: the option is on for DeepSeek's own host only (`api.deepseek.com`); other services may reject the unknown field and routers carry reasoning their own way. Linear: ATO-1397 --- .../provider/openai/openai-build-body.test.ts | 28 ++++++++ src/llm/provider/openai/openai-build-body.ts | 12 ++++ .../openai/openai-native-messages.test.ts | 69 +++++++++++++++++++ .../provider/openai/openai-native-messages.ts | 28 +++++++- .../provider/openai/openai-provider.test.ts | 34 ++++++++- src/llm/provider/openai/openai-provider.ts | 18 +++++ src/prompt/build-cloud-prompt.ts | 31 ++++++++- src/prompt/cloud-context.test.ts | 21 ++++++ 8 files changed, 236 insertions(+), 5 deletions(-) diff --git a/src/llm/provider/openai/openai-build-body.test.ts b/src/llm/provider/openai/openai-build-body.test.ts index 3198bb0f8..4d7a3c051 100644 --- a/src/llm/provider/openai/openai-build-body.test.ts +++ b/src/llm/provider/openai/openai-build-body.test.ts @@ -734,3 +734,31 @@ describe("buildOpenAiChatBody: reasoning handed back to a local template (ATO-88 } }); }); + +describe("buildOpenAiChatBody: reasoning DeepSeek requires back (ATO-1397)", () => { + const messages = { + system: "sys", + droppedSummary: null, + tail: "", + turns: [ + { kind: "user" as const, text: "read a.txt" }, + { kind: "assistant_tool_call" as const, tool: "os.fs.read", args: { path: "a.txt" }, reasoning: "need the file" }, + { kind: "tool_result" as const, tool: "os.fs.read", status: "ok" as const, body: "hello", truncated: false }, + ], + }; + const tools = [{ type: "function" as const, function: { name: "os__fs__read", parameters: { type: "object" } } }]; + const ask = (options: Parameters[7]) => + buildOpenAiChatBody({ prompt: "flat", messages, tools } as Parameters[0], "deepseek-v4-flash", false, undefined, undefined, undefined, undefined, options); + const assistant = (body: Record) => + (body.messages as Array>).find((m) => m.role === "assistant"); + + it("sends reasoning_content on a cloud body when the service requires it", () => { + expect( + assistant(ask({ providerKind: "openai-compatible", reasoningRequired: true }))?.reasoning_content, + ).toBe("need the file"); + }); + + it("keeps the field off the same cloud body otherwise", () => { + expect(assistant(ask({ providerKind: "openai-compatible" }))).not.toHaveProperty("reasoning_content"); + }); +}); diff --git a/src/llm/provider/openai/openai-build-body.ts b/src/llm/provider/openai/openai-build-body.ts index 7eb00906c..c60d5d3dd 100644 --- a/src/llm/provider/openai/openai-build-body.ts +++ b/src/llm/provider/openai/openai-build-body.ts @@ -61,6 +61,13 @@ export interface OpenAiBodyOptions { * `disableThinking` reach the wire. Absent, it is dropped. */ templateKwargs?: boolean; + /** + * The service requires the reasoning of earlier tool calls back on the + * assistant messages (DeepSeek's thinking mode, ATO-1397). Hands it back + * uncapped; see `NativeMessageOptions.reasoningRequired`. Absent, a cloud + * body carries no reasoning, as before. + */ + reasoningRequired?: boolean; } /** The wire layout `buildOpenAiChatBody` chose for a request. */ @@ -105,6 +112,11 @@ export function buildOpenAiChatBody( // A local server renders the template, which reads the // reasoning back where the model expects it (ATO-887). ...(options.templateKwargs ? { preserveReasoning: true } : {}), + // DeepSeek refuses a thinking-mode tool history without it + // (ATO-1397). + ...(options.reasoningRequired + ? { preserveReasoning: true, reasoningRequired: true } + : {}), }) : [ { diff --git a/src/llm/provider/openai/openai-native-messages.test.ts b/src/llm/provider/openai/openai-native-messages.test.ts index fadcdd64b..e3d4821f8 100644 --- a/src/llm/provider/openai/openai-native-messages.test.ts +++ b/src/llm/provider/openai/openai-native-messages.test.ts @@ -286,3 +286,72 @@ describe("isNativeShapeRejection", () => { expect(isNativeShapeRejection("role")).toBe(false); }); }); + +describe("buildNativeMessages: reasoning the service requires back (ATO-1397)", () => { + const words = (count: number, tag: string): string => + Array.from({ length: count }, (_, i) => `${tag}${i}`).join(" "); + const result: PromptMessages["turns"][number] = { + kind: "tool_result", + tool: "os.fs.read", + status: "ok", + body: "hello", + truncated: false, + }; + + it("hands every call's reasoning back whole, past the local cap", () => { + const older = words(3000, "old"); + const newer = words(3000, "new"); + const messages = buildNativeMessages( + prompt({ + turns: [ + { kind: "user", text: "go" }, + { kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "a" }, reasoning: older }, + result, + { kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "b" }, reasoning: newer }, + result, + ], + }), + { ...options, preserveReasoning: true, reasoningRequired: true }, + ); + const assistants = messages.filter((m) => m.role === "assistant"); + expect(estimateTokens(older + newer)).toBeGreaterThan(PRESERVED_REASONING_TOKEN_BUDGET); + expect(assistants.map((m) => m.reasoning_content)).toEqual([older, newer]); + }); + + it("gives a call without reasoning an empty field once the model is thinking", () => { + const messages = buildNativeMessages( + prompt({ + turns: [ + { kind: "user", text: "earlier" }, + { kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "a" } }, + result, + { kind: "assistant_reply", text: "done" }, + { kind: "user", text: "now" }, + { kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "b" }, reasoning: "need b" }, + result, + ], + }), + { ...options, reasoningRequired: true }, + ); + const calls = messages.filter((m) => m.role === "assistant" && m.tool_calls !== undefined); + expect(calls.map((m) => m.reasoning_content)).toEqual(["", "need b"]); + // A plain reply is not a tool-call message and keeps its old shape. + expect(messages.find((m) => m.role === "assistant" && m.content === "done")).not.toHaveProperty( + "reasoning_content", + ); + }); + + it("adds nothing when no call carries reasoning (thinking off)", () => { + const messages = buildNativeMessages( + prompt({ + turns: [ + { kind: "user", text: "go" }, + { kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "a" } }, + result, + ], + }), + { ...options, reasoningRequired: true }, + ); + expect(messages[2]).not.toHaveProperty("reasoning_content"); + }); +}); diff --git a/src/llm/provider/openai/openai-native-messages.ts b/src/llm/provider/openai/openai-native-messages.ts index 4c2355f82..fdc6fa3d3 100644 --- a/src/llm/provider/openai/openai-native-messages.ts +++ b/src/llm/provider/openai/openai-native-messages.ts @@ -47,6 +47,17 @@ export interface NativeMessageOptions { * unknown fields with a 400, so it is off by default. */ preserveReasoning?: boolean; + /** + * The service itself requires the reasoning back (DeepSeek's thinking + * mode answers a tool-call history without it with a 400, ATO-1397). + * Implies `preserveReasoning`, without the token cap: a dropped piece is + * what the service refuses, and a cloud window has room for it. Once + * any call in the request carries reasoning (the model is thinking), + * every assistant tool-call message carries the field, as an empty + * string where the transcript kept none (later calls of a batch, calls + * of answered turns), so no message lacks it. + */ + reasoningRequired?: boolean; } /** @@ -120,7 +131,18 @@ export function buildNativeMessages( // Consecutive orphan result lines, folded into one user message. let orphanLines: string[] = []; const reasoningByTurn = - options.preserveReasoning === true ? capPreservedReasoning(prompt.turns) : new Map(); + options.reasoningRequired === true + ? capPreservedReasoning(prompt.turns, Number.POSITIVE_INFINITY) + : options.preserveReasoning === true + ? capPreservedReasoning(prompt.turns) + : new Map(); + // What a tool-call message without reasoning of its own carries: + // nothing, unless the service requires the field and the model is + // thinking in this conversation. + const missingReasoning = + options.reasoningRequired === true && reasoningByTurn.size > 0 + ? "" + : undefined; const flushOrphans = (): void => { if (orphanLines.length === 0) return; @@ -162,7 +184,9 @@ export function buildNativeMessages( content: null, ...(reasoningByTurn.has(index) ? { reasoning_content: reasoningByTurn.get(index) } - : {}), + : missingReasoning !== undefined + ? { reasoning_content: missingReasoning } + : {}), tool_calls: [ { id, diff --git a/src/llm/provider/openai/openai-provider.test.ts b/src/llm/provider/openai/openai-provider.test.ts index 186d52ec2..ab58abeac 100644 --- a/src/llm/provider/openai/openai-provider.test.ts +++ b/src/llm/provider/openai/openai-provider.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import type { CompletionRequest } from "../completion-types.js"; -import { OpenAiProvider } from "./openai-provider.js"; +import { OpenAiProvider, isDeepSeekHost } from "./openai-provider.js"; const tools: NonNullable = [ { @@ -569,6 +569,27 @@ describe("OpenAiProvider — message shape on the wire", () => { await shaped(gpt.fetchImpl, { defaultChatModel: "openai/gpt-4.1" }).complete({ prompt: "flat", messages, tools }); expect(JSON.stringify(gpt.bodies[0])).not.toContain("ephemeral"); }); + + it("hands a call's reasoning back to DeepSeek's own API, and only there (ATO-1397)", async () => { + const thinking = { + ...messages, + turns: messages.turns.map((turn) => + turn.kind === "assistant_tool_call" ? { ...turn, reasoning: "need a" } : turn, + ), + }; + const assistant = (body: Record | undefined) => + (body?.messages as Array>).find((m) => m.role === "assistant"); + + const deepseek = capturing([okReply]); + await shaped(deepseek.fetchImpl, { baseUrl: "https://api.deepseek.com", defaultChatModel: "deepseek-v4-flash" }) + .complete({ prompt: "flat", messages: thinking, tools, sessionId: "s" }); + expect(assistant(deepseek.bodies[0])?.reasoning_content).toBe("need a"); + + const other = capturing([okReply]); + await shaped(other.fetchImpl, { defaultChatModel: "deepseek-v4-flash" }) + .complete({ prompt: "flat", messages: thinking, tools, sessionId: "s" }); + expect(assistant(other.bodies[0])).not.toHaveProperty("reasoning_content"); + }); }); describe("OpenAiProvider without defaultChatModel", () => { @@ -650,3 +671,14 @@ describe("OpenAiProvider template thinking switch", () => { expect(await sentBody("openai-compatible", "http://localhost:1234", { prompt: "turn" })).not.toHaveProperty("chat_template_kwargs"); }); }); + +describe("isDeepSeekHost (ATO-1397)", () => { + it("is DeepSeek's own API only", () => { + expect(isDeepSeekHost("https://api.deepseek.com")).toBe(true); + expect(isDeepSeekHost("https://api.deepseek.com/v1")).toBe(true); + expect(isDeepSeekHost("https://openrouter.ai/api")).toBe(false); + expect(isDeepSeekHost("https://deepseek.example.com")).toBe(false); + expect(isDeepSeekHost("not a url")).toBe(false); + expect(isDeepSeekHost(undefined)).toBe(false); + }); +}); diff --git a/src/llm/provider/openai/openai-provider.ts b/src/llm/provider/openai/openai-provider.ts index d6c22ba99..85ee39f3f 100644 --- a/src/llm/provider/openai/openai-provider.ts +++ b/src/llm/provider/openai/openai-provider.ts @@ -246,6 +246,7 @@ export class OpenAiProvider implements LlmProvider { ...(resolveAnthropicCacheControl(options) ? { anthropicCacheControl: true } : {}), + ...(isDeepSeekHost(options.baseUrl) ? { reasoningRequired: true } : {}), }; this.http = { baseUrl: normalizeOpenAiBaseUrl(options.baseUrl), @@ -869,6 +870,23 @@ function resolveAnthropicCacheControl(options: OpenAiProviderOptions): boolean { } } +/** + * Whether the base URL is DeepSeek's own API, whose thinking mode + * requires the previous tool calls' `reasoning_content` back on the next + * request and answers a history without it with a 400 (ATO-1397). Kept to + * the vendor's host: other services may reject the unknown field, and + * routers (OpenRouter) carry reasoning their own way. + */ +export function isDeepSeekHost(baseUrl: string | undefined): boolean { + if (!baseUrl) return false; + try { + const host = new URL(baseUrl).hostname.toLowerCase(); + return host === "deepseek.com" || host.endsWith(".deepseek.com"); + } catch { + return false; + } +} + /** The key a session's learned layout is remembered under. */ function sessionKey(request: Pick): string { return request.sessionId ?? "*"; diff --git a/src/prompt/build-cloud-prompt.ts b/src/prompt/build-cloud-prompt.ts index 36604a07a..21514b264 100644 --- a/src/prompt/build-cloud-prompt.ts +++ b/src/prompt/build-cloud-prompt.ts @@ -3,7 +3,9 @@ import { getReasoningTurnFraming } from "../llm/model-profile.js"; import { thinkingDisabledOnBuiltPrompt } from "../llm/server-template-policy.js"; import { extendCloudContext, cloudContextEntries, renderCloudTurn } from "../session/cloud-context.js"; import { projectSessionConversation } from "../session/session-compaction.js"; -import { macroTurnBoundaries } from "../session/conversation-turn.js"; +import { findCurrentMacroTurnStart, macroTurnBoundaries, type ConversationTurn } from "../session/conversation-turn.js"; +import type { CloudContextEntry } from "../session/cloud-context.js"; +import type { PromptTurn } from "../llm/provider/completion-types.js"; import type { BuildPromptInput, BuiltPrompt } from "./build-prompt-types.js"; import { cloudContextSections } from "./cloud-context-sections.js"; import { defaultBudget, estimateTokens } from "./token-budget.js"; @@ -49,7 +51,7 @@ export function buildCloudPrompt(input: BuildPromptInput, stablePrefix: string): const total = estimateTokens(text) + schemaTokens + entries.length * 12 + 32; return { text, stablePrefix, tail, cloudContext, modelMode: input.modelMode, - messages: { system: stablePrefix, turns: entries.map((entry) => entry.turn), tail: "", droppedSummary: null, + messages: { system: stablePrefix, turns: withCurrentReasoning(entries, input.session.turns), tail: "", droppedSummary: null, ...(projection.summary ? { contextSummary: projection.summary } : {}) }, tokens: { stablePrefix: estimateTokens(stablePrefix), loadedSkills: prefixTokens("skill:"), loadedTools: prefixTokens("tool:"), sessionFacts: sectionTokens("session-facts"), @@ -67,3 +69,28 @@ export function buildCloudPrompt(input: BuildPromptInput, stablePrefix: string): contextBudget: { window, replyReserve }, }; } + +/** + * The journal stores calls without their reasoning (it is persisted and + * append-only); the current, un-replied macro-turn's reasoning is put back + * on the outgoing copy from the transcript. Only a provider that hands + * reasoning back reads it (a template-rendering server, ATO-887; DeepSeek's + * thinking mode, which refuses a tool history without it, ATO-1397); every + * other body, and the text form, are unchanged. Answered turns get none, + * like the local path's `packedConversationTurns`. + */ +function withCurrentReasoning( + entries: readonly CloudContextEntry[], + turns: readonly ConversationTurn[], +): PromptTurn[] { + const currentStart = findCurrentMacroTurnStart(turns); + return entries.map((entry): PromptTurn => { + const turn = entry.turn; + if (turn.kind !== "assistant_tool_call" || !entry.id.startsWith("turn:")) return turn; + const index = Number(entry.id.slice("turn:".length)); + if (!Number.isInteger(index) || index < currentStart) return turn; + const source = turns[index]; + if (!source || source.kind !== "assistant_tool_call" || !source.reasoning) return turn; + return { ...turn, reasoning: source.reasoning }; + }); +} diff --git a/src/prompt/cloud-context.test.ts b/src/prompt/cloud-context.test.ts index 89a37f305..27520416b 100644 --- a/src/prompt/cloud-context.test.ts +++ b/src/prompt/cloud-context.test.ts @@ -128,4 +128,25 @@ describe("cloud message retention", () => { expect(buildPrompt(input).text).toContain(`${PROFILE_SECTION_CAPTION}\n- hobby_builder: true`); expect(buildPrompt({ ...input, profileFacts: [] }).text).not.toContain(PROFILE_SECTION_CAPTION); }); + + it("puts the current turn's call reasoning on the outgoing cloud turns, not in the journal (ATO-1397)", () => { + const input = fixture(); + input.session.turns.push( + assistantToolCallTurn({ tool: "os.fs.read", args: { path: "a" }, reasoning: "need a", at: 2 }), + toolResultTurn({ tool: "os.fs.read", status: "ok", summary: "A", at: 3 }), + ); + const first = buildPrompt(input); + const call = first.messages.turns.find((turn) => turn.kind === "assistant_tool_call"); + expect(call).toMatchObject({ reasoning: "need a" }); + expect(JSON.stringify(first.cloudContext)).not.toContain("need a"); + expect(first.text).not.toContain("need a"); + // A body that does not hand reasoning back is unchanged. + expect(JSON.stringify(native(first))).not.toContain("reasoning_content"); + + input.session.cloudContext = first.cloudContext; + input.session.turns.push(assistantReplyTurn("done", 4), userTurn("next", 5)); + const second = buildPrompt(input); + const answered = second.messages.turns.find((turn) => turn.kind === "assistant_tool_call"); + expect(answered).not.toHaveProperty("reasoning"); + }); }); From 1dc0b1997fa0a4eccc41b6dca01f53f496781663 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:03:00 +0300 Subject: [PATCH 006/155] fix(fusion): run local workers in the parent's working dir With the prompt now showing the session's folder, local Fusion workers were still created in the startup folder: only cloud passed the parent's working dir to the worker runner, together with the workspace inheritance flag. Pass the parent's folder in every model mode and keep the flag separate. createEphemeralSession takes an explicit inheritWorkspace argument (defaulting to the old behaviour), so a local worker runs in the parent's folder without inheriting a cloud workspace. Refs #1121 --- .../runtime-session-services.test.ts | 3 +++ .../composition/runtime-session-services.ts | 12 ++++++++--- src/runtime/runtime-contract.ts | 2 +- src/tools/fusion/fusion-delegate.test.ts | 21 ++++++++++++++++++- src/tools/fusion/fusion-delegate.ts | 11 +++++----- src/tools/fusion/worker-runner.ts | 12 ++++++----- 6 files changed, 45 insertions(+), 16 deletions(-) diff --git a/src/runtime/composition/runtime-session-services.test.ts b/src/runtime/composition/runtime-session-services.test.ts index 3eccdf923..2fe560832 100644 --- a/src/runtime/composition/runtime-session-services.test.ts +++ b/src/runtime/composition/runtime-session-services.test.ts @@ -113,6 +113,9 @@ describe("runtime session phases", () => { expect(inherited.workingDir).toBe(join(dir, "other")); expect(inherited.inheritedWorkspace).toBe(true); expect(worker.inheritedWorkspace).toBeUndefined(); + const local = factories.createEphemeralSession({ parentSessionId: deferred.id, taskId: "local" }, join(dir, "other"), false); + expect(local.workingDir).toBe(join(dir, "other")); + expect(local.inheritedWorkspace).toBeUndefined(); expect(calls).toEqual([]); const persistent = factories.createSession(); expect(calls).toEqual([`save:${persistent.id}`, `trace:${persistent.id}`]); diff --git a/src/runtime/composition/runtime-session-services.ts b/src/runtime/composition/runtime-session-services.ts index 43e5695e7..48c58d522 100644 --- a/src/runtime/composition/runtime-session-services.ts +++ b/src/runtime/composition/runtime-session-services.ts @@ -142,9 +142,15 @@ export function createRuntimeSessionFactories( // In memory only: no `sessionStore.save`, no `ensureRecorder`. The // worker stamp is what `executeTurn` keys its skips on. - const createEphemeralSession = (meta: FusionWorkerMeta, inheritedDir?: string): SessionState => ({ - ...createFusionWorkerSession({ workingDir: inheritedDir ?? workingDir, meta }), - ...(inheritedDir !== undefined ? { inheritedWorkspace: true } : {}), + // The folder and the cloud-workspace flag are separate: a local worker + // runs in its parent's folder without inheriting a workspace (#1121). + const createEphemeralSession = ( + meta: FusionWorkerMeta, + dir?: string, + inheritWorkspace: boolean = dir !== undefined, + ): SessionState => ({ + ...createFusionWorkerSession({ workingDir: dir ?? workingDir, meta }), + ...(inheritWorkspace ? { inheritedWorkspace: true } : {}), }); return { createSession, createEphemeralSession }; diff --git a/src/runtime/runtime-contract.ts b/src/runtime/runtime-contract.ts index 6b487fea5..00c0a5597 100644 --- a/src/runtime/runtime-contract.ts +++ b/src/runtime/runtime-contract.ts @@ -546,7 +546,7 @@ export interface AgentRuntime { * list. The orchestrator reads the returned transcript and discards * it. See `src/session/fusion-worker-session.ts`. */ - createEphemeralSession(meta: FusionWorkerMeta, workingDir?: string): SessionState; + createEphemeralSession(meta: FusionWorkerMeta, workingDir?: string, inheritWorkspace?: boolean): SessionState; /** Session-scoped cloud skills/instructions; null retains the legacy local view. */ getSessionWorkspace(session: SessionState | string): import("../session/workspace-context.js").SessionWorkspace | null; /** Maintain context without creating a turn or a transcript message. */ diff --git a/src/tools/fusion/fusion-delegate.test.ts b/src/tools/fusion/fusion-delegate.test.ts index fdce165e1..88200ea5d 100644 --- a/src/tools/fusion/fusion-delegate.test.ts +++ b/src/tools/fusion/fusion-delegate.test.ts @@ -12,7 +12,8 @@ import { join } from "node:path"; import type { RunTurnResult } from "../../agent/agent-loop.js"; import { resetConfigCache } from "../../config/index.js"; import type { ResolvedRunMode } from "../../llm/run-mode/index.js"; -import { createEmptySessionState } from "../../session/session-state.js"; +import { createEmptySessionState, type SessionState } from "../../session/session-state.js"; +import { createRuntimeSessionFactories } from "../../runtime/composition/runtime-session-services.js"; import { FUSION_WORKER_ID_PREFIX } from "../../session/fusion-worker-session.js"; import { StructuredLogger } from "../../tracing/structured-logger.js"; import type { ToolContext } from "../tool-registry.js"; @@ -119,6 +120,24 @@ describe("fusion.delegate", () => { const result = await buildFusionDelegateTool(deps({ createEphemeralSession: create, runTurn, workingDir: "/boot" })).run( { tasks: [TASKS[0]!] }, ctx({ workingDir: "/selected", modelMode: "cloud" })); expect(result.status).toBe("ok"); expect(create.mock.calls[0]?.[1]).toBe("/selected"); expect(runTurn).toHaveBeenCalledOnce(); + expect(create.mock.calls[0]?.[2]).toBe(true); + }); + + it("runs a local worker in the parent's folder without inheriting a workspace (issue #1121)", async () => { + // The real factory: bootstrap folder /boot, parent switched live to /selected. + const factories = createRuntimeSessionFactories("/boot", { save: () => {} }, () => null); + const create = vi.fn(factories.createEphemeralSession); + const runTurn = vi.fn(async (session: SessionState) => { + expect(session.workingDir).toBe("/selected"); + expect(session.inheritedWorkspace).toBeUndefined(); + return turnResult({ session }); + }); + const result = await buildFusionDelegateTool(deps({ createEphemeralSession: create, runTurn, workingDir: "/boot" })).run( + { tasks: [TASKS[0]!] }, ctx({ workingDir: "/selected", modelMode: "local" })); + expect(result.status).toBe("ok"); + expect(create.mock.calls[0]?.[1]).toBe("/selected"); + expect(create.mock.calls[0]?.[2]).toBe(false); + expect(runTurn).toHaveBeenCalledOnce(); }); it("is not readonly and carries its own name", () => { const tool = buildFusionDelegateTool(deps()); diff --git a/src/tools/fusion/fusion-delegate.ts b/src/tools/fusion/fusion-delegate.ts index 9482f045e..3e60a4394 100644 --- a/src/tools/fusion/fusion-delegate.ts +++ b/src/tools/fusion/fusion-delegate.ts @@ -782,9 +782,10 @@ export function buildFusionDelegateTool( allowSplit: boolean, ): Promise => runWorkerTasks( - ctx.modelMode === "cloud" - ? { ...deps, workingDir: ctx.workingDir, inheritWorkspace: true } - : deps, + // Workers run in the parent's folder in every model mode, so a + // live folder switch reaches them too (#1121). Only cloud + // workers inherit the parent's workspace. + { ...deps, workingDir: ctx.workingDir, ...(ctx.modelMode === "cloud" ? { inheritWorkspace: true } : {}) }, { ...(originalRequest === undefined ? {} : { originalRequest }), ...(contract === undefined ? {} : { contract }), @@ -845,9 +846,7 @@ export function buildFusionDelegateTool( ); const onlyRole = roles.size === 1 ? [...roles][0] : "-"; return runWorkerTasks( - ctx.modelMode === "cloud" - ? { ...deps, workingDir: ctx.workingDir, inheritWorkspace: true } - : deps, + { ...deps, workingDir: ctx.workingDir, ...(ctx.modelMode === "cloud" ? { inheritWorkspace: true } : {}) }, { ...(originalRequest === undefined ? {} : { originalRequest }), ...(contract === undefined ? {} : { contract }), diff --git a/src/tools/fusion/worker-runner.ts b/src/tools/fusion/worker-runner.ts index c0d72619f..812db95ba 100644 --- a/src/tools/fusion/worker-runner.ts +++ b/src/tools/fusion/worker-runner.ts @@ -441,7 +441,7 @@ export interface WorkerRunnerDeps { }, ) => Promise; /** `runtime.createEphemeralSession` — in-memory, never persisted. */ - createEphemeralSession: (meta: FusionWorkerMeta, workingDir?: string) => SessionState; + createEphemeralSession: (meta: FusionWorkerMeta, workingDir?: string, inheritWorkspace?: boolean) => SessionState; inheritWorkspace?: boolean; approvals: Pick & Partial< @@ -690,12 +690,13 @@ async function runOneTask( // the working directory everywhere below. Without it, nothing changes. const isolated = options.taskWorkspace?.(task.id); const workingDir = isolated?.workingDir ?? deps.workingDir; - const sessionDir = - isolated !== undefined || deps.inheritWorkspace ? workingDir : undefined; + // The worker always runs in this folder (the parent's, in every model + // mode, #1121); only an isolated or cloud worker inherits a workspace. + const inheritWorkspace = isolated !== undefined || deps.inheritWorkspace === true; const session = deps.createEphemeralSession({ parentSessionId: options.parentSessionId, taskId: task.id, - }, sessionDir); + }, workingDir, inheritWorkspace); // A worker that queued on the parent id would wait behind the // orchestrator's own turn — which is the turn calling this — and the // fan-out would deadlock rather than run. `createEphemeralSession` @@ -1483,7 +1484,8 @@ async function runOneTask( ) { const replaceSession = deps.createEphemeralSession( { parentSessionId: options.parentSessionId, taskId: task.id }, - sessionDir, + workingDir, + inheritWorkspace, ); armWorkerSession(replaceSession.id); const replaceCollector = new WorkerRunCollector(); From 195a2dca2df28f6fa2effd666003e829704a4b73 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:08:20 +0300 Subject: [PATCH 007/155] fix(tui): folded turn says what was done instead of "Used N tools" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fold line now takes its words from describeToolCalls, the same step verbs the tool cards use: "Read 2 files, ran 1 command · 12s". "Used N tools" stays the fallback when there is nothing to describe. Refs ATO-1380 --- src/tui/chat/README.md | 2 +- src/tui/chat/chat-log.test.tsx | 8 ++++---- src/tui/chat/turn-fold-line.tsx | 11 +++++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/tui/chat/README.md b/src/tui/chat/README.md index 39927270b..293314b7b 100644 --- a/src/tui/chat/README.md +++ b/src/tui/chat/README.md @@ -9,7 +9,7 @@ Read the inherited [TUI instructions](../AGENTS.md) and [interface contract](../ [chat-log.tsx](chat-log.tsx) composes finalised messages, streaming output and the empty-chat splash. Bubbles/tool cards, copy/link/retry/switch-back/fallback buttons, the `[copy path]` chip on finished read/write/edit cards ([tool-card-path-chip.tsx](tool-card-path-chip.tsx); copies, never opens), the stale-read note a collapsed card keeps on its own unclipped row, height measurement and splash fitting belong here. [turns-to-messages.ts](turns-to-messages.ts) projects persisted turns; [format-agent-error-for-chat.ts](format-agent-error-for-chat.ts) formats failures without changing agent policy. waiting-phrases.ts retains its existing exports. -A finished turn folds its reasoning and tool cards into one line, `Used 6 tools · 1 failed · 40s ▸` ([turn-fold-line.tsx](turn-fold-line.tsx), logic in [render/turn-collapse.ts](../render/turn-collapse.ts)). A click on it or `/expand` opens it (`/expand all` opens every turn and card, `/collapse` folds them back); an opened turn shows a card's result line only when the call failed. `tui.showSteps` (`/steps on|off`) keeps every finished turn open, as before. The running turn and its progress notes never fold. +A finished turn folds its reasoning and tool cards into one line that says what was done, `Read 3 files, ran 2 commands · 1 failed · 40s ▸` ([turn-fold-line.tsx](turn-fold-line.tsx), logic in [render/turn-collapse.ts](../render/turn-collapse.ts); the words come from `describeToolCalls` in [tools/describe-tool-call.ts](../../tools/describe-tool-call.ts), and `Used 6 tools` is the fallback when it has nothing to say). A click on it or `/expand` opens it (`/expand all` opens every turn and card, `/collapse` folds them back); an opened turn shows a card's result line only when the call failed. `tui.showSteps` (`/steps on|off`) keeps every finished turn open, as before. The running turn and its progress notes never fold. ChatMessage, pending output, tool expansion, turn marks and active-session attribution remain in [TuiState](../tui-state.ts) and [agent-event-reducer](../agent-event-reducer.ts). [ChatOrchestrator](../chat-orchestrator.ts) is the whole TUI runtime/session coordinator despite its name: switching, replay, detached turns, approvals, cancellation and feature orchestrators remain there. [TuiApp](../tui-app.tsx) composes chat with editor, panels and menus. Do not move runtime resources into bubbles. diff --git a/src/tui/chat/chat-log.test.tsx b/src/tui/chat/chat-log.test.tsx index 48bb82b01..6807b175f 100644 --- a/src/tui/chat/chat-log.test.tsx +++ b/src/tui/chat/chat-log.test.tsx @@ -372,7 +372,7 @@ describe("ChatLog", () => { it("hides the tool cards behind one summary line", () => { const { lastFrame } = renderAtSize(, BIG); const text = strip(lastFrame() ?? ""); - expect(text).toContain("Used 2 tools · 1 failed · 40s ▸"); + expect(text).toContain("Ran 1 search, ran 1 command · 1 failed · 40s ▸"); expect(text).toContain("Here you go."); expect(text).not.toContain("Listed files in src"); expect(text).not.toContain("LISTING-BODY"); @@ -385,7 +385,7 @@ describe("ChatLog", () => { BIG, ); const text = strip(lastFrame() ?? ""); - expect(text).toContain("Used 2 tools · 1 failed · 40s ▾"); + expect(text).toContain("Ran 1 search, ran 1 command · 1 failed · 40s ▾"); expect(text).toContain("Listed files in src"); expect(text).toContain("Ran false"); expect(text).toContain("EXIT-1-BODY"); @@ -398,7 +398,7 @@ describe("ChatLog", () => { BIG, ); const text = strip(lastFrame() ?? ""); - expect(text).not.toContain("Used 2 tools"); + expect(text).not.toContain("Ran 1 search, ran 1 command"); expect(text).toContain("LISTING-BODY"); expect(text).toMatch(/2 tool steps/); }); @@ -414,7 +414,7 @@ describe("ChatLog", () => { }; const { lastFrame } = renderAtSize(, BIG); const text = strip(lastFrame() ?? ""); - expect(text).not.toContain("Used 2 tools"); + expect(text).not.toContain("Ran 1 search, ran 1 command"); expect(text).toContain("Listed files in src"); }); }); diff --git a/src/tui/chat/turn-fold-line.tsx b/src/tui/chat/turn-fold-line.tsx index 8724fc944..2be272e53 100644 --- a/src/tui/chat/turn-fold-line.tsx +++ b/src/tui/chat/turn-fold-line.tsx @@ -1,5 +1,6 @@ import { Box, Text } from "ink"; import type { ReactElement } from "react"; +import { describeToolCalls } from "../../tools/describe-tool-call.js"; import { MouseTarget, useMouseCommands } from "../mouse/mouse-context.js"; import { isPrimaryPress } from "../mouse/mouse-event.js"; import { @@ -11,7 +12,9 @@ import { theme } from "../theme/theme.js"; /** * The one line a finished turn folds into (ATO-963): - * `Used 6 tools · 1 failed · 40s ▸`. A click opens the steps under it and + * `Read 3 files, ran 2 commands · 1 failed · 40s ▸`. The words come from + * the shared step verbs (ATO-962); "Used N tools" stays the fallback when + * they have nothing to say. A click opens the steps under it and * a second click folds them again; `/expand` does the same from the * keyboard. One row plus the blank row above it: the label truncates, the * failure count and the time never do. `estimateMessageHeight` charges the @@ -28,7 +31,7 @@ export function TurnFoldLine({ }): ReactElement { const parts = turnFoldLineParts(fold, { open, - ...(describe ? { describe } : {}), + describe: describe ?? describeFoldedCards, }); const sep = ` ${theme.glyphs.dotSeparator} `; const line = ( @@ -72,6 +75,10 @@ export function TurnFoldLine({ return {line}; } +/** "Read 3 files, ran 2 commands" for the folded cards; "" lets the caller fall back. */ +const describeFoldedCards: DescribeFoldedCards = (cards) => + describeToolCalls(cards.map((c) => ({ name: c.tool }))); + /** A click on the line opens / folds that one turn. Transparent without a mouse layer. */ function FoldToggle({ messageId, From 8f10e9829adc0bc8beb3089d52af97847265beb0 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:08:36 +0300 Subject: [PATCH 008/155] fix(budget): count context compaction in the task spend A compaction's summary calls are side calls, not step completions, so recordStepSpend never priced them and a long overnight run that compacted could spend past its budget unnoticed. The runner now reports each attempt's summary cost through CompactionStepOptions.onSpend, and the loop adds it to the turn spend at all three compaction sites. ATO-1322 --- src/agent/agent-loop.ts | 5 ++++- src/agent/compaction-control.ts | 6 ++++++ src/agent/turn/turn-spend.test.ts | 12 ++++++++++++ src/agent/turn/turn-spend.ts | 14 ++++++++++++++ src/runtime/context-compaction-runner.ts | 8 ++++++++ 5 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/agent/agent-loop.ts b/src/agent/agent-loop.ts index 14b5b7fb6..31602e34a 100644 --- a/src/agent/agent-loop.ts +++ b/src/agent/agent-loop.ts @@ -14,7 +14,7 @@ import { prepareAgentTurn, prepareTurnPolicies, prepareTurnBudgets, createTurnLo import { refreshMemoryContext, createSurfacedMemoryTracker } from "./turn/turn-memory-context.js"; export { MAX_SURFACED_NOTE_ALLOWLIST } from "./turn/turn-memory-context.js"; import { finalizeAgentTurn, finalizeFailedAgentTurn } from "./turn/turn-finalization.js"; -import { costCeilingNotice, costCeilingReached, recordStepSpend } from "./turn/turn-spend.js"; +import { costCeilingNotice, costCeilingReached, recordCompactionSpend, recordStepSpend } from "./turn/turn-spend.js"; export { formatTaskStoppedReply } from "./turn/turn-finalization.js"; import { delegationProducedWork, recordDelegation, wouldRefuse as fusionGateWouldRefuse } from "./policies/fusion-orchestrator-mode.js"; import { observeReviewStep, reviewStallSignal, reviewStallToolSet, takeReviewStallNotice, type ReviewStallSignal } from "./review-stall.js"; @@ -619,6 +619,7 @@ export class AgentLoop { signal: requestDeadline.signal, ...(options.providerId ? { providerId: options.providerId } : {}), ...(options.ephemeral ? { ephemeral: true } : {}), + onSpend: (usd) => recordCompactionSpend(turn, usd), }); while (maintenance) { state = await maintenance; @@ -639,6 +640,7 @@ export class AgentLoop { signal: requestDeadline.signal, ...(options.providerId ? { providerId: options.providerId } : {}), ...(options.ephemeral ? { ephemeral: true } : {}), + onSpend: (usd) => recordCompactionSpend(turn, usd), }); } } @@ -996,6 +998,7 @@ export class AgentLoop { signal: requestDeadline.signal, requested: "overflow", ...(options.providerId ? { providerId: options.providerId } : {}), ...(options.ephemeral ? { ephemeral: true } : {}), + onSpend: (usd) => recordCompactionSpend(turn, usd), }) ?? Promise.resolve(state)); } catch (compactionError) { this.deps.logger?.warn("overflow compaction failed", { ...logFields, diff --git a/src/agent/compaction-control.ts b/src/agent/compaction-control.ts index b8452c52f..d3dab60d7 100644 --- a/src/agent/compaction-control.ts +++ b/src/agent/compaction-control.ts @@ -15,6 +15,12 @@ export interface CompactionStepOptions { providerId?: string; ephemeral?: boolean; requested?: "overflow"; + /** + * Called once per compaction attempt with what its model calls cost, + * whether the checkpoint was saved or not. The loop adds it to the + * task's spend so a budget counts summaries too (ATO-1322). + */ + onSpend?: (usd: number) => void; } /** Runtime owns subcalls and durability; the loop supplies an exact, pure prompt input. */ diff --git a/src/agent/turn/turn-spend.test.ts b/src/agent/turn/turn-spend.test.ts index 94c948244..5aabe7441 100644 --- a/src/agent/turn/turn-spend.test.ts +++ b/src/agent/turn/turn-spend.test.ts @@ -5,6 +5,7 @@ import { costCeilingHeadline, costCeilingNotice, costCeilingReached, + recordCompactionSpend, recordStepSpend, resolveCostCeilingUsd, } from "./turn-spend.js"; @@ -69,4 +70,15 @@ describe("turn spend ceiling helpers", () => { expect(stopped).toMatch(/^\(stopped: spent \$5\.31 of \$5\.00 budget, after 7 steps over ~2 min\.\)/); expect(stopped).toContain("agent.task.maxCostUsd"); }); + + it("adds a compaction's summary cost to the turn spend (ATO-1322)", () => { + const turn = { spendUsd: 0.5, spendPriced: false }; + recordCompactionSpend(turn, 0.25); + expect(turn).toEqual({ spendUsd: 0.75, spendPriced: true }); + recordCompactionSpend(turn, 0); + recordCompactionSpend(turn, Number.NaN); + recordCompactionSpend(turn, -1); + expect(turn.spendUsd).toBe(0.75); + expect(costCeilingReached(turn.spendUsd, 0.75)).toBe(true); + }); }); diff --git a/src/agent/turn/turn-spend.ts b/src/agent/turn/turn-spend.ts index efe130a83..b7a161083 100644 --- a/src/agent/turn/turn-spend.ts +++ b/src/agent/turn/turn-spend.ts @@ -55,6 +55,20 @@ export function recordStepSpend( turn.spendPriced = true; } +/** + * Add what a context compaction's summary calls cost (ATO-1322). They + * are side calls, not step completions, so `recordStepSpend` never sees + * them; without this a long run that compacts spends past its budget. + */ +export function recordCompactionSpend( + turn: Pick, + usd: number, +): void { + if (!Number.isFinite(usd) || usd <= 0) return; + turn.spendUsd += usd; + turn.spendPriced = true; +} + /** Whether the spend has reached a ceiling that is set. */ export function costCeilingReached(spendUsd: number, ceilingUsd: number): boolean { return ceilingUsd > 0 && spendUsd >= ceilingUsd; diff --git a/src/runtime/context-compaction-runner.ts b/src/runtime/context-compaction-runner.ts index 67fff630b..91c21f876 100644 --- a/src/runtime/context-compaction-runner.ts +++ b/src/runtime/context-compaction-runner.ts @@ -48,6 +48,12 @@ export async function runContextCompaction( const emit = (event: CompactionEvent) => { try { deps.emit(event); } catch { /* Observers cannot roll back a saved checkpoint. */ } }; + // The summary calls were billed whatever happens to the checkpoint. + const reportSpend = () => { + const usd = tally.costUsd; + if (usd === undefined || !Number.isFinite(usd) || usd <= 0) return; + try { options.onSpend?.(usd); } catch { /* Spend accounting must not fail a compaction. */ } + }; let modelFailure: { timedOut: boolean; message: string } | undefined; let fellBack = false; emit({ type: "compaction_started", sessionId: state.id, reason: plan.reason, tokensBefore: plan.tokensBefore }); @@ -128,6 +134,7 @@ export async function runContextCompaction( ...(costUsd !== undefined ? { costUsd } : {}), ...(fellBack && modelFailure ? { message: `The model summary failed (${modelFailure.message.replace(/[.!?]+$/, "")}); an extractive summary was used instead.` } : {}), }; + reportSpend(); emit({ type: "compaction_completed", sessionId: state.id, result, usage, calls, contextUsage: next.contextUsage }); return { state: next, result, ...(modelFailure?.timedOut ? { timedOut: true } : {}) }; } catch (error) { @@ -141,6 +148,7 @@ export async function runContextCompaction( tokensBefore: plan.tokensBefore, ...(tally.costUsd !== undefined ? { costUsd: tally.costUsd } : {}), message, }; + reportSpend(); emit({ type: "compaction_failed", sessionId: state.id, result, usage: tally.usage, calls: tally.calls }); return { state, result, ...(failure?.timedOut ? { timedOut: true } : {}) }; } From d130e84175101e313718f404b73bcf609c13cbef Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:09:01 +0300 Subject: [PATCH 009/155] chore(tui): delete the unused chat-line-builder Nothing imports src/tui/render/chat-line-builder.ts except its own test; the chat is drawn by the Ink components in src/tui/chat. Drop both files and their entries in the test-type debt ledger (root files, the one diagnostic, the test TSX count). Refs ATO-1389 --- docs/testing/test-type-debt.json | 12 +- src/tui/render/chat-line-builder.test.tsx | 176 --------- src/tui/render/chat-line-builder.ts | 414 ---------------------- 3 files changed, 1 insertion(+), 601 deletions(-) delete mode 100644 src/tui/render/chat-line-builder.test.tsx delete mode 100644 src/tui/render/chat-line-builder.ts diff --git a/docs/testing/test-type-debt.json b/docs/testing/test-type-debt.json index 3a0531bd9..e8a1e8d4a 100644 --- a/docs/testing/test-type-debt.json +++ b/docs/testing/test-type-debt.json @@ -2679,8 +2679,6 @@ "src/tui/reduce-ui-actions.test.ts", "src/tui/reduce-ui-actions.ts", "src/tui/reducer-helpers.ts", - "src/tui/render/chat-line-builder.test.tsx", - "src/tui/render/chat-line-builder.ts", "src/tui/render/close-streaming-markdown.test.ts", "src/tui/render/close-streaming-markdown.ts", "src/tui/render/linkify-text.test.tsx", @@ -2886,7 +2884,7 @@ "src/update/run-app-update.ts", "src/version.ts" ], - "testTsx": 112, + "testTsx": 111, "diagnostics": [ { "path": "src/agent/agent-loop-fusion-seams.test.ts", @@ -5824,14 +5822,6 @@ "line": 8, "count": 1 }, - { - "path": "src/tui/render/chat-line-builder.test.tsx", - "code": 2739, - "message": "Type '{ sessionId: string; workingDir: string; llamaUrl: string; browserChannel: string; browserHeadless: false; approvalLevel: number; maxSteps: number; skillCount: number; }' is missing the following properties from type 'TuiSessionInfo': completionMaxTokens, localBackendConfigured", - "anchor": "SESSION", - "line": 12, - "count": 1 - }, { "path": "src/tui/run-mode/fusion-intro.test.ts", "code": 2741, diff --git a/src/tui/render/chat-line-builder.test.tsx b/src/tui/render/chat-line-builder.test.tsx deleted file mode 100644 index 159bfb1a1..000000000 --- a/src/tui/render/chat-line-builder.test.tsx +++ /dev/null @@ -1,176 +0,0 @@ -import { render } from "ink-testing-library"; -import { Box } from "ink"; -import { createElement } from "react"; -import { describe, expect, it } from "vitest"; -import { buildChatLines } from "./chat-line-builder.js"; -import { - createInitialTuiState, - type TuiSessionInfo, - type TuiState, -} from "../tui-state.js"; - -const SESSION: TuiSessionInfo = { - sessionId: "abc", - workingDir: "/tmp", - llamaUrl: "http://127.0.0.1:8080", - browserChannel: "chrome", - browserHeadless: false, - approvalLevel: 5, - maxSteps: 10, - skillCount: 0, -}; - -function renderLines(state: TuiState, columns = 80): string { - const lines = buildChatLines(state, { - columns, - toolsExpanded: state.toolsExpandedById, - }); - const tree = createElement(Box, { flexDirection: "column" }, ...lines); - const { lastFrame } = render(tree); - return (lastFrame() ?? "").replace(/\u001b\[[0-9;]*m/g, ""); -} - -describe("buildChatLines", () => { - it("returns no lines when there is no history and no streaming", () => { - const state = createInitialTuiState(SESSION); - const lines = buildChatLines(state, { - columns: 80, - toolsExpanded: state.toolsExpandedById, - }); - expect(lines).toHaveLength(0); - }); - - it("produces a separator + ribbon-padded body for a user message", () => { - const state: TuiState = { - ...createInitialTuiState(SESSION), - messages: [{ id: "u1", role: "user", text: "hello", timestamp: 1 }], - }; - const lines = buildChatLines(state, { - columns: 80, - toolsExpanded: state.toolsExpandedById, - }); - // marginTop + paddingTop + 1 body line + paddingBottom - expect(lines).toHaveLength(4); - expect(renderLines(state)).toContain("hello"); - }); - - it("strips simple markdown markers in finalised assistant text", () => { - const state: TuiState = { - ...createInitialTuiState(SESSION), - messages: [ - { - id: "a1", - role: "assistant", - text: "hello **bold** there `code` end", - toolSteps: 0, - timestamp: 1, - }, - ], - }; - const txt = renderLines(state); - expect(txt).toContain("hello bold there code end"); - expect(txt).not.toContain("**bold"); - expect(txt).not.toContain("`code`"); - }); - - it("keeps streaming assistant text verbatim (no markdown lex)", () => { - const state: TuiState = { - ...createInitialTuiState(SESSION), - streamingAssistantText: "hello **bold", - }; - expect(renderLines(state)).toContain("**bold"); - }); - - it("hides `reply` tool cards but renders other tools", () => { - const state: TuiState = { - ...createInitialTuiState(SESSION), - streamingToolCards: [ - { - id: "tc-real", - stepIndex: 0, - tool: "os.fs.read", - args: { path: "src/index.ts" }, - status: "ok", - summary: "read 42 lines", - truncated: false, - startedAt: 1, - finishedAt: 1, - }, - { - id: "tc-reply", - stepIndex: 0, - tool: "reply", - args: { text: "Done" }, - status: "ok", - summary: 'text="Done"', - truncated: false, - startedAt: 1, - finishedAt: 1, - }, - ], - }; - const txt = renderLines(state); - // ATO-962: the header says the call in words. - expect(txt).toContain("Read index.ts"); - expect(txt).not.toContain('text="Done"'); - }); - - it("wraps long body text into multiple physical rows", () => { - const long = "a ".repeat(80).trim(); - const state: TuiState = { - ...createInitialTuiState(SESSION), - messages: [{ id: "u1", role: "user", text: long, timestamp: 1 }], - }; - const lines = buildChatLines(state, { - columns: 30, - toolsExpanded: state.toolsExpandedById, - }); - // marginTop(1) + paddingTop(1) + ≥3 body rows + paddingBottom(1) - expect(lines.length).toBeGreaterThanOrEqual(6); - }); - - it("collapses long reasoning into a one-line summary when not expanded", () => { - const longThink = "lorem ipsum ".repeat(60); - const state: TuiState = { - ...createInitialTuiState(SESSION), - reasoning: [{ id: "r1", stepIndex: 0, text: longThink, timestamp: 1 }], - streamingAssistantText: "Hi", - }; - const txt = renderLines(state); - expect(txt).toContain("reasoning"); - // Summary clipping ends with the ellipsis sentinel. - expect(txt).toMatch(/…/); - }); - - it("expands reasoning while reply has not started streaming", () => { - const state: TuiState = { - ...createInitialTuiState(SESSION), - reasoning: [ - { - id: "r1", - stepIndex: 0, - text: "first I will enumerate the options", - timestamp: 1, - }, - ], - streamingAssistantText: null, - }; - expect(renderLines(state)).toContain("first I will enumerate the options"); - }); - - it("renders the assistant tool-step footer when toolSteps > 0", () => { - const state: TuiState = { - ...createInitialTuiState(SESSION), - messages: [ - { - id: "a1", - role: "assistant", - text: "done", - toolSteps: 3, - timestamp: 1, - }, - ], - }; - expect(renderLines(state)).toMatch(/3 tool steps/); - }); -}); diff --git a/src/tui/render/chat-line-builder.ts b/src/tui/render/chat-line-builder.ts deleted file mode 100644 index 229784ee5..000000000 --- a/src/tui/render/chat-line-builder.ts +++ /dev/null @@ -1,414 +0,0 @@ -import { Text } from "ink"; -import { - createElement, - Fragment, - type ReactElement, - type ReactNode, -} from "react"; -import { describeToolCall } from "../../tools/describe-tool-call.js"; -import { formatToolArgsBlock } from "./tool-args-preview.js"; -import { wrapText } from "./wrap-text.js"; -import { theme } from "../theme/theme.js"; -import type { - ChatMessage, - StreamingToolCall, - ToolCardEntry, - TuiState, -} from "../tui-state.js"; - -/** - * Builds a flat array of single-row `` elements that represent - * the entire chat history at the current width. Each element is - * exactly one terminal row (no internal `\n`, no implicit Ink wrap), - * which lets `chat-log.tsx` slice the buffer into a viewport-sized - * window for line-precise scrolling — exactly the behaviour we want - * from a "scrolls like a web page" UX. - * - * The builder is pure and deterministic: same `state` + `ctx` always - * produces the same `Line[]`. No refs, no measurement, no state hooks. - * - * Output ordering mirrors the previous `` component tree: - * - finalised messages, oldest first - * - streaming tail (reasoning → tool cards → in-flight tool calls - * → assistant reply preview) - * - thinking indicator (when the turn is in flight) - */ -const RIBBON_GLYPH = "│"; -const HIDDEN_TOOL_NAMES: ReadonlySet = new Set(["reply"]); -const SUMMARY_LIMIT = 180; -const PREVIEW_SUMMARY_LIMIT = 160; - -export interface ChatLineCtx { - /** Effective inner width available to a chat row, in columns. */ - columns: number; - /** Per-tool-card expand bit, indexed by `toolCardId`. */ - toolsExpanded: Readonly>; -} - -interface BuilderState { - lines: ReactElement[]; - next: number; -} - -export function buildChatLines( - state: TuiState, - ctx: ChatLineCtx, -): ReactElement[] { - const builder: BuilderState = { lines: [], next: 0 }; - for (const message of state.messages) { - appendMessage(builder, message, ctx); - } - appendStreamingTail(builder, state, ctx); - return builder.lines; -} - -function appendMessage( - b: BuilderState, - message: ChatMessage, - ctx: ChatLineCtx, -): void { - if (message.role === "user") { - appendBubble(b, message.text, theme.colors.user, ctx, { markdown: false }); - return; - } - if (message.role === "assistant") { - if (message.reasoningBlocks && message.reasoningBlocks.length > 0) { - appendReasoningBubble(b, message.reasoningBlocks, false, ctx); - } - if (message.toolCards) { - for (const card of message.toolCards) { - if (HIDDEN_TOOL_NAMES.has(card.tool)) continue; - appendToolCard(b, card, ctx.toolsExpanded[card.id] ?? false, ctx); - } - } - appendBubble(b, message.text, theme.colors.assistant, ctx, { - markdown: true, - }); - if ((message.toolSteps ?? 0) > 0) { - appendAssistantFooter(b, message.toolSteps ?? 0); - } - return; - } - appendSystemBubble(b, message.text, ctx); -} - -function appendStreamingTail( - b: BuilderState, - state: TuiState, - ctx: ChatLineCtx, -): void { - const hasStream = - state.streamingAssistantText !== null || - state.streamingToolCalls.length > 0 || - state.streamingToolCards.length > 0 || - state.reasoning.length > 0; - if (!hasStream) return; - const reasoningTexts = state.reasoning.map((r) => r.text); - const reasoningExpanded = - state.streamingAssistantText === null || - state.streamingAssistantText.length === 0; - if (reasoningTexts.length > 0) { - appendReasoningBubble(b, reasoningTexts, reasoningExpanded, ctx); - } - for (const card of state.streamingToolCards) { - if (HIDDEN_TOOL_NAMES.has(card.tool)) continue; - appendToolCard(b, card, ctx.toolsExpanded[card.id] ?? false, ctx); - } - for (const call of state.streamingToolCalls) { - if (HIDDEN_TOOL_NAMES.has(call.tool)) continue; - appendToolCard(b, call, ctx.toolsExpanded[call.id] ?? false, ctx); - } - if (state.streamingAssistantText !== null) { - appendBubble(b, state.streamingAssistantText, theme.colors.assistant, ctx, { - markdown: false, - }); - } -} - -interface BubbleOptions { - /** Strip simple markdown markers (`**bold**`, `*italic*`, `` `code` ``) before wrap. */ - markdown: boolean; -} - -function appendBubble( - b: BuilderState, - rawText: string, - color: string, - ctx: ChatLineCtx, - opts: BubbleOptions, -): void { - const text = opts.markdown ? stripMarkdownMarkers(rawText) : rawText; - const innerWidth = Math.max(10, ctx.columns - 4); - const wrapped = wrapText(text, innerWidth); - // marginTop=1 — empty separator row above the bubble. - pushBlank(b); - // paddingTop — ribbon-only row (border with no body content). - pushRibbon(b, color); - // Body rows. - if (wrapped.length === 0 || (wrapped.length === 1 && wrapped[0] === "")) { - pushRibbonBody(b, color, ""); - } else { - for (const row of wrapped) pushRibbonBody(b, color, row); - } - // paddingBottom — ribbon-only row. - pushRibbon(b, color); -} - -function appendReasoningBubble( - b: BuilderState, - blocks: readonly string[], - expanded: boolean, - ctx: ChatLineCtx, -): void { - if (blocks.length === 0) return; - const joined = blocks.join("\n\n").trim(); - if (joined.length === 0) return; - const innerWidth = Math.max(10, ctx.columns - 4); - const body = expanded - ? joined - : clip(joined.replace(/\s+/g, " "), SUMMARY_LIMIT); - const wrapped = wrapText(body, innerWidth); - const color = theme.colors.reasoning; - pushBlank(b); - pushRibbon(b, color); - // Header line: "◈ reasoning (N blocks)". - const headerKey = `cl-${b.next++}`; - b.lines.push( - createElement( - Text, - { key: headerKey }, - createElement(Text, { color }, `${RIBBON_GLYPH} `), - createElement( - Text, - { color, bold: true }, - `${theme.glyphs.reasoningMarker} reasoning`, - ), - createElement( - Text, - { color: theme.colors.muted }, - ` (${blocks.length} block${blocks.length === 1 ? "" : "s"})`, - ), - ), - ); - for (const row of wrapped) { - pushRibbonBodyMuted(b, color, row); - } - pushRibbon(b, color); -} - -function appendSystemBubble( - b: BuilderState, - text: string, - ctx: ChatLineCtx, -): void { - const color = theme.colors.system; - const innerWidth = Math.max(10, ctx.columns - 4); - const wrapped = wrapText(text, innerWidth); - pushBlank(b); - for (const row of wrapped) { - const key = `cl-${b.next++}`; - b.lines.push( - createElement( - Text, - { key }, - createElement(Text, { color }, ` ${theme.glyphs.systemMarker} `), - createElement(Text, { color }, row), - ), - ); - } -} - -function appendAssistantFooter(b: BuilderState, toolSteps: number): void { - const key = `cl-${b.next++}`; - b.lines.push( - createElement( - Text, - { key }, - createElement(Text, { color: theme.colors.assistant }, " ●"), - createElement( - Text, - { color: theme.colors.muted }, - ` ${toolSteps} tool step${toolSteps === 1 ? "" : "s"}`, - ), - ), - ); -} - -function appendToolCard( - b: BuilderState, - card: ToolCardEntry | StreamingToolCall, - expanded: boolean, - ctx: ChatLineCtx, -): void { - const isFinalised = "status" in card; - const status: "pending" | "ok" | "error" = isFinalised - ? card.status - : "pending"; - const color = toolColor(status); - const glyph = toolGlyph(status); - const duration = - isFinalised && card.finishedAt - ? `${card.finishedAt - card.startedAt}ms` - : "…"; - const truncated = isFinalised && card.truncated; - // ATO-962: the call in words, not `os.fs.read path="…"`; the raw name - // and args are in the expanded block. ATO-914: one row, always — the - // words are clipped to what the row has left after the frame and the - // duration, so a long edit path no longer runs off the right edge. - const prefix = ` ${theme.glyphs.toolBoxTopLeft}${theme.glyphs.toolBoxHorizontal} ${glyph} `; - const durationText = ` ${theme.glyphs.dotSeparator} ${duration}`; - const truncatedText = truncated ? ` ${theme.glyphs.dotSeparator} truncated` : ""; - const room = Math.max( - 8, - ctx.columns - prefix.length - durationText.length - truncatedText.length, - ); - const described = describeToolCall({ - name: card.tool, - args: card.args, - ...(status === "pending" ? {} : { ok: status === "ok" }), - }).line; - - const headerKey = `cl-${b.next++}`; - b.lines.push( - createElement( - Text, - { key: headerKey, wrap: "truncate-end" }, - createElement(Text, { color }, prefix + clipOneLine(described, room)), - createElement(Text, { color: theme.colors.muted }, durationText), - truncated - ? createElement(Text, { color: theme.colors.warn }, truncatedText) - : null, - ), - ); - - if (!expanded) { - if (isFinalised) { - const summary = clipOneLine(card.summary, PREVIEW_SUMMARY_LIMIT); - if (summary !== "") { - const sumKey = `cl-${b.next++}`; - b.lines.push( - createElement( - Text, - { key: sumKey, color }, - ` ${theme.glyphs.toolBoxBottomLeft}${theme.glyphs.toolBoxHorizontal} ${summary}`, - ), - ); - } - } - pushBlank(b); - return; - } - - // Expanded: args + result. - const innerWidth = Math.max(10, ctx.columns - 4); - pushPlain(b, clipOneLine(` tool ${card.tool}`, Math.max(10, ctx.columns)), theme.colors.muted); - pushPlain(b, " args", theme.colors.muted); - for (const row of wrapText(formatToolArgsBlock(card.args), innerWidth)) { - pushPlain(b, ` ${row}`); - } - if (isFinalised) { - pushPlain(b, " result", theme.colors.muted); - for (const row of wrapText(card.summary, innerWidth)) { - pushPlain(b, ` ${row}`, color); - } - } else { - pushPlain(b, " (pending)", theme.colors.muted); - } - pushBlank(b); -} - -function pushBlank(b: BuilderState): void { - const key = `cl-${b.next++}`; - b.lines.push(createElement(Text, { key }, " ")); -} - -function pushRibbon(b: BuilderState, color: string): void { - const key = `cl-${b.next++}`; - b.lines.push( - createElement(Text, { key }, createElement(Text, { color }, RIBBON_GLYPH)), - ); -} - -function pushRibbonBody(b: BuilderState, color: string, body: string): void { - const key = `cl-${b.next++}`; - b.lines.push( - createElement( - Text, - { key }, - createElement(Text, { color }, `${RIBBON_GLYPH} `), - createElement(Text, null, body), - ), - ); -} - -function pushRibbonBodyMuted( - b: BuilderState, - color: string, - body: string, -): void { - const key = `cl-${b.next++}`; - b.lines.push( - createElement( - Text, - { key }, - createElement(Text, { color }, `${RIBBON_GLYPH} `), - createElement(Text, { color: theme.colors.muted }, body), - ), - ); -} - -function pushPlain(b: BuilderState, body: string, color?: string): void { - const key = `cl-${b.next++}`; - const node: ReactNode = color ? createElement(Text, { color }, body) : body; - b.lines.push(createElement(Text, { key }, node ?? createElement(Fragment))); -} - -function toolColor(status: "pending" | "ok" | "error"): string { - switch (status) { - case "ok": - return theme.colors.toolOk; - case "error": - return theme.colors.toolError; - default: - return theme.colors.tool; - } -} - -function toolGlyph(status: "pending" | "ok" | "error"): string { - switch (status) { - case "ok": - return theme.glyphs.check; - case "error": - return theme.glyphs.cross; - default: - return theme.glyphs.chevronRight; - } -} - -function clip(value: string, limit: number): string { - if (value.length <= limit) return value; - return `${value.slice(0, limit - 1)}…`; -} - -function clipOneLine(value: string, limit: number): string { - const oneLine = value.replace(/\s+/g, " ").trim(); - if (oneLine.length <= limit) return oneLine; - return `${oneLine.slice(0, limit - 1)}…`; -} - -/** - * Strips the three markdown markers we render literally elsewhere - * (`**bold**`, `*italic*`, `` `code` ``) so the assistant's reply - * does not display the raw asterisks / backticks. Inline bold styling - * is intentionally **not** preserved — keeping the string flat is the - * load-bearing contract that lets `wrapText` count physical columns - * without reasoning about markup. Multi-line fences (```` ``` ````) - * are not stripped: they would be more confusing without the fence - * marker on a row of plain bodies. - */ -function stripMarkdownMarkers(text: string): string { - return text - .replace(/`([^`\n]+)`/g, "$1") - .replace(/\*\*([^*\n]+)\*\*/g, "$1") - .replace(/(? Date: Wed, 11 Nov 2026 02:09:18 +0300 Subject: [PATCH 010/155] Reply cleanup: keep trailing globs, indented fences, full paths on request, repeats on cloud models - A glob at the very end (src/**, : *) is no longer cut, and x**2 no longer gets a stray closing ** (ATO-1325). - A fenced block indented four or more spaces is treated as code, so nothing inside it is rewritten (ATO-1326). - Long home paths stay as written when the user asked for the full or absolute path (ATO-1327). - Repeated-paragraph removal runs for small local models only (ATO-1328). --- src/agent/close-streaming-markdown.ts | 2 +- src/agent/reply-polish.test.ts | 39 ++++++++++++++++++++--- src/agent/reply-polish.ts | 45 +++++++++++++++++++++------ 3 files changed, 72 insertions(+), 14 deletions(-) diff --git a/src/agent/close-streaming-markdown.ts b/src/agent/close-streaming-markdown.ts index 3ac2610ee..64c667aa8 100644 --- a/src/agent/close-streaming-markdown.ts +++ b/src/agent/close-streaming-markdown.ts @@ -23,7 +23,7 @@ * (`reply-polish.ts`) reuses it, keeping only plain closer fixes. */ -const FENCE = /^ {0,3}(```|~~~)/gm; +const FENCE = /^[ \t]*(```|~~~)/gm; export function closeStreamingMarkdown(text: string): string { if (text.length === 0) return text; diff --git a/src/agent/reply-polish.test.ts b/src/agent/reply-polish.test.ts index 8aad52f7a..686b18999 100644 --- a/src/agent/reply-polish.test.ts +++ b/src/agent/reply-polish.test.ts @@ -48,6 +48,14 @@ describe("polishReply: safety", () => { }); describe("polishReply: code is never rewritten", () => { + it("treats a fence indented four or more spaces as code", () => { + const reply = + "1. Step one\n - run:\n\n ```sh\n cd /Users/me/projects/atomic/src/agent/deep/folder\n ```\n\nDone."; + expect(polishReply(reply, tui)).toBe(reply); + const segs = splitFences(reply); + expect(segs.some((s) => s.kind === "code" && s.text.includes("cd /Users/me/projects"))).toBe(true); + }); + it("keeps blank lines, repeats and filler inside a fenced block", () => { const reply = [ "Here is the script:", @@ -102,6 +110,16 @@ describe("polishReply: open markup", () => { expect(polishReply("Done **", tui)).toBe("Done"); }); + it("keeps a glob or a star at the very end", () => { + expect(polishReply("Matches everything under src/**", tui)).toBe("Matches everything under src/**"); + expect(polishReply("The pattern is: *", tui)).toBe("The pattern is: *"); + }); + + it("does not close a marker inside a word, like x**2", () => { + expect(polishReply("The area is x**2", tui)).toBe("The area is x**2"); + expect(polishReply("In Python, write 2**10", tui)).toBe("In Python, write 2**10"); + }); + it("does not close a lone star, which is as likely a glob", () => { expect(polishReply("Use the glob src/*.ts", tui)).toBe("Use the glob src/*.ts"); }); @@ -205,11 +223,11 @@ describe("polishReply: echoed question", () => { describe("polishReply: repeats", () => { it("drops a paragraph repeated right after itself", () => { - expect(polishReply("Para A.\n\nPara A.\n\nPara B.", tui)).toBe("Para A.\n\nPara B."); + expect(polishReply("Para A.\n\nPara A.\n\nPara B.", small)).toBe("Para A.\n\nPara B."); }); it("treats whitespace-only differences as the same paragraph", () => { - expect(polishReply("The build\npasses.\n\nThe build passes.", tui)).toBe("The build\npasses."); + expect(polishReply("The build\npasses.\n\nThe build passes.", small)).toBe("The build\npasses."); }); it("keeps a paragraph repeated with something between", () => { @@ -218,12 +236,18 @@ describe("polishReply: repeats", () => { }); it("drops a second half that restates the whole answer", () => { - expect(polishReply("First.\n\nSecond.\n\nFirst.\n\nSecond.", tui)).toBe("First.\n\nSecond."); + expect(polishReply("First.\n\nSecond.\n\nFirst.\n\nSecond.", small)).toBe("First.\n\nSecond."); }); it("drops the whole answer restated around a code block", () => { const once = "Use this:\n```\nnpm ci\n```"; - expect(polishReply(`${once}\n${once}`, tui)).toBe(once); + expect(polishReply(`${once}\n${once}`, small)).toBe(once); + }); + + it("keeps repeated paragraphs for a cloud model, which repeats only when asked", () => { + expect(polishReply("Para A.\n\nPara A.\n\nPara B.", tui)).toBe("Para A.\n\nPara A.\n\nPara B."); + const halves = "First.\n\nSecond.\n\nFirst.\n\nSecond."; + expect(polishReply(halves, tui)).toBe(halves); }); it("keeps two identical code blocks with no prose between", () => { @@ -278,6 +302,13 @@ describe("polishReply: home paths", () => { ); }); + it("keeps the full path when the user asked for it", () => { + const reply = "Saved to /Users/me/projects/atomic/src/agent/reply-polish.ts now"; + expect(polishReply(reply, { ...tui, userMessage: "What is the full path of the polish file?" })).toBe(reply); + expect(polishReply(reply, { ...tui, userMessage: "give me the absolute path" })).toBe(reply); + expect(polishReply(reply, { ...tui, userMessage: "Напиши полный путь к файлу" })).toBe(reply); + }); + it("leaves a short path alone", () => { expect(polishReply("see /Users/me/a.ts", tui)).toBe("see /Users/me/a.ts"); }); diff --git a/src/agent/reply-polish.ts b/src/agent/reply-polish.ts index 834433341..62842348b 100644 --- a/src/agent/reply-polish.ts +++ b/src/agent/reply-polish.ts @@ -13,13 +13,15 @@ * ("I hope this helps!", "Надеюсь, это поможет!"), at the very start * or end only, and only when something else is left; * - drops a first paragraph that only repeats the user's question; - * - drops a paragraph that repeats the one right before it (exactly or - * up to whitespace), and a reply whose second half repeats its first; + * - small models only: drops a paragraph that repeats the one right + * before it (exactly or up to whitespace), and a reply whose second + * half repeats its first; a cloud model repeats only when asked to; * - small models only: a line repeated three or more times in a row is * kept once (a decoding loop, not content); * - collapses runs of blank lines to one; * - shortens a long absolute path under the home folder to `~/...` - * when it is plain prose (not code, not a link target); + * when it is plain prose (not code, not a link target), unless the + * user's message asks for the full or absolute path; * - Telegram only: turns a markdown table into a list, since the chat * has no tables and a pipe grid wraps into noise on a phone. * @@ -82,7 +84,9 @@ interface Segment { text: string; } -const FENCE_OPEN = /^ {0,3}(`{3,}|~{3,})/; +// Any indent: a fence nested under a list item (`1. ` then a deeper +// bullet) sits four or more spaces in, and is still code. +const FENCE_OPEN = /^[ \t]*(`{3,}|~{3,})/; /** * Split into prose and fenced-code segments, line by line. An unclosed @@ -108,7 +112,7 @@ export function splitFences(text: string): Segment[] { } const marker = open[1]!; const ch = marker[0]!; - const close = new RegExp(`^ {0,3}${ch === "`" ? "`" : "~"}{${marker.length},}\\s*$`); + const close = new RegExp(`^[ \\t]*${ch === "`" ? "`" : "~"}{${marker.length},}\\s*$`); const body: string[] = [line]; let j = i + 1; let closed = false; @@ -178,14 +182,14 @@ function polish(input: string, options: PolishReplyOptions): string { let body = seg.text; if (options.smallModel === true) body = collapseRepeatedLines(body); if (options.channel === "telegram") body = tablesToLists(body); - body = shortenHomePaths(body, options.homeDir ?? safeHomedir()); + if (!asksForFullPath(options.userMessage)) body = shortenHomePaths(body, options.homeDir ?? safeHomedir()); body = collapseBlankLines(body); return { kind: "prose", text: body }; }); if (options.userMessage) segments = dropEchoedQuestion(segments, options.userMessage); segments = dropFillerAtEdges(segments); - segments = dropRepeatedParagraphs(segments); + if (options.smallModel === true) segments = dropRepeatedParagraphs(segments); let out = segments .map((s) => s.text) @@ -235,16 +239,31 @@ function closeOpenMarkup(text: string): string { const base = text.trimEnd(); if (fixed.startsWith(base)) { const suffix = fixed.slice(base.length); - return /^(?:`+|\*\*|~~)+$/.test(suffix) ? fixed : text; + if (!/^(?:`+|\*\*|~~)+$/.test(suffix)) return text; + // `x**2` or `a~~b`: a marker inside a word is maths or a name, not + // an opener, so closing it would add stray stars. + for (const marker of ["**", "~~"]) { + if (suffix.includes(marker) && openerInsideWord(base, marker)) return text; + } + return fixed; } const kept = fixed.trimEnd(); if (kept.length > 0 && base.startsWith(kept)) { + // Only a spaced-off opener ("Done **"). `src/**` and `: *` end in a + // glob, which is content and stays. const removed = base.slice(kept.length); - return /^[\s*~`]+$/.test(removed) ? kept : text; + return /^\s+(?:\*\*|~~|`+)+$/.test(removed) ? kept : text; } return text; } +/** The last `marker` in `text` sits between two word characters. */ +function openerInsideWord(text: string, marker: string): boolean { + const at = text.lastIndexOf(marker); + if (at <= 0) return false; + return /[\p{L}\p{N}_]/u.test(text[at - 1]!); +} + // --- paragraphs ------------------------------------------------------------ function paragraphsOf(text: string): string[] { @@ -455,6 +474,14 @@ function dropEchoedQuestion(segments: Segment[], userMessage: string): Segment[] // --- home paths ------------------------------------------------------------ +/** The user asked for the path as it is: "full path", "абсолютный путь". */ +const FULL_PATH_ASK = + /\b(?:full|absolute|complete|whole|exact)\s+(?:file\s+)?path|\bpath\s+in\s+full\b|(?:полн|абсолютн|цельн|точн)\S*\s+(?:\S+\s+)?пут|пут\S*\s+(?:целиком|полностью)/iu; + +function asksForFullPath(userMessage: string | null | undefined): boolean { + return typeof userMessage === "string" && FULL_PATH_ASK.test(userMessage); +} + /** Paths shorter than this are left alone: they already read fine. */ const LONG_PATH = 40; From 7bea0eb49069d75ae494c7c94d3c72fde225bf15 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:09:41 +0300 Subject: [PATCH 011/155] fix(overnight): say stopped and exit 2 when a limit ended the run The task store records a run a budget, time or step ceiling ended as completed, with the cause beside it. The overnight command printed "overnight completed" and exited 0 for it, so a script calling it took unfinished work for a success. It now prints "stopped ()" with the plain reason line and exits 2; 0 stays for a run the agent finished on its own. ATO-1323 --- src/cli/overnight-command.ts | 3 ++- src/cli/overnight-progress.test.ts | 17 +++++++++++---- src/cli/overnight-progress.ts | 33 ++++++++++++++++++++++++++---- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/cli/overnight-command.ts b/src/cli/overnight-command.ts index 192302752..35ad49628 100644 --- a/src/cli/overnight-command.ts +++ b/src/cli/overnight-command.ts @@ -59,6 +59,7 @@ const HELP = [ " is denied, since nobody is there to answer.", "", "Check a run with `atomic-agent task show `, stop it with `atomic-agent task cancel `.", + "Exit code: 0 finished, 2 stopped at the budget, time or step limit, 1 failed or cancelled.", "", "Examples:", ' atomic-agent overnight "migrate the tests to vitest" --budget 5 --dir ~/code/app', @@ -344,7 +345,7 @@ async function runForeground(args: OvernightArgs, taskId: string): Promise { expect(failed).not.toContain("denied"); }); - it("exits 0 only on completed", () => { - expect(overnightExitCode("completed")).toBe(0); - expect(overnightExitCode("failed")).toBe(1); - expect(overnightExitCode("cancelled")).toBe(1); + it("exits 0 only when the agent finished on its own", () => { + expect(overnightExitCode({ status: "completed", stopCause: null })).toBe(0); + expect(overnightExitCode({ status: "failed", stopCause: null })).toBe(1); + expect(overnightExitCode({ status: "cancelled", stopCause: null })).toBe(1); + }); + + it("exits 2 and says stopped when a limit ended the run (ATO-1323)", () => { + for (const cause of ["cost_ceiling", "time_ceiling", "step_ceiling", "no_progress"]) { + expect(overnightExitCode({ status: "completed", stopCause: cause })).toBe(2); + expect(overnightOutcome({ status: "completed", stopCause: cause })).toBe(`stopped (${cause})`); + } + expect(overnightOutcome({ status: "completed", stopCause: null })).toBe("completed"); }); }); diff --git a/src/cli/overnight-progress.ts b/src/cli/overnight-progress.ts index 6dc6c19f1..aefa18701 100644 --- a/src/cli/overnight-progress.ts +++ b/src/cli/overnight-progress.ts @@ -5,7 +5,7 @@ */ import type { AgentLoopEvent } from "../agent/index.js"; import { formatUsd } from "../llm/provider/usage-cost.js"; -import type { TaskRecord } from "../tasks/index.js"; +import { stopCauseLine, type TaskRecord } from "../tasks/index.js"; const SUMMARY_MAX = 100; @@ -81,10 +81,13 @@ export function formatOvernightSummary(input: OvernightSummaryInput): string { const spent = formatUsd(task.spentUsd ?? 0); const lines = [ "", - `overnight ${task.status}: ${task.id}`, + `overnight ${overnightOutcome(task)}: ${task.id}`, ` time: ${formatElapsed(input.elapsedMs)}`, ` spent: ${input.capUsd > 0 ? `${spent} of ${formatUsd(input.capUsd)}` : spent} (estimate; local models count as $0)`, ]; + if (stoppedByLimit(task)) { + lines.push(` ${stopCauseLine(task.stopCause!)}`); + } if (input.deniedApprovals > 0) { lines.push( ` denied ${input.deniedApprovals} action(s) that needed approval; rerun with --no-approval to allow them`, @@ -115,6 +118,28 @@ export function formatDetachedNotice(taskId: string, logPath: string, dir: strin ].join("\n"); } -export function overnightExitCode(status: TaskRecord["status"]): number { - return status === "completed" ? 0 : 1; +/** + * The store calls a run a ceiling ended `completed` and keeps the cause + * beside it (ATO-1323). For the person or script reading the result + * that is not "done": the work may be unfinished. + */ +function stoppedByLimit(task: Pick): boolean { + return task.status === "completed" && typeof task.stopCause === "string" && task.stopCause.length > 0; +} + +/** `completed`, `stopped (cost_ceiling)`, `failed`, ... */ +export function overnightOutcome(task: Pick): string { + return stoppedByLimit(task) ? `stopped (${task.stopCause})` : task.status; +} + +/** Exit code when a step, time or money limit (or no progress) ended the run. */ +export const OVERNIGHT_STOPPED_EXIT_CODE = 2; + +/** + * 0 only when the agent finished on its own; 2 when a limit stopped it + * (the run itself worked, the task may not be done); 1 otherwise. + */ +export function overnightExitCode(task: Pick): number { + if (stoppedByLimit(task)) return OVERNIGHT_STOPPED_EXIT_CODE; + return task.status === "completed" ? 0 : 1; } From 8f28f45d3f9aa195ead8f74593573ffa1bbc8954 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:10:18 +0300 Subject: [PATCH 012/155] fix(models): leave Atomic Core alone when an update is refused for a busy engine `models update` retired an idle Core after every failure. When the refusal was "Another engine operation is running", nothing had been installed or started by the command, and the lock's holder (Studio putting Core in place, a model load) may have just started a Core it was about to use. Retiring it then made that caller's next request fail and start Core again. The desktop's retry loop for a held lock (#1105) makes this window come up more often. A busy-lock refusal now prints its line and returns 1 without touching Core. Every other failure still retires an idle Core as before. ATO-1348 --- src/cli/models-handlers.ts | 10 +++ src/cli/models-update-busy.test.ts | 105 +++++++++++++++++++++++++++ src/local-llm/core/core-operation.ts | 10 +++ 3 files changed, 125 insertions(+) create mode 100644 src/cli/models-update-busy.test.ts diff --git a/src/cli/models-handlers.ts b/src/cli/models-handlers.ts index 25c394a97..0e2d6989f 100644 --- a/src/cli/models-handlers.ts +++ b/src/cli/models-handlers.ts @@ -79,6 +79,7 @@ import { } from "../local-llm/index.js"; import { CORE_FALLBACK_LINE, CoreUnavailableError, shouldFallBackFromCore, type CoreFailureInfo } from "../local-llm/core/core-errors.js"; import { retireIdleCore } from "../local-llm/core/core-sessions.js"; +import { isCoreOperationBusy } from "../local-llm/core/core-operation.js"; import { activeCoreVersion, CORE_VERSION } from "../local-llm/core/core-state.js"; import { selectManagedEngine } from "../local-llm/engine-selection.js"; import { persistManagedEngine } from "./models-engine.js"; @@ -1519,6 +1520,15 @@ export async function runLocalModelsUpdate(signal?: AbortSignal): Promise ({ + ...(await vi.importActual("../local-llm/index.js")), + checkForBackendUpdate: vi.fn(async () => ({ updateAvailable: true, latestTag: "core-test", currentTag: null })), + downloadBackend: vi.fn(), +})); + +vi.mock("../local-llm/core/core-sessions.js", async () => ({ + ...(await vi.importActual("../local-llm/core/core-sessions.js")), + retireIdleCore: vi.fn(async () => false), +})); + +vi.mock("../analytics/index.js", async () => ({ + ...(await vi.importActual("../analytics/index.js")), + reportLocalEngineFailed: vi.fn(async () => {}), +})); + +import { getUserConfigPath, writeUserConfigFileSync } from "../config/config-file.js"; +import { USER_CONFIG_DEFAULTS } from "../config/config-schema.js"; +import { getConfig, resetConfigCache } from "../config/index.js"; +import * as localLlm from "../local-llm/index.js"; +import { retireIdleCore } from "../local-llm/core/core-sessions.js"; +import { isCoreOperationBusy, withCoreOperation } from "../local-llm/core/core-operation.js"; +import { coreRoot } from "../local-llm/core/core-state.js"; +import { runLocalModelsUpdate } from "./models-handlers.js"; + +describe("runLocalModelsUpdate when the engine is busy (ATO-1348)", () => { + let stateDir: string; + let stderrChunks: string[]; + + beforeEach(() => { + stateDir = mkdtempSync(join(tmpdir(), "atomic-models-update-busy-")); + process.env.ATOMIC_AGENT_STATE_DIR = stateDir; + resetConfigCache(); + writeUserConfigFileSync(getUserConfigPath(stateDir), { + ...USER_CONFIG_DEFAULTS, + localModels: { + ...USER_CONFIG_DEFAULTS.localModels, + mode: "managed", + managed: { ...USER_CONFIG_DEFAULTS.localModels.managed, engine: "atomic-core" }, + }, + }); + resetConfigCache(); + stderrChunks = []; + vi.spyOn(process.stdout, "write").mockImplementation(() => true); + vi.spyOn(process.stderr, "write").mockImplementation((chunk: unknown) => { + stderrChunks.push(typeof chunk === "string" ? chunk : String(chunk)); + return true; + }); + vi.mocked(retireIdleCore).mockClear(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + rmSync(stateDir, { recursive: true, force: true }); + delete process.env.ATOMIC_AGENT_STATE_DIR; + resetConfigCache(); + }); + + /** The refusal exactly as the install gets it: a live process (this one) holds the lock. */ + async function realBusyRefusal(): Promise { + const root = coreRoot(getConfig().paths.localModelsDataDir); + mkdirSync(root, { recursive: true }); + writeFileSync(join(root, "operation.lock"), String(process.pid)); + const err = await withCoreOperation(getConfig().paths.localModelsDataDir, async () => undefined, { waitMs: 0 }).then( + () => null, + (e: unknown) => e, + ); + rmSync(join(root, "operation.lock"), { force: true }); + return err; + } + + it("leaves Core running and says why", async () => { + const busy = await realBusyRefusal(); + expect(isCoreOperationBusy(busy)).toBe(true); + vi.mocked(localLlm.downloadBackend).mockRejectedValueOnce(busy); + + await expect(runLocalModelsUpdate()).resolves.toBe(1); + expect(retireIdleCore).not.toHaveBeenCalled(); + expect(stderrChunks.join("")).toMatch(/Another engine operation is running/); + }); + + it("still retires an idle Core after any other failure", async () => { + vi.mocked(localLlm.downloadBackend).mockRejectedValueOnce(new Error("disk full")); + + await expect(runLocalModelsUpdate()).resolves.toBe(1); + expect(retireIdleCore).toHaveBeenCalledTimes(1); + }); + + it("does not take an ordinary error for a busy engine", () => { + expect(isCoreOperationBusy(new Error("Another engine operation is running. Wait for it to finish."))).toBe(false); + }); +}); diff --git a/src/local-llm/core/core-operation.ts b/src/local-llm/core/core-operation.ts index 239da684e..afd6dd666 100644 --- a/src/local-llm/core/core-operation.ts +++ b/src/local-llm/core/core-operation.ts @@ -16,6 +16,16 @@ class CoreOperationBusyError extends Error { override readonly name = "CoreOperationBusyError"; } +/** + * The engine operation lock was held by another live process: the refusal + * says nothing about Core itself. ATO-1348: a command refused this way must + * not then tidy up a Core the lock's owner (Studio, a model load) may have + * just started and still needs. + */ +export function isCoreOperationBusy(err: unknown): boolean { + return err instanceof CoreOperationBusyError; +} + function ownerIsDead(path: string): boolean { let pid: number; try { pid = Number(readFileSync(path, "utf8")); } catch { return false; } From 8b8be8ce55cb72baee0d91e24e191de9daf643e2 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:10:27 +0300 Subject: [PATCH 013/155] Config: keep a newer build's agent.prompt.compact value on disk A compact prompt mode this build does not know was read as auto, and the next whole-file write (any settings toggle) put auto over the value the newer build wrote. The key now joins the other set-aside enum settings: read as auto with one notice, kept on disk, refused when typed in config set. --- src/config/agent/prompt-config.ts | 10 ++++------ src/config/docs/compatibility.md | 2 +- src/config/keep-set-aside-values.test.ts | 1 + src/config/keep-set-aside-values.ts | 2 ++ 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/config/agent/prompt-config.ts b/src/config/agent/prompt-config.ts index 30b17ac91..fef98326b 100644 --- a/src/config/agent/prompt-config.ts +++ b/src/config/agent/prompt-config.ts @@ -1,5 +1,5 @@ import { ConfigValidationError } from "../config-validation-error.js"; -import { emitConfigNoticeOnce } from "../config-primitives.js"; +import { isSetAsideString } from "../config-primitives.js"; /** * `agent.prompt.compact`: whether the stable prefix uses the compact @@ -23,17 +23,15 @@ export function createPromptDefaults(): PromptConfig { * Lenient on an unknown string, strict on a wrong type: a newer build * may write a mode this build does not know (the desktop app and the * terminal share one config file), and that must read as the default - * rather than fail every command. Same rule as #1020's - * `isSetAsideString`; switch to that helper once it is on main. + * rather than fail every command. A whole-file write keeps the value on + * disk (`SET_ASIDE_ENUM_KEYS` in `keep-set-aside-values.ts`, ATO-1330). */ function parseCompactPromptMode(raw: unknown, fallback: CompactPromptMode): CompactPromptMode { if (typeof raw !== "string") { throw new ConfigValidationError("agent.prompt.compact", `expected one of ${COMPACT_PROMPT_MODES.join(", ")}`); } if ((COMPACT_PROMPT_MODES as readonly string[]).includes(raw)) return raw as CompactPromptMode; - emitConfigNoticeOnce( - `config: agent.prompt.compact "${raw}" is not a value this build knows; using "${fallback}"`, - ); + isSetAsideString(raw, COMPACT_PROMPT_MODES, "agent.prompt.compact", fallback); return fallback; } diff --git a/src/config/docs/compatibility.md b/src/config/docs/compatibility.md index 3be1595d4..e14087b12 100644 --- a/src/config/docs/compatibility.md +++ b/src/config/docs/compatibility.md @@ -25,7 +25,7 @@ Version 77 makes `atomic-core` the default `localModels.managed.engine`. Before ## Values a newer build wrote -The desktop app and the terminal share one state dir and one `config.json`, and either can be the newer build. An enum-like value this build does not know is not a broken file: it is read as the default (or unset) with one notice per process, kept on disk by a whole-file write as long as the in-memory value is still the one read from it, and refused when typed in `config set`. This covers `localModels.managed.modelId`, `engine` and `backendVariant` (ATO-1239), and `llm.toolTransport`, `llm.runMode.mode`, `llm.runMode.fusion.workerReasoning`, `escalation`, `judge` and `isolation`, provider `modelMode` / `modelModes` values, `agent.compaction.mode`, `agent.tools.search` and `tui.whileBusySubmit` (ATO-1241). A key under `llm.runMode.fusion.roles` that is not a role here is skipped the same way; numeric bounds stay strict. A provider entry whose `kind` or `subscriptionCli.cli` is unknown is left out of the parsed config, and every reference to it (active providers, fallback chain, fusion leg pins with their model labels, escalation, judge and role pins) reads as unpinned; a whole-file write puts the entry and the references back. A value of the wrong type, and an id that names no provider at all, still fail validation. [Read side](../config-primitives.ts) (`isSetAsideString`), [provider entries](../llm-set-aside-providers.ts), [write side](../keep-set-aside-values.ts). +The desktop app and the terminal share one state dir and one `config.json`, and either can be the newer build. An enum-like value this build does not know is not a broken file: it is read as the default (or unset) with one notice per process, kept on disk by a whole-file write as long as the in-memory value is still the one read from it, and refused when typed in `config set`. This covers `localModels.managed.modelId`, `engine` and `backendVariant` (ATO-1239), and `llm.toolTransport`, `llm.runMode.mode`, `llm.runMode.fusion.workerReasoning`, `escalation`, `judge` and `isolation`, provider `modelMode` / `modelModes` values, `agent.compaction.mode`, `agent.tools.search` and `tui.whileBusySubmit` (ATO-1241), and `agent.prompt.compact` (ATO-1330). A key under `llm.runMode.fusion.roles` that is not a role here is skipped the same way; numeric bounds stay strict. A provider entry whose `kind` or `subscriptionCli.cli` is unknown is left out of the parsed config, and every reference to it (active providers, fallback chain, fusion leg pins with their model labels, escalation, judge and role pins) reads as unpinned; a whole-file write puts the entry and the references back. A value of the wrong type, and an id that names no provider at all, still fail validation. [Read side](../config-primitives.ts) (`isSetAsideString`), [provider entries](../llm-set-aside-providers.ts), [write side](../keep-set-aside-values.ts). ## Terminal chat step folding diff --git a/src/config/keep-set-aside-values.test.ts b/src/config/keep-set-aside-values.test.ts index 0204df74a..a4bddcd0b 100644 --- a/src/config/keep-set-aside-values.test.ts +++ b/src/config/keep-set-aside-values.test.ts @@ -74,6 +74,7 @@ describe("enum-like values a newer build wrote (ATO-1241)", () => { { key: "llm.runMode.fusion.isolation", future: "future-isolation-1241", reads: undefined, chosen: "worktree" }, { key: "agent.tools.search", future: "future-search-1241", reads: "auto", chosen: "on" }, { key: "agent.compaction.mode", future: "future-compaction-1241", reads: "auto", chosen: "model" }, + { key: "agent.prompt.compact", future: "future-compact-1330", reads: "auto", chosen: "on" }, { key: "tui.whileBusySubmit", future: "future-submit-1241", reads: "steer", chosen: "queue" }, ]; diff --git a/src/config/keep-set-aside-values.ts b/src/config/keep-set-aside-values.ts index 12d309296..79f772b8c 100644 --- a/src/config/keep-set-aside-values.ts +++ b/src/config/keep-set-aside-values.ts @@ -2,6 +2,7 @@ import { isDeepStrictEqual } from "node:util"; import { TOOL_SEARCH_MODES } from "./agent/agent-parser.js"; import { COMPACTION_MODES } from "./agent/compaction-config.js"; +import { COMPACT_PROMPT_MODES } from "./agent/prompt-config.js"; import { parseUserConfigFile, type UserConfigFile } from "./config-schema.js"; import { LLM_TOOL_TRANSPORTS } from "./llm-config.js"; import { @@ -32,6 +33,7 @@ export const SET_ASIDE_ENUM_KEYS: ReadonlyMap = new M ["llm.runMode.fusion.isolation", FUSION_ISOLATION_MODES], ["agent.tools.search", TOOL_SEARCH_MODES], ["agent.compaction.mode", COMPACTION_MODES], + ["agent.prompt.compact", COMPACT_PROMPT_MODES], ["tui.whileBusySubmit", WHILE_BUSY_SUBMIT_MODES], ]); From 147c1dd608c73ec07e6519c95bc244e0524bef08 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:11:13 +0300 Subject: [PATCH 014/155] fix(fs): search the working folder before crowded parent folders on a missing path The missing-path hint searched the working folder and the folders above it under one shared entry budget. A crowded parent (the system temp folder, ~20,000 entries) used it up, so a match below the working folder was never reached and the answer said the quick search found nothing. Roots above the session are now marked outer and searched after the near ones, under their own small budget and a per-folder cap; their matches are listed after the near ones. Fixes ATO-1333. --- src/tools/os/fs/README.md | 2 +- src/tools/os/fs/fs-missing-path.test.ts | 16 ++++- src/tools/os/fs/path-candidate-roots.ts | 19 ++++-- src/tools/os/fs/path-candidates.test.ts | 39 ++++++++++++ src/tools/os/fs/path-candidates.ts | 81 ++++++++++++++++++++----- 5 files changed, 133 insertions(+), 24 deletions(-) diff --git a/src/tools/os/fs/README.md b/src/tools/os/fs/README.md index af9e69c87..a436a0803 100644 --- a/src/tools/os/fs/README.md +++ b/src/tools/os/fs/README.md @@ -7,7 +7,7 @@ This area implements filesystem operations and the guards used by their mutation ## Operations and entry points -- Read and discover: [read](fs-read.ts), [list](fs-list.ts), [glob](fs-glob.ts), [grep](fs-grep.ts), [hash](fs-hash.ts), [diff](fs-diff.ts) and [watch](fs-watch.ts). The [read-coverage contract](fs-read-coverage.ts) is shared with the agent's repeated-read detector: resolved identity, visible content digest and actual line coverage belong to the result, rather than being reconstructed from requested arguments. A path that exists but is not a regular file gets a [next-move error](fs-read-not-a-file.ts): a directory points at list, a device says there is nothing to read. A path that does not exist (read, list, hash, grep, and a glob root; ENOENT or ENOTDIR) gets a [next-move error](fs-missing-path.ts) instead of a bare ENOENT: [a bounded search](path-candidates.ts) by name (ignoring case), names only and under ~300 ms, in [roots near the session](path-candidate-roots.ts): the working folder, user-named paths and the nearest existing ancestor of the missing path with a level or two above it, never the whole home. macOS privacy-guarded folders (Desktop, Documents, Downloads, Pictures, Movies, Music) are listed only when the missing path, the working folder or a named path is already inside them, so no permission prompt appears. The candidates are offered as advice (the model may only have been checking whether the path exists); several fit, ask the user; a match outside the read scope is marked and reading it will ask; dot-names outside the scope are never offered; a fusion worker is told the path is missing and searches nowhere. The result carries `errorCode` and `path` for the log. Glob walks in-process under a [walk budget](fs-walk-budget.ts) (wall clock and scanned entries) and returns partial results with a leading note when it runs out; its [pattern compiler](fs-glob-pattern.ts) lives beside it. Grep is bounded by ripgrep's own timeout instead, and [keeps the matches found before it](fs-grep-timeout.ts) with the same kind of leading note; its [argument parsing](fs-grep-args.ts), [ripgrep JSON parser](fs-grep-ripgrep-json.ts) and [result shaping](fs-grep-format.ts) live beside it. +- Read and discover: [read](fs-read.ts), [list](fs-list.ts), [glob](fs-glob.ts), [grep](fs-grep.ts), [hash](fs-hash.ts), [diff](fs-diff.ts) and [watch](fs-watch.ts). The [read-coverage contract](fs-read-coverage.ts) is shared with the agent's repeated-read detector: resolved identity, visible content digest and actual line coverage belong to the result, rather than being reconstructed from requested arguments. A path that exists but is not a regular file gets a [next-move error](fs-read-not-a-file.ts): a directory points at list, a device says there is nothing to read. A path that does not exist (read, list, hash, grep, and a glob root; ENOENT or ENOTDIR) gets a [next-move error](fs-missing-path.ts) instead of a bare ENOENT: [a bounded search](path-candidates.ts) by name (ignoring case), names only and under ~300 ms, in [roots near the session](path-candidate-roots.ts): the working folder, user-named paths and the nearest existing ancestor of the missing path with a level or two above it, never the whole home. Folders above the session are searched last under a small budget of their own (and a cap per folder), so a crowded parent such as the system temp folder cannot starve the search under the working folder, and their matches are listed after the near ones. macOS privacy-guarded folders (Desktop, Documents, Downloads, Pictures, Movies, Music) are listed only when the missing path, the working folder or a named path is already inside them, so no permission prompt appears. The candidates are offered as advice (the model may only have been checking whether the path exists); several fit, ask the user; a match outside the read scope is marked and reading it will ask; dot-names outside the scope are never offered; a fusion worker is told the path is missing and searches nowhere. The result carries `errorCode` and `path` for the log. Glob walks in-process under a [walk budget](fs-walk-budget.ts) (wall clock and scanned entries) and returns partial results with a leading note when it runs out; its [pattern compiler](fs-glob-pattern.ts) lives beside it. Grep is bounded by ripgrep's own timeout instead, and [keeps the matches found before it](fs-grep-timeout.ts) with the same kind of leading note; its [argument parsing](fs-grep-args.ts), [ripgrep JSON parser](fs-grep-ripgrep-json.ts) and [result shaping](fs-grep-format.ts) live beside it. - Mutate and recover: [write](fs-write.ts), [edit](fs-edit.ts), [patch](fs-patch.ts), [move](fs-move.ts), [trash](fs-trash.ts) and [restore](fs-restore.ts). Edit diff and patch preview helpers keep rendering/dry-run reporting beside the operation. When an edit's `oldString` is not found exactly, [the tolerant chain](fs-edit-tolerant.ts) tries, strictest first, blank lines and trailing whitespace ([the edit matcher](fs-edit-match.ts)), trimmed boundaries, shifted or differing indentation, collapsed inner whitespace, backslash escapes and typographic quotes ([folding](fs-edit-folding.ts)), then a [block anchored](fs-edit-block-anchor.ts) on its first and last lines. A level applies only to exactly one span and the result names it in a leading note; an ambiguous level is refused and stops the chain, `newString` goes in as sent (on the indentation level it is shifted by the same indentation), and no match returns the nearest fragment of the file as JSON. Restore copies precede qualifying replacements; advisory parse/content warnings do not invent an atomic rollback. - Move/rename: [the planner](fs-move-plan.ts) checks every item before the single approval (source present, no silent replacement, no folder into itself, no order-dependent batch), then [the move](fs-move.ts) renames in order with a copy-and-remove fallback across volumes and creates missing parent folders. A `to` ending in a separator, or naming an existing folder, means "inside it". Undo is the reversed move list in the result; a file replaced with `overwrite: true` is saved to the restore store first. Sources count as read targets for read scope, since a move could otherwise carry an outside file into the workspace, and a moved folder that holds the agent's config or `.env` is classified `trust_config`. Across volumes the copy lands under a temporary name beside the destination and is renamed into place, so a failed copy never touches the file being replaced. [Card and result wording](fs-move-text.ts) lives beside it. [Alias shapes](fs-move-aliases.ts) small models send (`paths`/`sources`/`src`/`source` with `destination`/`dest`/`target`) are read as `from`/`to` or `moves` by the registry before any wrapper, read scope included, sees the call, and the result ends with a one-line note; a list alias reads its destination as a folder unless it looks like a file name, and an alias beside a canonical key is still refused. - Locate a project: [location](fs-locate-project.ts) and [known sources](fs-locate-project-sources.ts) search the working directory and ancestors, recent session directories and direct children of configured project roots. Ambiguity remains an ambiguity; the operation does not crawl the whole disk. [Project test helpers](fs-locate-project-test-helpers.ts) are fixtures, not a runtime API. diff --git a/src/tools/os/fs/fs-missing-path.test.ts b/src/tools/os/fs/fs-missing-path.test.ts index be9b423e0..e2aa2f522 100644 --- a/src/tools/os/fs/fs-missing-path.test.ts +++ b/src/tools/os/fs/fs-missing-path.test.ts @@ -128,7 +128,21 @@ describe("candidateRoots", () => { exists, }); expect(plan.blocked).not.toContain("/Users/u/Desktop"); - expect(plan.roots).toContainEqual({ dir: "/Users/u/Desktop", depth: 3 }); + expect(plan.roots).toContainEqual({ dir: "/Users/u/Desktop", depth: 3, outer: true }); + }); + + // ATO-1333: a crowded parent must not use up the working folder's budget. + it("marks folders above the working folder as outer, and the working folder as near", async () => { + const plan = await candidateRoots({ + missing: "/Users/u/work/projects/backlinks_demo", + workingDir: "/Users/u/work/projects", + home: "/Users/u/home", + platform: "darwin", + exists, + }); + expect(plan.roots).toContainEqual({ dir: "/Users/u/work/projects", depth: 3 }); + expect(plan.roots).toContainEqual({ dir: "/Users/u/work", depth: 1, outer: true }); + expect(plan.roots).toContainEqual({ dir: "/Users/u", depth: 1, outer: true }); }); }); diff --git a/src/tools/os/fs/path-candidate-roots.ts b/src/tools/os/fs/path-candidate-roots.ts index c40981165..7e9f70e0a 100644 --- a/src/tools/os/fs/path-candidate-roots.ts +++ b/src/tools/os/fs/path-candidate-roots.ts @@ -14,6 +14,11 @@ import { isInside } from "../../../approval/fanout-scope.js"; * So such a folder is searched, or even descended into, only when the * missing path, the working folder or a user-named path is already * inside it; otherwise it is blocked. + * + * A root that is not inside the working folder or a user-named path (the + * missing path's ancestors, a protected folder above the session) is + * marked `outer`: it is searched after the others, under a small budget + * of its own, so a crowded one cannot starve the search near the session. */ /** One directory to search and how many levels below it to look. */ @@ -21,6 +26,8 @@ export interface CandidateRoot { dir: string; /** 1 = the root's direct children only. */ depth: number; + /** Above the session: searched last, under its own small budget. */ + outer?: boolean; } export interface CandidatePlan { @@ -78,6 +85,10 @@ export async function candidateRoots( const { searchable, guarded } = protectedFolders(input); const blocked = [...searchable, ...guarded].filter((dir) => !qualifies(dir)); + const near = [resolve(input.workingDir), ...named]; + const placed = (dir: string, depth: number): CandidateRoot => + near.some((root) => isInside(root, dir)) ? { dir, depth } : { dir, depth, outer: true }; + const roots: CandidateRoot[] = [ { dir: resolve(input.workingDir), depth: WORKING_DEPTH }, ...named.map((dir) => ({ dir, depth: NAMED_DEPTH })), @@ -85,16 +96,12 @@ export async function candidateRoots( const ancestor = await nearestExisting(dirname(missing), exists); if (ancestor !== null) { const parent = dirname(ancestor); - roots.push( - { dir: ancestor, depth: 2 }, - { dir: parent, depth: 1 }, - { dir: dirname(parent), depth: 1 }, - ); + roots.push(placed(ancestor, 2), placed(parent, 1), placed(dirname(parent), 1)); } roots.push( ...searchable .filter((dir) => qualifies(dir)) - .map((dir) => ({ dir, depth: PROTECTED_DEPTH })), + .map((dir) => placed(dir, PROTECTED_DEPTH)), ); return { roots: roots.filter(({ dir }) => dir !== home && !isFilesystemRoot(dir)), diff --git a/src/tools/os/fs/path-candidates.test.ts b/src/tools/os/fs/path-candidates.test.ts index 9bc235e36..a2f4076ef 100644 --- a/src/tools/os/fs/path-candidates.test.ts +++ b/src/tools/os/fs/path-candidates.test.ts @@ -101,6 +101,45 @@ describe("findPathCandidates", () => { expect(found).toHaveLength(3); }); + // ATO-1333: the working folder in the system temp folder, whose ~20,000 + // entries used to spend the whole shared budget before the search + // reached a match two levels below the working folder. + it("a crowded outer root cannot starve the near search", async () => { + await mkdir(join(base, "work", "projects", "target"), { recursive: true }); + await mkdir(join(base, "crowd"), { recursive: true }); + for (let i = 0; i < 40; i++) { + await writeFile(join(base, "crowd", `entry-${i}`), "x"); + } + const search = await findPathCandidates( + join(base, "work", "target"), + [{ ...at("crowd", 1), outer: true }, at("work")], + { + ...DEFAULT_CANDIDATE_LIMITS, + maxEntries: 20, + maxOuterEntries: 10, + maxEntriesPerOuterDir: 10, + }, + ); + expect(search.found).toEqual([ + { path: join(base, "work", "projects", "target"), kind: "folder" }, + ]); + // The crowded folder was read only in part, so the search says it was cut. + expect(search.complete).toBe(false); + }); + + it("lists near matches before outer ones, however shallow", async () => { + await mkdir(join(base, "work", "a", "b", "same"), { recursive: true }); + await mkdir(join(base, "above", "same"), { recursive: true }); + const { found } = await findPathCandidates(join(base, "work", "same"), [ + { ...at("above", 1), outer: true }, + at("work"), + ]); + expect(found.map((c) => c.path)).toEqual([ + join(base, "work", "a", "b", "same"), + join(base, "above", "same"), + ]); + }); + it("never throws, and says when a budget cut the search short", async () => { await mkdir(join(base, "desk", "thing"), { recursive: true }); expect( diff --git a/src/tools/os/fs/path-candidates.ts b/src/tools/os/fs/path-candidates.ts index 460784761..e432a27fa 100644 --- a/src/tools/os/fs/path-candidates.ts +++ b/src/tools/os/fs/path-candidates.ts @@ -1,5 +1,5 @@ import type { Dirent } from "node:fs"; -import { readdir } from "node:fs/promises"; +import { opendir, readdir } from "node:fs/promises"; import { basename, join, resolve } from "node:path"; import { isInside } from "../../../approval/fanout-scope.js"; @@ -23,6 +23,13 @@ export { candidateRoots, type CandidateRoot } from "./path-candidate-roots.js"; * hidden folders and known heavy trees, never descends into a folder the * roots marked as blocked (macOS privacy-protected folders the session * has not touched), does not follow symlinked folders, and never throws. + * + * Roots marked `outer` (folders above the session, such as the parent of + * a working folder in the system temp folder, which can hold ~20,000 + * entries) are searched only after the others, under their own small + * budget and a per-folder cap, so a crowded parent cannot use up the + * search near the session (ATO-1333). Their matches are listed after the + * near ones. */ export interface PathCandidate { @@ -40,8 +47,12 @@ export interface CandidateSearch { export interface PathCandidateLimits { /** Wall-clock budget for the whole search. */ timeBudgetMs: number; - /** Directory entries examined before giving up. */ + /** Directory entries examined under the near roots before giving up. */ maxEntries: number; + /** Directory entries examined under `outer` roots, after the near ones. */ + maxOuterEntries: number; + /** Entries read from any one folder under an `outer` root. */ + maxEntriesPerOuterDir: number; /** Matches collected before stopping (one more than will be shown). */ maxResults: number; } @@ -49,6 +60,8 @@ export interface PathCandidateLimits { export const DEFAULT_CANDIDATE_LIMITS: PathCandidateLimits = { timeBudgetMs: 300, maxEntries: 20_000, + maxOuterEntries: 2_000, + maxEntriesPerOuterDir: 500, maxResults: 6, }; @@ -91,8 +104,10 @@ export function nameKey(name: string): string { } /** - * Entries named like the last segment of `missing`, shallowest first, at - * most `limits.maxResults`. `missing` itself is never returned. + * Entries named like the last segment of `missing`, at most + * `limits.maxResults`: those under the near roots first, shallowest + * first, then those under `outer` roots. `missing` itself is never + * returned. */ export async function findPathCandidates( missing: string, @@ -115,7 +130,15 @@ export async function findPathCandidates( timer.unref?.(); }); const walk = new Walk(nameKey(name), resolve(missing), limits, deadline, state, found, filter); - const search = walk.run(roots).catch(() => { + const near = roots.filter((root) => root.outer !== true); + const outer = roots.filter((root) => root.outer === true); + const search = (async () => { + await walk.run(near, { maxEntries: limits.maxEntries }); + await walk.run(outer, { + maxEntries: limits.maxOuterEntries, + perDir: limits.maxEntriesPerOuterDir, + }); + })().catch(() => { // Best effort: whatever was found before the failure stands. }); try { @@ -134,13 +157,21 @@ interface WalkState { stopped: boolean; /** The time or entry budget ran out before the walk ended. */ cut: boolean; + /** Entries examined in the current pass. */ scanned: number; } +/** One pass's entry budget; `perDir` caps the entries read from each folder. */ +interface PassBudget { + maxEntries: number; + perDir?: number; +} + /** - * Breadth-first over every root at once, level by level, so a match one - * level below the nearest folder is found before a deep one elsewhere. - * A folder reached from two roots is listed once. + * Breadth-first over every root of a pass at once, level by level, so a + * match one level below the nearest folder is found before a deep one + * elsewhere. A folder reached from two roots, or in both passes, is + * listed once. */ class Walk { private readonly visited = new Set(); @@ -160,7 +191,8 @@ class Walk { this.blocked = (filter.blocked ?? []).map((dir) => resolve(dir)); } - async run(roots: readonly CandidateRoot[]): Promise { + async run(roots: readonly CandidateRoot[], budget: PassBudget): Promise { + this.state.scanned = 0; let level = roots .map((root) => ({ dir: resolve(root.dir), left: root.depth })) .filter(({ dir }) => !this.isBlocked(dir)); @@ -169,9 +201,10 @@ class Walk { for (const { dir, left } of level) { if (left <= 0 || this.visited.has(dir)) continue; this.visited.add(dir); - if (this.shouldStop()) return; - const entries = await listDir(dir); + if (this.shouldStop(budget)) return; + const { entries, more } = await listDir(dir, budget.perDir); if (this.state.stopped) return; + if (more) this.state.cut = true; for (const entry of entries) { this.state.scanned++; const path = join(dir, entry.name); @@ -201,9 +234,9 @@ class Walk { return this.found.length >= this.limits.maxResults; } - private shouldStop(): boolean { + private shouldStop(budget: PassBudget): boolean { if (this.state.stopped || this.full()) return true; - if (this.state.scanned >= this.limits.maxEntries || Date.now() >= this.deadline) { + if (this.state.scanned >= budget.maxEntries || Date.now() >= this.deadline) { this.state.cut = true; return true; } @@ -215,11 +248,27 @@ class Walk { } } -async function listDir(dir: string): Promise { +/** + * The entries of `dir`, or none when it cannot be listed. With `max`, at + * most that many are read and `more` says whether the folder held more. + */ +async function listDir( + dir: string, + max?: number, +): Promise<{ entries: Dirent[]; more: boolean }> { try { - return (await readdir(dir, { withFileTypes: true })) as Dirent[]; + if (max === undefined) { + return { entries: (await readdir(dir, { withFileTypes: true })) as Dirent[], more: false }; + } + const entries: Dirent[] = []; + // Breaking out of the loop closes the handle. + for await (const entry of await opendir(dir)) { + if (entries.length >= max) return { entries, more: true }; + entries.push(entry); + } + return { entries, more: false }; } catch { - return []; + return { entries: [], more: false }; } } From e5d8c319f283635a451b8b2e4b3df20cf2e38dcc Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:11:25 +0300 Subject: [PATCH 015/155] Resolve qwen-think from a strict ChatML think template under a neutral alias A fine-tune with the Qwen or Nemotron ChatML template served under a neutral alias fell to plain-instruct and lost its reasoning channel, because the think-tags profile required a family name in the alias or GGUF file name. The template alone now resolves qwen-think when it has ChatML markers, a / pair, a thinking control, and a generation prompt that opens the assistant turn with . GLM and Harmony templates that only mention stay plain, as does a ChatML template that only strips old think blocks from history. The "model profile resolved" log line now says what matched: template, name or fallback. ATO-1294 --- src/llm/docs/profiles.md | 2 +- src/llm/index.ts | 7 +- src/llm/model-profile.test.ts | 75 ++++++++++++++-- src/llm/model-profile.ts | 89 +++++++++++++++++-- .../composition/runtime-local-profile.ts | 6 +- 5 files changed, 162 insertions(+), 17 deletions(-) diff --git a/src/llm/docs/profiles.md b/src/llm/docs/profiles.md index b6ade2f8f..a8deb2e09 100644 --- a/src/llm/docs/profiles.md +++ b/src/llm/docs/profiles.md @@ -13,7 +13,7 @@ Thinking=off on a hand-built Qwen prompt uses its explicit disabled marker and p ## Detection -`detectModelProfile` reads `/props`. A template that writes the full Gemma 4 thinking framing (`<|turn>system`, `<|think|>`, `<|turn>model` and `<|channel>thought`) resolves `gemma4-think` whatever the model is called, before any name-gated branch: Gemma 4 fine-tunes ship under other names (Nemotron 3.5 Bolt E4B). The ChatML `` profiles stay name-gated (qwen/qwq/deepseek-r1, nemotron), because many unrelated templates mention ``. Names are the served alias plus the file name from `model_path`, so the managed daemon (catalog id alias) and an external server (path or custom alias) resolve the same GGUF alike. +`detectModelProfile` reads `/props`. A template that writes the full Gemma 4 thinking framing (`<|turn>system`, `<|think|>`, `<|turn>model` and `<|channel>thought`) resolves `gemma4-think` whatever the model is called, before any name-gated branch: Gemma 4 fine-tunes ship under other names (Nemotron 3.5 Bolt E4B). The ChatML `` profile is name-gated (qwen/qwq/deepseek-r1, nemotron) for loose template matches, because many unrelated templates mention ``. Without a name hint it still resolves `qwen-think` from a strict template signature: ChatML markers (`<|im_start|>`, `<|im_end|>`), a ``/`` pair, a thinking control (`enable_thinking`, `preserve_thinking` or the `supports_preserve_reasoning` cap), and a generation prompt that opens the assistant turn with ``. A fine-tune under a neutral alias keeps its reasoning channel; GLM or Harmony templates that only mention `` stay plain (ATO-1294). The boot line `model profile resolved` carries `matchedBy`: `template`, `name` or `fallback`. Names are the served alias plus the file name from `model_path`, so the managed daemon (catalog id alias) and an external server (path or custom alias) resolve the same GGUF alike. ## Bounds and wire contract diff --git a/src/llm/index.ts b/src/llm/index.ts index 0b554f6cb..899c5978c 100644 --- a/src/llm/index.ts +++ b/src/llm/index.ts @@ -29,12 +29,17 @@ export { SlotManager, hashPrefix, DEFAULT_SLOT_COUNT } from "./slot-manager.js"; export type { SlotAssignment } from "./slot-manager.js"; export { detectModelProfile, + detectModelProfileWithMatch, extractTotalSlots, GEMMA4_THINK_PROFILE, PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE, } from "./model-profile.js"; -export type { ModelProfile, ReasoningStyle } from "./model-profile.js"; +export type { + ModelProfile, + ModelProfileMatch, + ReasoningStyle, +} from "./model-profile.js"; export { ModelProfileManager } from "./model-profile-manager.js"; export type { ModelProfileManagerOptions, diff --git a/src/llm/model-profile.test.ts b/src/llm/model-profile.test.ts index 696faa248..8f53e2f75 100644 --- a/src/llm/model-profile.test.ts +++ b/src/llm/model-profile.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { detectModelProfile, + detectModelProfileWithMatch, extractTotalSlots, detectVisionSupport, GEMMA4_THINK_PROFILE, @@ -75,14 +76,33 @@ describe("detectModelProfile", () => { expect(profile.reasoningStyle).toBe("think-tags"); }); - // Pins the alias gate: the Nemotron detector must require a `nemotron` - // alias, not fire on the template markers alone. An alias carrying none of - // the qwen/qwq/deepseek-r1/nemotron hints must fall through to plain even - // though the template is a full ChatML + + enable_thinking match. - it("requires a nemotron alias — template markers alone do not classify", () => { + // ATO-1294. A fine-tune with the full ChatML + `` thinking + // template keeps its reasoning channel under a neutral alias: the + // template alone is enough when it is the exact framing the qwen + // profile reproduces. + it("resolves a neutral-alias ChatML + template from the template alone", () => { + for (const props of [NEMOTRON_PROPS, QWEN3_PROPS]) { + const detected = detectModelProfileWithMatch({ + ...props, + model_alias: "some-other-chatml-think-model", + }); + expect(detected.profile).toEqual(QWEN_THINK_PROFILE); + expect(detected.matchedBy).toBe("template"); + } + }); + + // The Nemotron gate still matters for a template that is not ChatML: + // without ChatML markers the template alone does not classify, and the + // `nemotron` name hint is what tips it onto the think-tags profile. + it("requires a nemotron alias when the template is not the ChatML framing", () => { + const nonChatMl = NEMOTRON_PROPS.chat_template.replaceAll( + "<|im_start|>", + "<|turn_start|>", + ); expect( detectModelProfile({ ...NEMOTRON_PROPS, + chat_template: nonChatMl, model_alias: "some-other-chatml-think-model", }), ).toEqual({ @@ -90,6 +110,45 @@ describe("detectModelProfile", () => { ...PLAIN_INSTRUCT_PROFILE, supportsThinkingSwitch: true, }); + const named = detectModelProfileWithMatch({ + ...NEMOTRON_PROPS, + chat_template: nonChatMl, + }); + expect(named.profile).toEqual(QWEN_THINK_PROFILE); + expect(named.matchedBy).toBe("name"); + }); + + it("keeps a ChatML template plain when the generation prompt does not open ", () => { + // Non-thinking ChatML fine-tune: reads `enable_thinking` and strips old + // think blocks from history, but never writes `` at the + // generation point. + const historyStripOnly = { + model_alias: "my-finetune", + chat_template: + "{%- for m in messages -%}{{- '<|im_start|>' + m['role'] + '\\n' -}}" + + "{%- set c = m['content'].split('')[-1] if '' in m['content'] else m['content'] -%}" + + "{{- c -}}{{- '<|im_end|>\\n' -}}{%- endfor -%}" + + "{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}" + + "{%- if enable_thinking is defined -%}{%- endif -%}", + }; + const detected = detectModelProfileWithMatch(historyStripOnly); + expect(detected.profile.id).toBe("plain-instruct"); + expect(detected.matchedBy).toBe("fallback"); + }); + + it("keeps a GLM-like template plain under a neutral alias", () => { + const glmLike = { + model_alias: "my-local-model", + chat_template: + "[gMASK]{% for m in messages %}<|{{ m['role'] }}|>\n{{ m['content'] }}{% endfor %}{% if add_generation_prompt %}<|assistant|>\n{% endif %}{% if enable_thinking is defined and not enable_thinking %}{% endif %}", + }; + expect(detectModelProfile(glmLike).id).toBe("plain-instruct"); + }); + + it("reports what matched: name for an alias-gated profile, fallback for plain", () => { + expect(detectModelProfileWithMatch(QWEN3_PROPS).matchedBy).toBe("name"); + expect(detectModelProfileWithMatch(LLAMA3_PROPS).matchedBy).toBe("fallback"); + expect(detectModelProfileWithMatch(GEMMA4_PROPS).matchedBy).toBe("template"); }); // Pins branch ordering. This alias satisfies BOTH gates (it contains @@ -207,6 +266,12 @@ describe("detectModelProfile", () => { expect( detectModelProfile({ ...NEMOTRON_PROPS, + // Not ChatML, so the template alone does not classify (ATO-1294) + // and only a name hint could — which the directory must not give. + chat_template: NEMOTRON_PROPS.chat_template.replaceAll( + "<|im_start|>", + "<|turn_start|>", + ), model_alias: "some-other-chatml-think-model", model_path: "/home/me/nemotron/llama-3.1-8b-instruct.gguf", }).id, diff --git a/src/llm/model-profile.ts b/src/llm/model-profile.ts index fc1f2e9ce..10b7aedd1 100644 --- a/src/llm/model-profile.ts +++ b/src/llm/model-profile.ts @@ -198,9 +198,31 @@ export const GEMMA4_THINK_PROFILE: TaggedReasoningModelProfile = { vision: VISION_ABSENT, }; +/** + * What decided the profile: `"template"` when the chat template alone + * identified the reasoning framing, `"name"` when an alias / GGUF file + * name hint was needed on top of the template, `"fallback"` when nothing + * matched and the model runs as `plain-instruct`. Logged on the + * "model profile resolved" line so a wrong profile can be traced to its + * cause from the boot log. + */ +export type ModelProfileMatch = "template" | "name" | "fallback"; + +export interface DetectedModelProfile { + profile: ModelProfile; + matchedBy: ModelProfileMatch; +} + export function detectModelProfile( props: Record, ): ModelProfile { + return detectModelProfileWithMatch(props).profile; +} + +/** `detectModelProfile` plus which signal decided it (see `ModelProfileMatch`). */ +export function detectModelProfileWithMatch( + props: Record, +): DetectedModelProfile { const modelNames = readModelNameHints(props); const chatTemplate = readString(props.chat_template); const templateLower = chatTemplate.toLowerCase(); @@ -208,7 +230,7 @@ export function detectModelProfile( const supportsPreserveReasoning = readBoolean(caps.supports_preserve_reasoning) ?? false; - const base = selectBaseProfile( + const { profile: base, matchedBy } = selectBaseProfile( modelNames, templateLower, supportsPreserveReasoning, @@ -222,8 +244,8 @@ export function detectModelProfile( : {}), vision, } as ModelProfile; - if (contextWindow === null) return enriched; - return { ...enriched, contextWindow }; + if (contextWindow === null) return { profile: enriched, matchedBy }; + return { profile: { ...enriched, contextWindow }, matchedBy }; } /** @@ -277,7 +299,7 @@ function selectBaseProfile( modelAlias: string, templateLower: string, supportsPreserveReasoning: boolean, -): ModelProfile { +): DetectedModelProfile { // The template is what the model was trained on; the alias is whatever // the operator (or our daemon) typed. A full Gemma 4 native thinking // template decides on its own, before any alias-gated branch: Gemma 4 @@ -286,7 +308,7 @@ function selectBaseProfile( // nemotron-named alias must not push a `<|turn>`-framed model onto the // ChatML `` prefill. if (hasGemma4NativeThinkTemplate(templateLower)) { - return GEMMA4_THINK_PROFILE; + return { profile: GEMMA4_THINK_PROFILE, matchedBy: "template" }; } if ( looksLikeQwenThinkModel( @@ -295,7 +317,7 @@ function selectBaseProfile( supportsPreserveReasoning, ) ) { - return QWEN_THINK_PROFILE; + return { profile: QWEN_THINK_PROFILE, matchedBy: "name" }; } // Nemotron needs its own detector but not its own profile. Its ChatML // template is qwen-shaped — ``/`` prefilled at the @@ -305,12 +327,21 @@ function selectBaseProfile( // alias, which Nemotron's does not match, so without this branch it would // fall through to `plain-instruct` and lose its reasoning channel. if (looksLikeNemotronThinkModel(modelAlias, templateLower)) { - return QWEN_THINK_PROFILE; + return { profile: QWEN_THINK_PROFILE, matchedBy: "name" }; + } + // No name hint, but the template itself is the ChatML + `` + // framing QWEN_THINK_PROFILE reproduces (ATO-1294): a Qwen or Nemotron + // fine-tune served under a neutral `--alias` / file name keeps its + // reasoning channel. The signature is strict on purpose — see + // `hasChatMlThinkTemplate` — so GLM-style or Harmony templates that + // merely mention `` stay plain. + if (hasChatMlThinkTemplate(templateLower, supportsPreserveReasoning)) { + return { profile: QWEN_THINK_PROFILE, matchedBy: "template" }; } if (looksLikeGemma4ThinkModel(modelAlias, templateLower)) { - return GEMMA4_THINK_PROFILE; + return { profile: GEMMA4_THINK_PROFILE, matchedBy: "name" }; } - return PLAIN_INSTRUCT_PROFILE; + return { profile: PLAIN_INSTRUCT_PROFILE, matchedBy: "fallback" }; } /** @@ -412,6 +443,46 @@ function hasGemma4NativeThinkTemplate(templateLower: string): boolean { ); } +/** + * The ChatML thinking template, recognised from the template alone + * (ATO-1294). Qwen 3 and Nemotron both ship it: ChatML turn markers + * (`<|im_start|>` and `<|im_end|>`), a ``/`` pair, a + * thinking control (`enable_thinking` / `preserve_thinking`, or the + * server's `supports_preserve_reasoning` cap), and — the strict part — a + * generation prompt that opens the assistant turn and writes `` + * itself. That last check reads only what follows the final + * `add_generation_prompt`, so a ChatML template that just strips old + * `` blocks from history (a non-thinking fine-tune) does not + * qualify. GLM's `<|assistant|>` framing and Harmony's + * `<|start|>`/`<|channel|>` framing carry no ChatML markers and never + * match. + */ +function hasChatMlThinkTemplate( + templateLower: string, + supportsPreserveReasoning: boolean, +): boolean { + if ( + !templateLower.includes("<|im_start|>") || + !templateLower.includes("<|im_end|>") || + !templateLower.includes("") || + !templateLower.includes("") + ) { + return false; + } + const hasThinkingControl = + templateLower.includes("enable_thinking") || + templateLower.includes("preserve_thinking") || + supportsPreserveReasoning; + if (!hasThinkingControl) return false; + const generationAt = templateLower.lastIndexOf("add_generation_prompt"); + if (generationAt < 0) return false; + const generation = templateLower.slice(generationAt); + return ( + generation.includes("<|im_start|>assistant") && + generation.includes("") + ); +} + /** * Lower-cased model names to match the alias hints against: the served * alias plus the GGUF file name from `/props.model_path`. The managed diff --git a/src/runtime/composition/runtime-local-profile.ts b/src/runtime/composition/runtime-local-profile.ts index 255eb3629..ae93cf910 100644 --- a/src/runtime/composition/runtime-local-profile.ts +++ b/src/runtime/composition/runtime-local-profile.ts @@ -7,6 +7,7 @@ import { LlamaServerClient } from "../../llm/llama-server-client.js"; import { buildGrammar, detectModelProfile, + detectModelProfileWithMatch, extractTotalSlots, ModelProfileManager, PLAIN_INSTRUCT_PROFILE, @@ -388,7 +389,7 @@ function logResolvedProfile( props: Record, logger: StructuredLogger, ): ResolvedModelProfile { - const resolved = detectModelProfile(props); + const { profile: resolved, matchedBy } = detectModelProfileWithMatch(props); const alias = typeof props.model_alias === "string" ? props.model_alias : null; const totalSlots = extractTotalSlots(props); @@ -401,6 +402,9 @@ function logResolvedProfile( ); logger.info("model profile resolved", { id: resolved.id, + // What decided the profile: the chat template alone, a model-name + // hint (alias / GGUF file name), or nothing (plain fallback). ATO-1294. + matchedBy, alias, contextWindow: resolved.contextWindow ?? null, totalSlots, From 92496be9ca1ecba94f78ded6f2897087e90e92c2 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:11:34 +0300 Subject: [PATCH 016/155] fix(llm): decode native tool names through the request's wire-name map (ATO-1365) nameEscape is not injective: mcp.srv.foo__bar and mcp.srv.foo.bar both escape to mcp__srv__foo__bar, and nameUnescape read it back as the dotted sibling, so a call to the first tool ran the second. The OpenAI adapter now assigns each descriptor a distinct wire name (names that round-trip keep their plain escape; a colliding name gets a deterministic _2/_3 suffix) and exposes a wire-to-canonical map built in the same memoized pass. Step parsing feeds it into toolCallsToBatch and the content-array recovery paths; nameUnescape stays the fallback for names the request did not offer. --- src/agent/step/step-parsing.test.ts | 28 ++++- src/agent/step/step-parsing.ts | 29 ++++- .../provider/adapters/tool-call-adapter.ts | 16 +++ .../openai/openai-tool-call-adapter.test.ts | 119 ++++++++++++++++++ .../openai/openai-tool-call-adapter.ts | 110 +++++++++++++++- 5 files changed, 293 insertions(+), 9 deletions(-) diff --git a/src/agent/step/step-parsing.test.ts b/src/agent/step/step-parsing.test.ts index 22b7e91e4..b28fdfba3 100644 --- a/src/agent/step/step-parsing.test.ts +++ b/src/agent/step/step-parsing.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; import { GEMMA4_THINK_PROFILE, PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE } from "../../llm/model-profile.js"; import { DEFAULT_REASONING_BUDGET_TOKENS } from "../../llm/reasoning-budget.js"; -import { REPAIR_MAX_TOKENS, repairReplyCap } from "./step-parsing.js"; +import type { CompletionResult } from "../../llm/llama-server-client.js"; +import type { ToolDescriptor } from "../../prompt/stable-prefix.js"; +import { REPAIR_MAX_TOKENS, repairReplyCap, tryParseToolCalls } from "./step-parsing.js"; describe("repair completion cap", () => { it("keeps the 1024 anti-loop guard for a model without thinking", () => { @@ -34,3 +36,27 @@ describe("repair completion cap", () => { expect(repairReplyCap("native_tools", 20_000, { profile: QWEN_THINK_PROFILE, budgetTokens: 1500 })).toBe(20_000); }); }); + +// ATO-1365: a native call is decoded against the tools the request +// offered, not by textual unescape, so `mcp.srv.foo__bar` never runs as +// its dotted sibling `mcp.srv.foo.bar`. +describe("native tool-call names", () => { + const tool = (name: string): ToolDescriptor => ({ name, tier: "frequent", summary: name, argsSchema: "" }); + const descriptors = [tool("mcp.srv.foo__bar"), tool("mcp.srv.foo.bar")]; + const native = (name: string): CompletionResult => + ({ content: "", reasoningContent: "", toolCalls: [{ id: "c1", type: "function", function: { name, arguments: "{}" } }] }) as unknown as CompletionResult; + const deps = { toolTransport: "native_tools" as const, toolCallAdapter: null }; + + it("resolves each wire name to the tool it was offered for", () => { + const first = tryParseToolCalls(native("mcp__srv__foo__bar_2"), PLAIN_INSTRUCT_PROFILE, deps, descriptors, false); + expect(first.ok && first.batch.calls[0]?.tool).toBe("mcp.srv.foo__bar"); + const second = tryParseToolCalls(native("mcp__srv__foo__bar"), PLAIN_INSTRUCT_PROFILE, deps, descriptors, false); + expect(second.ok && second.batch.calls[0]?.tool).toBe("mcp.srv.foo.bar"); + }); + + it("resolves an escaped name copied into content the same way", () => { + const content = ({ content: '[{"tool":"mcp__srv__foo__bar_2","args":{}}]', reasoningContent: "" }) as unknown as CompletionResult; + const parsed = tryParseToolCalls(content, PLAIN_INSTRUCT_PROFILE, deps, descriptors, false); + expect(parsed.ok && parsed.batch.calls[0]?.tool).toBe("mcp.srv.foo__bar"); + }); +}); diff --git a/src/agent/step/step-parsing.ts b/src/agent/step/step-parsing.ts index 903d88633..9dcb5d7a4 100644 --- a/src/agent/step/step-parsing.ts +++ b/src/agent/step/step-parsing.ts @@ -6,6 +6,7 @@ import type { ModelProfile } from "../../llm/model-profile.js"; import type { ToolDescriptor } from "../../prompt/stable-prefix.js"; import { completionAssumesOpenReasoning, resolveReasoning, normalizeContent, getReasoningTagOptions, memoizeText } from "./step-reasoning.js"; import { openAiToolCallAdapter } from "../../llm/provider/openai/openai-tool-call-adapter.js"; +import type { ToolCallAdapter } from "../../llm/provider/adapters/tool-call-adapter.js"; import { parseToolCalls, extractReasoning } from "../../llm/grammar/tool-call-grammar.js"; import { BatchValidationError } from "./step-errors.js"; import { getReasoningTurnFraming } from "../../llm/model-profile.js"; @@ -156,6 +157,22 @@ export function parseDepsFor( */ const REPAIR_JSON = { repairJson: true } as const; +/** + * The request's own wire-name -> tool-name map (ATO-1365), from the same + * adapter and descriptor list the request was built from. `nameUnescape` + * alone reads `mcp__srv__foo__bar` as `mcp.srv.foo.bar` even when the + * function was offered for `mcp.srv.foo__bar`, and would run the sibling. + */ +function requestWireNames( + adapter: ToolCallAdapter, + deps: ParseDeps, + toolDescriptors: readonly ToolDescriptor[], +): ReadonlyMap | undefined { + return adapter.wireToCanonical?.(toolDescriptors, { + strict: deps.strictTools === true, + }); +} + /** * Non-throwing parser wrapper. The step executor uses it to distinguish * a malformed first attempt (retryable) from any other error shape. @@ -195,11 +212,13 @@ export function tryParseToolCalls( deps.strictTools === true && adapter.strictWidenedArgs ? adapter.strictWidenedArgs(toolDescriptors, { strict: true }) : undefined; + const wireToCanonical = requestWireNames(adapter, deps, toolDescriptors); const batch = adapter.toolCallsToBatch( completion.toolCalls, reasoning, { ...(strictWidenedArgs ? { strictWidenedArgs } : {}), + ...(wireToCanonical ? { wireToCanonical } : {}), repairJson: true, }, ); @@ -245,9 +264,12 @@ export function tryParseToolCalls( // `registry.has(...)` resolves — `nameUnescape` is a no-op for // names that are already dotted, so this is idempotent. const adapter = deps.toolCallAdapter ?? openAiToolCallAdapter; + const wireToCanonical = requestWireNames(adapter, deps, toolDescriptors); const calls = grammarBatch.calls.map((call) => ({ ...call, - tool: adapter.nameUnescape(call.tool), + tool: + wireToCanonical?.get(call.tool) ?? + adapter.nameUnescape(call.tool), })); return { ok: true, batch: { ...grammarBatch, calls } }; } @@ -297,9 +319,12 @@ export function tryParseToolCalls( ); if (grammarBatch.calls.length > 0) { const adapter = deps.toolCallAdapter ?? openAiToolCallAdapter; + const wireToCanonical = requestWireNames(adapter, deps, toolDescriptors); const calls = grammarBatch.calls.map((call) => ({ ...call, - tool: adapter.nameUnescape(call.tool), + tool: + wireToCanonical?.get(call.tool) ?? + adapter.nameUnescape(call.tool), })); return { ok: true, batch: { ...grammarBatch, calls } }; } diff --git a/src/llm/provider/adapters/tool-call-adapter.ts b/src/llm/provider/adapters/tool-call-adapter.ts index 2b52bc6b9..dfff40b7e 100644 --- a/src/llm/provider/adapters/tool-call-adapter.ts +++ b/src/llm/provider/adapters/tool-call-adapter.ts @@ -34,6 +34,12 @@ export interface ToolBatchOptions { * report it on `ToolCallBatch.jsonRepair` (ATO-637). Off by default. */ repairJson?: boolean; + /** + * Wire function name -> qualified tool name, from `wireToCanonical` + * for the descriptors the request was built from. Consulted before + * `nameUnescape`, which is not injective (ATO-1365). + */ + wireToCanonical?: ReadonlyMap; } /** @@ -63,6 +69,16 @@ export interface ToolCallAdapter { descriptors: readonly ToolDescriptor[], options?: ToolDefinitionOptions, ): ReadonlyMap>; + /** + * For the same descriptors `descriptorsToTools` was called with: each + * emitted function name mapped to the qualified tool name it stands + * for. Fed back into `toolCallsToBatch` as `wireToCanonical`. An + * adapter whose escape is injective may omit this. + */ + wireToCanonical?( + descriptors: readonly ToolDescriptor[], + options?: ToolDefinitionOptions, + ): ReadonlyMap; /** Convert provider tool_calls into the runtime `ToolCallBatch`. */ toolCallsToBatch( toolCalls: ReadonlyArray, diff --git a/src/llm/provider/openai/openai-tool-call-adapter.test.ts b/src/llm/provider/openai/openai-tool-call-adapter.test.ts index bc07029fc..2f9f3a7d7 100644 --- a/src/llm/provider/openai/openai-tool-call-adapter.test.ts +++ b/src/llm/provider/openai/openai-tool-call-adapter.test.ts @@ -7,8 +7,12 @@ import { descriptorsToOpenAiTools, strictOpenAiWidenedArgs, openAiToolCallsToBatch, + openAiWireToCanonical, + withStrictNullArgumentDrop, + openAiToolCallAdapter, ToolCallArgumentsParseError, } from "./openai-tool-call-adapter.js"; +import type { ToolDescriptor } from "../../../prompt/stable-prefix.js"; describe("OpenAiToolCallAdapter", () => { it("round-trips dotted tool names via __ escape", () => { @@ -615,3 +619,118 @@ describe("OpenAiToolCallAdapter", () => { }); }); }); + +// ATO-1365: `nameEscape` is not injective, so the wire name alone cannot +// say which tool was meant. The request's own descriptor list does. +describe("wire names that escape alike", () => { + const tool = (name: string): ToolDescriptor => ({ + name, + tier: "frequent", + summary: `${name} summary`, + argsSchema: "", + }); + // The underscore pair is part of the MCP server's own tool name. + const underscored = tool("mcp.srv.foo__bar"); + const dotted = tool("mcp.srv.foo.bar"); + const wireNamesOf = (descriptors: readonly ToolDescriptor[]): string[] => + descriptorsToOpenAiTools(descriptors).map( + (t) => (t.function as { name: string }).name, + ); + + it("offers both tools under distinct wire names, whatever the order", () => { + for (const descriptors of [ + [underscored, dotted], + [dotted, underscored], + ]) { + const names = wireNamesOf(descriptors); + expect(names).toContain("mcp__srv__foo__bar"); + expect(names).toContain("mcp__srv__foo__bar_2"); + expect(new Set(names).size).toBe(names.length); + const map = openAiWireToCanonical(descriptors); + // The dotted tool keeps the spelling it always had. + expect(map.get("mcp__srv__foo__bar")).toBe("mcp.srv.foo.bar"); + expect(map.get("mcp__srv__foo__bar_2")).toBe("mcp.srv.foo__bar"); + } + }); + + it("routes a call to the underscored tool's wire name to that tool, not its dotted sibling", () => { + const descriptors = [underscored, dotted]; + const wireToCanonical = openAiWireToCanonical(descriptors); + const wire = [...wireToCanonical].find( + ([, canonical]) => canonical === "mcp.srv.foo__bar", + )?.[0]; + expect(wire).toBeDefined(); + expect(wireNamesOf(descriptors)).toContain(wire); + const batch = openAiToolCallsToBatch( + [{ function: { name: wire!, arguments: "{}" } }], + undefined, + { wireToCanonical }, + ); + expect(batch.calls[0]?.tool).toBe("mcp.srv.foo__bar"); + const sibling = openAiToolCallsToBatch( + [{ function: { name: "mcp__srv__foo__bar", arguments: "{}" } }], + undefined, + { wireToCanonical }, + ); + expect(sibling.calls[0]?.tool).toBe("mcp.srv.foo.bar"); + }); + + it("decodes an underscored tool through the map even with no sibling present", () => { + const descriptors = [underscored]; + expect(wireNamesOf(descriptors)).toContain("mcp__srv__foo__bar"); + const batch = openAiToolCallsToBatch( + [{ function: { name: "mcp__srv__foo__bar", arguments: "{}" } }], + undefined, + { wireToCanonical: openAiWireToCanonical(descriptors) }, + ); + // `nameUnescape` alone would say `mcp.srv.foo.bar`, a tool that + // does not exist. + expect(batch.calls[0]?.tool).toBe("mcp.srv.foo__bar"); + }); + + it("keeps a qualified name written as is, and a name the request did not offer still unescapes", () => { + const wireToCanonical = openAiWireToCanonical([underscored, dotted]); + const batch = openAiToolCallsToBatch( + [ + { function: { name: "mcp.srv.foo__bar", arguments: "{}" } }, + { function: { name: "os__fs__read", arguments: "{}" } }, + ], + undefined, + { wireToCanonical }, + ); + expect(batch.calls.map((c) => c.tool)).toEqual([ + "mcp.srv.foo__bar", + "os.fs.read", + ]); + }); + + it("a suffix never takes a spelling another tool would claim naturally", () => { + const names = wireNamesOf([ + underscored, + dotted, + tool("mcp.srv.foo.bar_2"), + ]); + expect(names).toContain("mcp__srv__foo__bar"); + expect(names).toContain("mcp__srv__foo__bar_2"); + expect(names).toContain("mcp__srv__foo__bar_3"); + expect(new Set(names).size).toBe(names.length); + const map = openAiWireToCanonical([ + underscored, + dotted, + tool("mcp.srv.foo.bar_2"), + ]); + expect(map.get("mcp__srv__foo__bar_2")).toBe("mcp.srv.foo.bar_2"); + expect(map.get("mcp__srv__foo__bar_3")).toBe("mcp.srv.foo__bar"); + }); + + it("the strict null-drop wrapper still passes the map through", () => { + const descriptors = [underscored, dotted]; + const wrapped = withStrictNullArgumentDrop(openAiToolCallAdapter); + const batch = wrapped.toolCallsToBatch( + [{ function: { name: "mcp__srv__foo__bar_2", arguments: "{}" } }], + undefined, + { wireToCanonical: openAiWireToCanonical(descriptors) }, + ); + expect(batch.calls[0]?.tool).toBe("mcp.srv.foo__bar"); + }); +}); diff --git a/src/llm/provider/openai/openai-tool-call-adapter.ts b/src/llm/provider/openai/openai-tool-call-adapter.ts index 7a344b70e..9a30865b2 100644 --- a/src/llm/provider/openai/openai-tool-call-adapter.ts +++ b/src/llm/provider/openai/openai-tool-call-adapter.ts @@ -107,6 +107,59 @@ interface BuiltFunctions { * maps to an empty set. */ widenedArgs: ReadonlyMap>; + /** + * Wire function name -> the qualified name it was emitted for, plus + * every qualified name mapped to itself where that spelling is not + * also somebody's wire name. See `assignWireNames`. + */ + wireToCanonical: ReadonlyMap; +} + +/** + * The function name each descriptor goes out under, chosen so that two + * distinct tools never share one (ATO-1365). + * + * `nameEscape` is not injective: `mcp.srv.foo__bar` (an MCP server's own + * tool name with an underscore pair in it) and `mcp.srv.foo.bar` both + * escape to `mcp__srv__foo__bar`, and `nameUnescape` reads that back as + * the second one. Decoding textually therefore ran the sibling tool when + * the model called the first. + * + * Names that round-trip through `nameUnescape` claim their plain escape + * first, so the common dotted tool keeps the spelling it always had and + * the outcome does not depend on descriptor order. A name that does not + * round-trip takes its plain escape only if it is still free, otherwise + * the first free `_2`, `_3`… suffix. A suffix never takes a spelling any + * other descriptor would naturally claim. + */ +function assignWireNames( + names: readonly string[], +): Map { + const unique = [...new Set(names)]; + const natural = new Set(unique.map(nameEscape)); + const taken = new Set(); + const wireOf = new Map(); + const roundTrips = (name: string): boolean => + nameUnescape(nameEscape(name)) === name; + for (const name of unique) { + if (!roundTrips(name)) continue; + const wire = nameEscape(name); + wireOf.set(name, wire); + taken.add(wire); + } + for (const name of unique) { + if (roundTrips(name)) continue; + let wire = nameEscape(name); + if (taken.has(wire)) { + const base = wire; + for (let n = 2; taken.has(wire) || natural.has(wire); n += 1) { + wire = `${base}_${n}`; + } + } + wireOf.set(name, wire); + taken.add(wire); + } + return wireOf; } /** @@ -124,8 +177,18 @@ function buildFunctions( const widenedArgs = new Map>(); const out: Record[] = []; const all = [...descriptors, ...replyFinishDescriptors()]; + const wireOf = assignWireNames(all.map((d) => d.name)); + const wireToCanonical = new Map(); + for (const [name, wire] of wireOf) wireToCanonical.set(wire, name); + // A qualified name the model writes as is (the text-JSON fallback, a + // local server that accepts dots) is itself, unless that exact string + // is another tool's wire name: then the wire reading wins, since that + // is the spelling the request offered. + for (const name of wireOf.keys()) { + if (!wireToCanonical.has(name)) wireToCanonical.set(name, name); + } for (const d of all) { - const escaped = nameEscape(d.name); + const escaped = wireOf.get(d.name)!; if (seen.has(escaped)) continue; seen.add(escaped); const parameters = descriptorToJsonSchema(d); @@ -140,7 +203,7 @@ function buildFunctions( }, }); } - return { tools: out, widenedArgs }; + return { tools: out, widenedArgs, wireToCanonical }; } /** @@ -225,6 +288,34 @@ export function strictOpenAiWidenedArgs( return buildFunctionsMemo(descriptors, options).widenedArgs; } +/** + * For the same descriptors `descriptorsToOpenAiTools` was called with: + * each emitted function name mapped back to the qualified tool name it + * stands for. `openAiToolCallsToBatch` decodes through this before it + * falls back to `nameUnescape`, which cannot tell `foo__bar` from + * `foo.bar` (ATO-1365). Independent of `strict`; the option is accepted + * only so the call shares the memo slot with the request build. + */ +export function openAiWireToCanonical( + descriptors: readonly ToolDescriptor[], + options?: ToolDefinitionOptions, +): ReadonlyMap { + return buildFunctionsMemo(descriptors, options).wireToCanonical; +} + +/** + * The qualified tool name a provider function name refers to: the + * request's own mapping when there is one, the textual unescape + * otherwise (a name the request did not offer, or a caller that has no + * descriptor list). + */ +export function decodeToolName( + providerName: string, + wireToCanonical?: ReadonlyMap, +): string { + return wireToCanonical?.get(providerName) ?? nameUnescape(providerName); +} + const EMPTY_WIDENED: ReadonlyMap> = new Map< string, ReadonlySet @@ -343,7 +434,7 @@ export function openAiToolCallsToBatch( } : undefined; for (const tc of toolCalls) { - const name = nameUnescape(tc.function.name); + const name = decodeToolName(tc.function.name, options?.wireToCanonical); let args: Record; try { args = parseArguments(tc.function.arguments, noteRepair); @@ -388,12 +479,18 @@ export function withStrictNullArgumentDrop( return { ...adapter, toolCallsToBatch: (toolCalls, reasoningText, options) => { - // Only the repair flag goes through: this wrapper already drops - // every top-level null, so the strict map stays unused as before. + // Only the repair flag and the name map go through: this wrapper + // already drops every top-level null, so the strict map stays + // unused as before. + const wireToCanonical = options?.wireToCanonical; + const forwarded: ToolBatchOptions = { + ...(options?.repairJson ? { repairJson: true } : {}), + ...(wireToCanonical ? { wireToCanonical } : {}), + }; const batch = adapter.toolCallsToBatch( toolCalls, reasoningText, - options?.repairJson ? { repairJson: true } : undefined, + Object.keys(forwarded).length > 0 ? forwarded : undefined, ); return { ...batch, @@ -411,5 +508,6 @@ export const openAiToolCallAdapter: ToolCallAdapter = { nameUnescape, descriptorsToTools: descriptorsToOpenAiTools, strictWidenedArgs: strictOpenAiWidenedArgs, + wireToCanonical: openAiWireToCanonical, toolCallsToBatch: openAiToolCallsToBatch, }; From 99d80ad1adf4f97bc8acf24f4141a00ecbc60e59 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:11:45 +0300 Subject: [PATCH 017/155] fix(trace): replay matches sessions recorded with search mode off trace replay handed the raw default catalog to the prefix builder. That catalog now holds tool.search, which the runtime mounts only in tool search mode, so every replay rendered the search-mode wording and a session recorded with the mode off reported drift on every step. Traces do not record the mode, so the replay now builds the prefix with and without it (applyToolSearch, as the runtime does) and a recorded hash is clean when it matches either, the same way it already treats transport and the compact scaffold. A new toolSearch option pins it; matchedToolSearch reports which variant matched. Refs ATO-1335 --- src/cli/trace-command.ts | 2 ++ src/replay/replay-session.test.ts | 48 +++++++++++++++++++++++++++++++ src/replay/replay-session.ts | 47 ++++++++++++++++++++++++------ 3 files changed, 88 insertions(+), 9 deletions(-) diff --git a/src/cli/trace-command.ts b/src/cli/trace-command.ts index 64590f1f3..2c3b55acd 100644 --- a/src/cli/trace-command.ts +++ b/src/cli/trace-command.ts @@ -158,6 +158,8 @@ async function handleReplay(args: string[]): Promise { browserChannel: config.browser.channel, }); + // The full catalog, `tool.search` included: the replay derives the + // search-mode-off and -on prompts from it itself (ATO-1335). const report = await replaySession({ path, context: { diff --git a/src/replay/replay-session.test.ts b/src/replay/replay-session.test.ts index 725993a46..1ded808ca 100644 --- a/src/replay/replay-session.test.ts +++ b/src/replay/replay-session.test.ts @@ -6,6 +6,7 @@ import { join } from "node:path"; import { buildStablePrefix } from "../prompt/stable-prefix.js"; import { hashPrefix } from "../llm/slot-manager.js"; import { PLAIN_INSTRUCT_PROFILE } from "../llm/index.js"; +import { applyToolSearch } from "../tools/tool-search/index.js"; import { replaySession } from "./replay-session.js"; @@ -241,4 +242,51 @@ describe("replaySession", () => { rmSync(fx.tmp, { recursive: true, force: true }); } }); + + it("matches a trace recorded with search mode off or on (ATO-1335)", async () => { + // The runtime mounts `tool.search` only in search mode, and traces do + // not record the mode: the replay gets the full catalog and must try + // the prefix with and without it. + const base = buildCurrentContext(); + const context = { + ...base, + toolDescriptors: [ + ...base.toolDescriptors, + { name: "os.git.show", summary: "show a commit", argsSchema: "{rev:string}" }, + { name: "tool.search", summary: "find a tool", argsSchema: "{query:string}" }, + ], + }; + const hashWith = (toolSearch: boolean): string => + hashPrefix( + buildStablePrefix({ + toolDescriptors: applyToolSearch(context.toolDescriptors, toolSearch), + capabilities: context.capabilities, + skillCatalog: context.skillCatalog, + reasoningSystemToken: context.profile.reasoningSystemToken, + }), + ); + expect(hashWith(false)).not.toBe(hashWith(true)); + + const off = writeTrace({ recordedHash: hashWith(false) }); + const on = writeTrace({ recordedHash: hashWith(true) }); + try { + const offReport = await replaySession({ path: off.path, context }); + expect(offReport.driftCount).toBe(0); + expect(offReport.steps.every((s) => s.matchedToolSearch === false)).toBe(true); + + const onReport = await replaySession({ path: on.path, context }); + expect(onReport.driftCount).toBe(0); + expect(onReport.steps.every((s) => s.matchedToolSearch === true)).toBe(true); + + const pinnedOn = await replaySession({ + path: off.path, + context: { ...context, toolSearch: true }, + }); + expect(pinnedOn.driftCount).toBe(2); + expect(pinnedOn.steps.every((s) => s.matchedToolSearch === null)).toBe(true); + } finally { + rmSync(off.tmp, { recursive: true, force: true }); + rmSync(on.tmp, { recursive: true, force: true }); + } + }); }); diff --git a/src/replay/replay-session.ts b/src/replay/replay-session.ts index c0a7104f1..9e7e03d59 100644 --- a/src/replay/replay-session.ts +++ b/src/replay/replay-session.ts @@ -7,11 +7,17 @@ import { buildStablePrefix } from "../prompt/stable-prefix.js"; import type { ModelProfile } from "../llm/model-profile.js"; import type { ToolCallTransport } from "../llm/provider/completion-types.js"; import { hashPrefix } from "../llm/slot-manager.js"; +import { applyToolSearch } from "../tools/tool-search/index.js"; import type { TraceEvent } from "../tracing/index.js"; import { iterateTraceFile } from "../cli/trace-file-io.js"; export interface ReplayContext { + /** + * The full catalog, before search mode (`tool.search` included). The + * replay derives what the prompt described from it with + * `applyToolSearch`, the way the runtime does. + */ toolDescriptors: readonly ToolDescriptor[]; capabilities: CapabilitiesSummary; skillCatalog: readonly SkillCatalogEntry[]; @@ -51,6 +57,16 @@ export interface ReplayContext { * `skillCatalogDropped` say: pass the catalog as the session saw it. */ compact?: boolean; + /** + * Whether the traced session ran in tool search mode (ATO-558, + * `agent.tools.search`): `tool.search` in the catalog, the rest as + * one-liners. The runtime mounts `tool.search` only then, and traces + * do not record the mode, so when omitted the replay builds the prefix + * with and without it and a recorded hash is clean when it matches + * either (ATO-1335). Before this a replay of a session with search mode + * off always carried the search-mode wording and reported drift. + */ + toolSearch?: boolean; } export interface ReplayStepReport { @@ -67,6 +83,8 @@ export interface ReplayStepReport { matchedTransport: ToolCallTransport | null; /** Whether the matched variant was the compact scaffold; `null` on drift. */ matchedCompact: boolean | null; + /** Whether the matched variant was in tool search mode; `null` on drift. */ + matchedToolSearch: boolean | null; drift: boolean; tokens: { recordedStablePrefix: number; @@ -98,9 +116,13 @@ export async function replaySession(options: { context: ReplayContext; }): Promise { const { path, context } = options; - const prefixFor = (transport: ToolCallTransport, compact: boolean): string => + const prefixFor = ( + transport: ToolCallTransport, + compact: boolean, + toolSearch: boolean, + ): string => buildStablePrefix({ - toolDescriptors: context.toolDescriptors, + toolDescriptors: applyToolSearch(context.toolDescriptors, toolSearch), capabilities: context.capabilities, skillCatalog: context.skillCatalog, ...(context.skillCatalogDropped !== undefined @@ -115,19 +137,25 @@ export async function replaySession(options: { }); // Candidate order matters only for the no-match `currentHash` fallback: // the first entry (pinned transport, else `"grammar"`; pinned compact, - // else the full scaffold) supplies it. + // else the full scaffold; pinned search mode, else search mode off) + // supplies it. const transports: readonly ToolCallTransport[] = context.toolTransport !== undefined ? [context.toolTransport] : ["grammar", "native_tools"]; const compactVariants: readonly boolean[] = context.compact !== undefined ? [context.compact] : [false, true]; - const candidates = compactVariants.flatMap((compact) => - transports.map((transport) => ({ - transport, - compact, - hash: hashPrefix(prefixFor(transport, compact)), - })), + const toolSearchVariants: readonly boolean[] = + context.toolSearch !== undefined ? [context.toolSearch] : [false, true]; + const candidates = toolSearchVariants.flatMap((toolSearch) => + compactVariants.flatMap((compact) => + transports.map((transport) => ({ + transport, + compact, + toolSearch, + hash: hashPrefix(prefixFor(transport, compact, toolSearch)), + })), + ), ); let sessionId = ""; @@ -151,6 +179,7 @@ export async function replaySession(options: { currentHash: matched !== null ? matched.hash : candidates[0]!.hash, matchedTransport: matched !== null ? matched.transport : null, matchedCompact: matched !== null ? matched.compact : null, + matchedToolSearch: matched !== null ? matched.toolSearch : null, drift: matched === null, tokens: { recordedStablePrefix: recorded.tokens.stablePrefix, From f73f2dd6411a2cd4e056d98731ae5aa5e3628944 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:11:54 +0300 Subject: [PATCH 018/155] fix(prompt): keep one descriptor each for os.git.push, commit and checkout (ATO-1296) The catalog carried two descriptors for each of the three git write tools. The full prompt showed both copies, and getToolDescriptorByName returned the second, so tool.view and error auto-expansion described os.git.commit without paths although the tool accepts it. Keep the copy that matches the implementation and its JSON Schema (commit with paths, push with setUpstream, checkout in schema order), fold in the accurate bits of the dropped copies (Remote sync note, branches only, the commit example), and drop the duplicate ledger from the conformance test so descriptor names must now be unique. --- docs/proposals/tool-catalog-deduplication.md | 6 ++- src/prompt/README.md | 2 +- src/prompt/default-tool-descriptors-a.ts | 33 +++---------- src/prompt/tool-descriptors.test.ts | 49 ++++++++++++++++++++ src/tools/docs/contracts.md | 2 +- src/tools/os/fs/docs/contracts.md | 2 +- src/tools/tool-contract-conformance.test.ts | 44 +++++------------- 7 files changed, 75 insertions(+), 63 deletions(-) create mode 100644 src/prompt/tool-descriptors.test.ts diff --git a/docs/proposals/tool-catalog-deduplication.md b/docs/proposals/tool-catalog-deduplication.md index fc4175934..40d45c329 100644 --- a/docs/proposals/tool-catalog-deduplication.md +++ b/docs/proposals/tool-catalog-deduplication.md @@ -1,6 +1,6 @@ # Remove existing Git tool descriptor duplicates -Status: proposed +Status: done (ATO-1296) Owner: src/prompt/ + src/tools/os/git/ Stage05l found 88 DEFAULT_TOOL_DESCRIPTORS entries for 85 names. os.git.checkout occurs at zero-based positions31/48, commit32/47 and push33/43. These copies predate the reorganization. [The conformance test](../../src/tools/tool-contract-conformance.test.ts) freezes complete ordered fingerprints and positions with owner/reason; changed, removed, third or newly duplicated entries fail. @@ -9,4 +9,6 @@ Removing entries during a mechanical move would change behavior: both copies con A separate change should select one descriptor from actual Git parser/schema/approval semantics, remove the second copies and retire the exact ledger entries together. Verify plain/strict native wire, tool.view under all roles, rare-tool loading, grammar offered names, Git arguments and approval. Record the expected stable-prefix byte change and one-time cache invalidation; do not merely regenerate golden outputs. Require 85 unique entries with no duplicate ledger afterward. Do not infer Git policy from readonly or change Git execution incidentally. -This is explicit behavioral debt outside mechanical05l acceptance. Other families' canonical ownership can expand separately: the current global gate proves presence/format for them, rather than semantic equality of every representation. [Stage05 acceptance](../testing/stage-05-validation.md) records that limit. +Resolved by ATO-1296. Each tool kept the copy that matches its implementation in `src/tools/os/git/` and the JSON Schema in `src/prompt/github-tool-args-schemas.ts`: commit keeps `paths` (the second copy had dropped it), push keeps `setUpstream`, and checkout keeps its schema argument order. All three keep the default tier, as the first copy (the one native wire already used) had; `compact-tool-forms.ts` still collapses push and checkout under the compact prompt. The full prompt loses the three second copies, so the stable prefix changes once. The ledger is gone and the conformance test now requires every descriptor name to be unique. + +This was explicit behavioral debt outside mechanical05l acceptance. Other families' canonical ownership can expand separately: the current global gate proves presence/format for them, rather than semantic equality of every representation. [Stage05 acceptance](../testing/stage-05-validation.md) records that limit. diff --git a/src/prompt/README.md b/src/prompt/README.md index abd81ecda..64fb76636 100644 --- a/src/prompt/README.md +++ b/src/prompt/README.md @@ -18,7 +18,7 @@ build-prompt composes the bounded model context from explicit session inputs. st [Model behavior mode](../llm/docs/model-mode.md) selects two projections. `local` retains existing budgets, packing and result aging. [Cloud assembly](build-cloud-prompt.ts) preserves full selected content in a persisted, append-only message history, with state updates after conversation and no presentation quotas. Both paths share the stable prefix. Assembly and preview produce a pure candidate; runtime owns committing it before inference. -The 14 core filesystem entries in descriptor-A and the default JSON schema map consume explicit projections from [import-free operation contracts](../tools/os/fs/docs/contracts.md). Catalog positions, args text and serialized schema order stay unchanged. Rendering does not import execution or own its argument semantics. [Global conformance](../tools/tool-contract-conformance.test.ts) checks the actual catalog, including narrowly frozen [existing Git duplicates](../../docs/proposals/tool-catalog-deduplication.md); other families retain their existing owners. +The 14 core filesystem entries in descriptor-A and the default JSON schema map consume explicit projections from [import-free operation contracts](../tools/os/fs/docs/contracts.md). Catalog positions, args text and serialized schema order stay unchanged. Rendering does not import execution or own its argument semantics. [Global conformance](../tools/tool-contract-conformance.test.ts) checks the actual catalog and requires unique descriptor names (the old [Git duplicates](../../docs/proposals/tool-catalog-deduplication.md) are gone); other families retain their existing owners. ## Task-specific reading diff --git a/src/prompt/default-tool-descriptors-a.ts b/src/prompt/default-tool-descriptors-a.ts index d86c5c5f5..7324561bc 100644 --- a/src/prompt/default-tool-descriptors-a.ts +++ b/src/prompt/default-tool-descriptors-a.ts @@ -157,24 +157,28 @@ export const DEFAULT_TOOL_DESCRIPTORS_A: readonly ToolDescriptor[] = [ "{ repo?: string, includeRemote?: boolean, contains?: string, pattern?: string }", tier: "rare", }, + // One descriptor per git write tool (ATO-1296): each matches the + // implementation in tools/os/git/ and its JSON Schema in + // github-tool-args-schemas.ts. { name: "os.git.checkout", summary: - "Switch to a branch, or create one first (may require approval). Never discards local changes.", + "Switch to a branch, or create one first (`startPoint` only with `create`). Branches only, never paths (may require approval). Never discards local changes.", argsSchema: "{ branch: string, create?: boolean, startPoint?: string, repo?: string }", }, { name: "os.git.commit", summary: - "Stage (`paths` or `all`) and commit with `message` (may require approval). Neither → commits what is already staged.", + "Stage (`paths` or `all`, not both) and commit with `message` (may require approval). Neither → commits what is already staged.", argsSchema: "{ message: string, paths?: string[], all?: boolean, repo?: string }", + examples: ['{"message":"feat: add parser"}'], }, { name: "os.git.push", summary: - "Push the current (or named) branch to a remote, `-u` by default (may require approval). No force-push. Uses the connected GitHub token for github.com.", + "Push the current (or named) branch to a remote (default origin), `-u` on first push (may require approval). No force-push. Needs Remote sync on. Uses the connected GitHub token for github.com.", argsSchema: "{ remote?: string, branch?: string, setUpstream?: boolean, repo?: string }", }, @@ -241,14 +245,6 @@ export const DEFAULT_TOOL_DESCRIPTORS_A: readonly ToolDescriptor[] = [ argsSchema: "{ repo?: string, remote?: string, branch?: string, rebase?: boolean }", tier: "rare", }, - { - name: "os.git.push", - summary: - "Push a branch (default: current) to a remote (default origin); sets upstream on first push; never forces. Needs Remote sync on; approval-gated.", - argsSchema: "{ repo?: string, remote?: string, branch?: string }", - examples: ['{"remote":"origin"}'], - tier: "rare", - }, { name: "os.git.clone", summary: "Clone a repository into the working directory (or dest). Needs Remote sync on; approval-gated.", @@ -272,21 +268,6 @@ export const DEFAULT_TOOL_DESCRIPTORS_A: readonly ToolDescriptor[] = [ "{ repo?: string, paths?: string[], all?: boolean, unstage?: boolean }", examples: ['{"all":true}'], }, - { - name: "os.git.commit", - summary: - "Commit staged changes with a message; `all` also commits tracked modifications. Approval like a file write.", - argsSchema: "{ repo?: string, message: string, all?: boolean }", - examples: ['{"message":"feat: add parser"}'], - }, - { - name: "os.git.checkout", - summary: - "Switch branches; `create` makes a new one (optionally from `startPoint`). Branches only, never paths. Approval like a file write.", - argsSchema: - "{ repo?: string, branch: string, create?: boolean, startPoint?: string }", - tier: "rare", - }, { name: "os.proc.list", summary: "List processes (filter, limit). Read-only.", diff --git a/src/prompt/tool-descriptors.test.ts b/src/prompt/tool-descriptors.test.ts new file mode 100644 index 000000000..2f83f7544 --- /dev/null +++ b/src/prompt/tool-descriptors.test.ts @@ -0,0 +1,49 @@ +import { describe, it, expect } from "vitest"; + +import { + DEFAULT_TOOL_DESCRIPTORS, + getToolDescriptorByName, +} from "./tool-descriptors.js"; + +/** + * ATO-1296: `os.git.push`, `os.git.commit` and `os.git.checkout` each + * had two descriptors with different args. The prompt showed both, and + * the by-name lookup (last-wins) served the second, so `tool.view` and + * error auto-expansion showed `os.git.commit` without `paths` although + * the tool accepts it. + */ +describe("DEFAULT_TOOL_DESCRIPTORS", () => { + it("has exactly one descriptor per tool name", () => { + const seen = new Set(); + const duplicates: string[] = []; + for (const descriptor of DEFAULT_TOOL_DESCRIPTORS) { + if (seen.has(descriptor.name)) duplicates.push(descriptor.name); + seen.add(descriptor.name); + } + expect(duplicates).toEqual([]); + }); + + it("resolves each name to the one descriptor the prompt renders", () => { + for (const descriptor of DEFAULT_TOOL_DESCRIPTORS) { + expect(getToolDescriptorByName(descriptor.name), descriptor.name).toBe(descriptor); + } + }); + + it("describes os.git.commit with `paths`, in the text and the JSON Schema", () => { + const commit = getToolDescriptorByName("os.git.commit")!; + expect(commit.argsSchema).toContain("paths?: string[]"); + const properties = commit.argsJsonSchema?.properties as Record; + expect(Object.keys(properties).sort()).toEqual(["all", "message", "paths", "repo"]); + expect(commit.argsJsonSchema?.required).toEqual(["message"]); + }); + + it("keeps the git write tools' text args in step with their JSON Schema", () => { + for (const name of ["os.git.checkout", "os.git.commit", "os.git.push"]) { + const descriptor = getToolDescriptorByName(name)!; + const properties = descriptor.argsJsonSchema?.properties as Record; + const textKeys = [...descriptor.argsSchema.matchAll(/(\w+)\??:/g)].map((m) => m[1]); + expect(textKeys.sort(), name).toEqual(Object.keys(properties).sort()); + } + expect(getToolDescriptorByName("os.git.push")!.argsSchema).toContain("setUpstream?: boolean"); + }); +}); diff --git a/src/tools/docs/contracts.md b/src/tools/docs/contracts.md index 447ffc5fb..a117b1ded 100644 --- a/src/tools/docs/contracts.md +++ b/src/tools/docs/contracts.md @@ -9,7 +9,7 @@ ToolRegistry owns explicit definitions and invocation. A new tool must be regist [The 14 core filesystem contracts](../os/fs/docs/contracts.md) own descriptor, wire schema, definition metadata and class without imports. Pure parsers share those owners; IO/path-policy validation stays in execution. Consumers select projections explicitly, preserving serialized order and text. [Global conformance](../tool-contract-conformance.test.ts) joins real static registrations before Map replacement with descriptor/schema/class inventories. It proves canonical agreement for these 14 tools, and presence/format plus actual taxonomy lookup for the other built-ins. Independent fixtures detect missing, orphan, duplicate and changed canonical facets; there is no catalog-wide generator. -The catalog currently has 98 entries for 95 names. Three pre-existing Git descriptor duplicates are frozen by exact positions and full ordered fingerprints in that test. New, changed or removed exceptions fail. [Their separate behavioral follow-up](../../../docs/proposals/tool-catalog-deduplication.md) explains why removing them would change prompt bytes and discovery. Static registrations have no duplicate exemption. ToolRegistry replacement remains intentional for read-scope decoration and dynamic MCP lifecycle, each tested separately. +Every descriptor name is unique, with no exemption: the three old Git duplicates were removed by ATO-1296 ([follow-up](../../../docs/proposals/tool-catalog-deduplication.md)). Static registrations have no duplicate exemption either. ToolRegistry replacement remains intentional for read-scope decoration and dynamic MCP lifecycle, each tested separately. Both reply and finish have static descriptors and schemas. The native adapter supplies terminal wire overrides and may include them beyond a role's visible catalog. Conditional registration (browser/memory/tasks/vision/screen), descriptor-only availability (web/email/GitHub/fusion/image/video), four static MCP meta tools and server-qualified external schemas have explicit owners in the conformance tests. Do not infer roles or batching permission from readonly metadata. diff --git a/src/tools/os/fs/docs/contracts.md b/src/tools/os/fs/docs/contracts.md index a7cc7f76c..768596b97 100644 --- a/src/tools/os/fs/docs/contracts.md +++ b/src/tools/os/fs/docs/contracts.md @@ -27,4 +27,4 @@ Run `npx vitest run src/tools/os/fs src/tools/tool-contract-conformance.test.ts [Global conformance](../../../tool-contract-conformance.test.ts) records real static definitions before ToolRegistry's Map can hide duplicate registration, inventories the private schema map via test-only TypeScript AST and compares the 14 canonical projections. Negative fixtures mutate each facet independently. Other static families get presence/format and taxonomy checks, not a claim of canonical semantic agreement. [Runtime composition](../../../../runtime/tool-contract-composition.test.ts) separates native registration from read-scope replacements and checks feature switches. MCP external schema/replacement/resolver teardown is tested independently. -The three existing Git descriptor duplicates are narrowly frozen; [follow-up](../../../../../docs/proposals/tool-catalog-deduplication.md) describes the behavioral fix. Unit checks establish neither native trash behavior on every platform nor model-quality/performance gains. [Stage 05 acceptance](../../../../../docs/testing/stage-05-validation.md) records baseline comparisons and remaining debt. +The three old Git descriptor duplicates were removed by ATO-1296 ([follow-up](../../../../../docs/proposals/tool-catalog-deduplication.md)). Unit checks establish neither native trash behavior on every platform nor model-quality/performance gains. [Stage 05 acceptance](../../../../../docs/testing/stage-05-validation.md) records baseline comparisons and remaining debt. diff --git a/src/tools/tool-contract-conformance.test.ts b/src/tools/tool-contract-conformance.test.ts index b2066f0f8..a9c12f60c 100644 --- a/src/tools/tool-contract-conformance.test.ts +++ b/src/tools/tool-contract-conformance.test.ts @@ -1,7 +1,6 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { createHash } from "node:crypto"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import ts from "typescript"; @@ -114,14 +113,10 @@ const CONDITIONAL = { wrapper: { owner: "tools/read-scope/confine-reads", reason: "The protection phase replaces selected definitions without changing their catalog metadata" }, }; -// Existing catalog debt, independently captured from the immutable pre-05l graph. -// Both copies affect stable-prefix bytes; discovery is last-wins and native wire is first-wins. -// This bounded compatibility ledger grants no permission for another duplicate. -const KNOWN_DESCRIPTOR_DUPLICATES = new Map([ - ["os.git.push", { indices: [34, 44], fingerprints: ["093e96423e4d2e226a0c520bea7398f10f6f9fc22a02b382e57191b57412e67f", "9d8ae0fa05777782f8218cfa69b6661e2c85c0a191406ee581cc52a9e2602a48"], owner: "prompt/default-tool-descriptors-a", reason: "Preserve shipped prefix entries and first-wire/last-discovery resolution during the mechanical move" }], - ["os.git.commit", { indices: [33, 48], fingerprints: ["95ef51ed40cc6ce5a762a7138f9f840dfa3e932b6f0f01bea11dd81a02fed773", "76a621b86514c6c0de7bdc5987127ccb26ebec8534bdf273cf46a5fb2f83227e"], owner: "prompt/default-tool-descriptors-a", reason: "Preserve shipped prefix entries and first-wire/last-discovery resolution during the mechanical move" }], - ["os.git.checkout", { indices: [32, 49], fingerprints: ["838fd065d1f322f4948821d4e787582d513bd86336b003f3e7143142080030ce", "981cfd260f769a8d38355113818ad41766037e142d4e5f5bbe37fff689ee2e48"], owner: "prompt/default-tool-descriptors-a", reason: "Preserve shipped prefix entries and first-wire/last-discovery resolution during the mechanical move" }], -]); +// Descriptor names are unique, with no exceptions: ATO-1296 removed the last +// duplicates (os.git.push/commit/checkout), whose second copies hid the first +// from discovery (`getToolDescriptorByName` is last-wins) and doubled them in +// the prompt. const sourceRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -200,8 +195,7 @@ interface Catalog { /** * A descriptor without its compact-prompt form (`compact`), which * `tool-descriptors.ts` attaches from its own table like - * `argsJsonSchema`: neither is part of a tool's contract or of a known - * duplicate's fingerprint. + * `argsJsonSchema`: neither is part of a tool's contract. */ function withoutCompactForm(descriptor: GateDescriptor): GateDescriptor { const core = { ...descriptor }; @@ -210,17 +204,9 @@ function withoutCompactForm(descriptor: GateDescriptor): GateDescriptor { } function conformanceErrors(catalog: Catalog): string[] { const errors: string[] = []; - const admittedDebt = new Set(); - for (const [name, debt] of KNOWN_DESCRIPTOR_DUPLICATES) { - const entries = catalog.descriptors.flatMap((descriptor, index) => descriptor.name === name ? [{ descriptor, index }] : []); - const indices = entries.map((entry) => entry.index); - const fingerprints = entries.map((entry) => createHash("sha256").update(JSON.stringify(withoutCompactForm(entry.descriptor))).digest("hex")); - if (JSON.stringify(indices) === JSON.stringify(debt.indices) && JSON.stringify(fingerprints) === JSON.stringify(debt.fingerprints)) admittedDebt.add(name); - else errors.push(`changed known descriptor debt: ${name}`); - } const unique = (facet: string, names: readonly string[]): Set => { const seen = new Set(); - for (const name of names) { if (seen.has(name) && !(facet === "descriptor" && admittedDebt.has(name))) errors.push(`duplicate ${facet}: ${name}`); seen.add(name); } + for (const name of names) { if (seen.has(name)) errors.push(`duplicate ${facet}: ${name}`); seen.add(name); } return seen; }; const descriptors = unique("descriptor", catalog.descriptors.map((d) => d.name)); @@ -347,14 +333,8 @@ describe("whole built-in tool catalog conformance", () => { expect(names(DEFAULT_TOOL_DESCRIPTORS)).toContain(name); expect(resourceClassFor(name)).toBe("terminal"); } - expect(DEFAULT_TOOL_DESCRIPTORS).toHaveLength(100); - expect(new Set(names(DEFAULT_TOOL_DESCRIPTORS)).size).toBe(97); - for (const [name, debt] of KNOWN_DESCRIPTOR_DUPLICATES) { - expect(debt.owner).not.toBe(""); expect(debt.reason).not.toBe(""); - const copies = DEFAULT_TOOL_DESCRIPTORS.filter((descriptor) => descriptor.name === name); - expect(copies).toHaveLength(2); - expect(getToolDescriptorByName(name)).toBe(copies[1]); - } + expect(DEFAULT_TOOL_DESCRIPTORS).toHaveLength(97); + expect(new Set(names(DEFAULT_TOOL_DESCRIPTORS)).size).toBe(DEFAULT_TOOL_DESCRIPTORS.length); expect(registry.list()).toHaveLength(registry.registrations.length); for (const definition of registry.registrations) { expect(typeof definition.readonly).toBe("boolean"); @@ -382,10 +362,10 @@ describe("whole built-in tool catalog conformance", () => { { mutate: (input) => ({ ...input, descriptors: input.descriptors.map((item) => item === descriptor ? { ...item, argsJsonSchema: { type: "object", properties: {}, required: ["absent"], additionalProperties: true } } : item) }), expected: `invalid schema: ${descriptor.name}` }, ]; expect(conformanceErrors({ ...base, descriptors: [...base.descriptors, { ...descriptor, name: "orphan" }] })).toContain("missing registration: orphan"); - const known = base.descriptors.find((item) => item.name === "os.git.push")!; - expect(conformanceErrors({ ...base, descriptors: base.descriptors.map((item) => item === known ? { ...item, summary: "changed known copy" } : item) })).toContain("changed known descriptor debt: os.git.push"); - expect(conformanceErrors({ ...base, descriptors: [...base.descriptors, known] })).toContain("duplicate descriptor: os.git.push"); - expect(conformanceErrors({ ...base, descriptors: base.descriptors.filter((item) => item.name !== "os.git.push") })).toContain("changed known descriptor debt: os.git.push"); + for (const name of ["os.git.push", "os.git.commit", "os.git.checkout"]) { + const git = base.descriptors.find((item) => item.name === name)!; + expect(conformanceErrors({ ...base, descriptors: [...base.descriptors, { ...git, summary: "second copy" }] })).toContain(`duplicate descriptor: ${name}`); + } expect(conformanceErrors({ ...base, classes: new Map([...base.classes, [descriptor.name, "unrecognized"]]) })).toContain(`missing class: ${descriptor.name}`); for (const mutation of mutations) expect(conformanceErrors(mutation.mutate(base))).toContain(mutation.expected); expect(conformanceErrors(base)).toEqual([]); // Fixtures never mutate shipped references. From f6c506c4356f1a5c9eef1f63ebad68d974312cc7 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:11:59 +0300 Subject: [PATCH 019/155] fix(fs): keep the folder that holds the missing path in the near search The nearest existing folder of the missing path and a protected folder that holds it are where the file was expected, so they keep the full budget. Only folders above them are outer. --- src/tools/os/fs/fs-missing-path.test.ts | 2 +- src/tools/os/fs/path-candidate-roots.ts | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/tools/os/fs/fs-missing-path.test.ts b/src/tools/os/fs/fs-missing-path.test.ts index e2aa2f522..addfaceb0 100644 --- a/src/tools/os/fs/fs-missing-path.test.ts +++ b/src/tools/os/fs/fs-missing-path.test.ts @@ -128,7 +128,7 @@ describe("candidateRoots", () => { exists, }); expect(plan.blocked).not.toContain("/Users/u/Desktop"); - expect(plan.roots).toContainEqual({ dir: "/Users/u/Desktop", depth: 3, outer: true }); + expect(plan.roots).toContainEqual({ dir: "/Users/u/Desktop", depth: 3 }); }); // ATO-1333: a crowded parent must not use up the working folder's budget. diff --git a/src/tools/os/fs/path-candidate-roots.ts b/src/tools/os/fs/path-candidate-roots.ts index 7e9f70e0a..d36d85659 100644 --- a/src/tools/os/fs/path-candidate-roots.ts +++ b/src/tools/os/fs/path-candidate-roots.ts @@ -15,8 +15,8 @@ import { isInside } from "../../../approval/fanout-scope.js"; * missing path, the working folder or a user-named path is already * inside it; otherwise it is blocked. * - * A root that is not inside the working folder or a user-named path (the - * missing path's ancestors, a protected folder above the session) is + * A root that is not inside the working folder or a user-named path and + * does not hold the missing path (the folders above the session) is * marked `outer`: it is searched after the others, under a small budget * of its own, so a crowded one cannot starve the search near the session. */ @@ -96,12 +96,16 @@ export async function candidateRoots( const ancestor = await nearestExisting(dirname(missing), exists); if (ancestor !== null) { const parent = dirname(ancestor); - roots.push(placed(ancestor, 2), placed(parent, 1), placed(dirname(parent), 1)); + // The folder the missing path points into stays near: it is where the + // file was expected. Only the folders above it are outer. + roots.push({ dir: ancestor, depth: 2 }, placed(parent, 1), placed(dirname(parent), 1)); } roots.push( ...searchable .filter((dir) => qualifies(dir)) - .map((dir) => placed(dir, PROTECTED_DEPTH)), + .map((dir) => + isInside(dir, missing) ? { dir, depth: PROTECTED_DEPTH } : placed(dir, PROTECTED_DEPTH), + ), ); return { roots: roots.filter(({ dir }) => dir !== home && !isFilesystemRoot(dir)), From 4f4a7e24d58955029d8765216d81dd7d6c703058 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:12:29 +0300 Subject: [PATCH 020/155] fix(llm): anchor the Claude temperature rule and skip a resend that changes nothing - model-params: the Claude id rule is anchored at the start of the id (after the router prefix, with an optional `us.` / `anthropic.` Bedrock prefix), so a fine-tune that only names Claude, such as `Qwen3-8B-Claude-Sonnet-5-Distill`, keeps its temperature. - openai-provider: a refused-temperature 400 for a model already recorded as taking none, on a body that carried no temperature, gets no resend, since the same body would go out again. Linear: ATO-1397 --- src/llm/provider/openai/model-params.test.ts | 6 ++++- src/llm/provider/openai/model-params.ts | 9 +++++--- src/llm/provider/openai/openai-provider.ts | 13 ++++++++--- .../openai/openai-temperature-retry.test.ts | 22 ++++++++++++++++++- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/llm/provider/openai/model-params.test.ts b/src/llm/provider/openai/model-params.test.ts index 8a4b25de0..d0c2d53b3 100644 --- a/src/llm/provider/openai/model-params.test.ts +++ b/src/llm/provider/openai/model-params.test.ts @@ -85,6 +85,7 @@ describe("modelParamProfile — Claude models without sampling parameters (ATO-1 "anthropic/claude-opus-5-fast", "anthropic/claude-sonnet-5.5", "anthropic.claude-haiku-5-5", + "us.anthropic.claude-opus-5-5", "claude-sonnet-5-20260101", "Claude-Sonnet-5-5", // An unknown future version falls on the side of the line that omits. @@ -105,7 +106,10 @@ describe("modelParamProfile — Claude models without sampling parameters (ATO-1 "claude-haiku-4-5", "anthropic/claude-haiku-4.5", "claude-3-5-sonnet-20241022", - ])("leaves temperature free for the older %s", (model) => { + // Names Claude but is not one: the rule is anchored at the id's start. + "Qwen3-8B-Claude-Sonnet-5-Distill", + "someorg/Qwen3-8B-Claude-Sonnet-5-Distill", + ])("leaves temperature free for %s", (model) => { expect(modelParamProfile(model).temperature).toEqual({ kind: "free" }); }); diff --git a/src/llm/provider/openai/model-params.ts b/src/llm/provider/openai/model-params.ts index 2ffb04cd1..fcab69628 100644 --- a/src/llm/provider/openai/model-params.ts +++ b/src/llm/provider/openai/model-params.ts @@ -65,13 +65,16 @@ const KIMI_K31_MODEL_RE = /^kimi-k3\.1(?![0-9])/i; /** * A Claude model id in Anthropic's naming: family, major version and an * optional minor version (`claude-sonnet-5-5`, `claude-opus-4.8`, - * `anthropic/claude-fable-5`, Bedrock's `anthropic.claude-haiku-5-5`). The - * minor is one or two digits so a date suffix + * `anthropic/claude-fable-5`, Bedrock's `anthropic.claude-haiku-5-5` and + * `us.anthropic.claude-opus-5-5`). Anchored at the start of the id (after + * any router prefix, which `modelParamProfile` strips at the last `/`), so + * a fine-tune that only names Claude (`Qwen3-8B-Claude-Sonnet-5-Distill`) + * keeps its temperature. The minor is one or two digits so a date suffix * (`claude-sonnet-5-20260101`) is not read as one. The old naming * (`claude-3-5-sonnet`) does not match and keeps the free rule. */ const CLAUDE_MODEL_RE = - /(?:^|[^a-z0-9])claude-(opus|sonnet|haiku|fable|mythos)-(\d+)(?:[-.](\d{1,2}))?(?![0-9])/i; + /^(?:[a-z]+\.)?(?:anthropic\.)?claude-(opus|sonnet|haiku|fable|mythos)-(\d+)(?:[-.](\d{1,2}))?(?![0-9])/i; /** * Whether a Claude model rejects `temperature`. Anthropic removed the diff --git a/src/llm/provider/openai/openai-provider.ts b/src/llm/provider/openai/openai-provider.ts index 85ee39f3f..234e8350a 100644 --- a/src/llm/provider/openai/openai-provider.ts +++ b/src/llm/provider/openai/openai-provider.ts @@ -319,7 +319,7 @@ export class OpenAiProvider implements LlmProvider { try { return await send(shape); } catch (err) { - if (temperatureRetried || !this.learnTemperature(request, err)) { + if (temperatureRetried || !this.learnTemperature(request, err, sentBody)) { throw err; } temperatureRetried = true; @@ -407,17 +407,24 @@ export class OpenAiProvider implements LlmProvider { * ATO-1397). The lesson is remembered for the model for the rest of the * process, so the next body built — the resend, and every later turn — * carries it. The caller allows one such resend per completion. + * + * A refusal already on record for a body that carried no temperature + * is not this lesson at all (the 400 is about something else that names + * temperature), so it gets no resend: the same body would go out again. */ private learnTemperature( request: CompletionRequest, err: unknown, + sentBody: Record | undefined, ): boolean { if (request.signal?.aborted) return false; const lesson = temperatureLessonFromError(err); if (lesson === undefined) return false; const model = this.defaultChatModel!; if (lesson.kind === "forbidden") { - if (!rememberTemperatureForbidden(model)) return true; + if (!rememberTemperatureForbidden(model)) { + return sentBody === undefined || "temperature" in sentBody; + } this.http.logger?.warn( `llm: "${this.id}" refused temperature for this model; resending once without it and keeping that for this model`, { provider: this.id, model, status: 400, temperature: null }, @@ -617,7 +624,7 @@ export class OpenAiProvider implements LlmProvider { if ( !committed && !temperatureRetried && - this.learnTemperature(request, err) + this.learnTemperature(request, err, body) ) { temperatureRetried = true; body = this.buildBody(request, true, shape); diff --git a/src/llm/provider/openai/openai-temperature-retry.test.ts b/src/llm/provider/openai/openai-temperature-retry.test.ts index 633a1760a..c04319f83 100644 --- a/src/llm/provider/openai/openai-temperature-retry.test.ts +++ b/src/llm/provider/openai/openai-temperature-retry.test.ts @@ -4,7 +4,10 @@ import type { CompletionResult, StreamChunk, } from "../completion-types.js"; -import { forgetLearnedModelParams } from "./model-params.js"; +import { + forgetLearnedModelParams, + rememberTemperatureForbidden, +} from "./model-params.js"; import { OpenAiProvider } from "./openai-provider.js"; /** @@ -177,6 +180,23 @@ describe("OpenAiProvider — refused-temperature retry (ATO-1397)", () => { expect(bodies[2]).not.toHaveProperty("temperature"); }); + it("does not resend when the refusal is on record and the body carried no temperature", async () => { + rememberTemperatureForbidden(MODEL); + const { fetchImpl, bodies } = scripted([ + () => json(DEPRECATED_400, 400), + okUnary, + ]); + await expect(provider(fetchImpl).complete({ prompt: "hi" })).rejects.toThrow(/400/); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(bodies[0]).not.toHaveProperty("temperature"); + + const streamed = scripted([() => json(DEPRECATED_400, 400), okStream]); + await expect(drain(provider(streamed.fetchImpl).completeStream({ prompt: "hi" }))).rejects.toThrow( + /400/, + ); + expect(streamed.fetchImpl).toHaveBeenCalledTimes(1); + }); + it("sends no temperature to claude-sonnet-5-5 in the first place", async () => { const { fetchImpl, bodies } = scripted([okUnary]); const p = new OpenAiProvider({ From e1950ff76785cbc6ca101a4a9999a88a39a3e929 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:12:45 +0300 Subject: [PATCH 021/155] fix(shell-guard): let scripts print a sign-in hint that names a URL The browser bypass check read the text of print-only calls as if it ran, so python print("open https://..."), console.log(...), sys.stdout.write and process.stdout.write were refused at every approval level (ATO-1349, ATO-1353). String literals passed straight to these calls are now emptied before the line rules run; strings inside nested calls, f-string and template interpolations, and any output piped on to a runner are still read. echo/printf with several quoted arguments are exempt too. --- .../browser-bypass.test.ts | 24 ++++ .../os/shell-command-guard/browser-bypass.ts | 123 +++++++++++++++++- 2 files changed, 143 insertions(+), 4 deletions(-) diff --git a/src/tools/os/shell-command-guard/browser-bypass.test.ts b/src/tools/os/shell-command-guard/browser-bypass.test.ts index 165599993..5aa776018 100644 --- a/src/tools/os/shell-command-guard/browser-bypass.test.ts +++ b/src/tools/os/shell-command-guard/browser-bypass.test.ts @@ -24,6 +24,8 @@ beforeAll(async () => { await writeFile(join(cwd, "sign-in.sh"), "#!/bin/sh\ni=$((1+1))\necho 'To sign in, open https://example.test/device and enter ABCD-1234'\n"); await writeFile(join(cwd, "opens.sh"), "#!/bin/sh\ni=$((1+1))\necho 'starting'\nopen https://x.com\n"); await writeFile(join(cwd, "open.py"), 'import subprocess\nsubprocess.run(["open", "-a", "Safari", "https://x.com"])\n'); + await writeFile(join(cwd, "sign-in.py"), 'code = "ABCD-1234"\nprint(f"To sign in, open https://example.test/device and enter {code}")\n'); + await writeFile(join(cwd, "sign-in.js"), 'console.log("To sign in, open https://example.test/device and enter ABCD-1234");\n'); }); afterAll(async () => { await rm(cwd, { recursive: true, force: true }); @@ -53,6 +55,20 @@ const CASES: [string, boolean][] = [ [`echo 'To sign in, open https://example.test/device and enter the code'`, false], [`printf "open https://example.test/device\\n" >&2`, false], [`./sign-in.sh`, false], + // ATO-1349 / ATO-1353: a script that only prints a sign-in hint is not a browser launch. + [`echo 'To sign in,' "open https://example.test/device"`, false], + [`printf '%s\\n' 'open https://example.test/device'`, false], + [`bash <<'EOF'\necho "open https://example.test/device and enter the code"\nEOF`, false], + [`python3 -c 'print("To sign in, open https://example.test/device and enter ABCD-1234")'`, false], + [`python3 -c "print(\\"open https://example.test/device\\")"`, false], + [`python3 -c 'import sys; sys.stdout.write("open https://example.test/device\\n")'`, false], + [`python3 -c 'code = "ABCD"; print(f"Open https://example.test/device and enter {code}")'`, false], + [`python3 <<'EOF'\nprint("""Open https://example.test/device\nand enter the code""")\nEOF`, false], + [`node -e 'console.log("open https://example.test/device")'`, false], + [`node -e 'const c = "ABCD"; console.log(\`Open https://example.test/device and enter \${c}\`)'`, false], + [`node -e 'process.stdout.write("open https://example.test/device\\n")'`, false], + [`python3 sign-in.py`, false], + [`node sign-in.js`, false], [`osascript ok.applescript`, false], [`osascript -e 'tell application "Google Chrome" to set URL of active tab of front window to "https://mail.google.com"'`, true], [`osascript -e 'tell application "Google Chrome" to make new tab at end of tabs of front window with properties {URL:"https://x.com"}'`, true], @@ -94,6 +110,14 @@ const CASES: [string, boolean][] = [ [`echo 'open https://x.com' > run.sh`, true], [`echo hi; open https://x.com`, true], [`./opens.sh`, true], + [`python3 -c 'import subprocess; subprocess.run(["open", "https://x.com"])'`, true], + [`python3 -c 'import os; print(os.system("open https://x.com"))'`, true], + [`python3 -c 'print("hi"); import os; os.system("open https://x.com")'`, true], + [`python3 -c 'print("open https://x.com")' | sh`, true], + [`node -e 'require("child_process").exec("open https://x.com")'`, true], + [`node -e 'console.log("hi"); require("child_process").execSync("xdg-open https://x.com")'`, true], + [`node -e 'console.log(\`\${require("child_process").execSync("open https://x.com")}\`)'`, true], + [`echo 'hi' "open https://x.com" | sh`, true], ]; describe("checkBrowserBypassLine", () => { diff --git a/src/tools/os/shell-command-guard/browser-bypass.ts b/src/tools/os/shell-command-guard/browser-bypass.ts index 3e6d0dcdf..6d1aaeedd 100644 --- a/src/tools/os/shell-command-guard/browser-bypass.ts +++ b/src/tools/os/shell-command-guard/browser-bypass.ts @@ -56,7 +56,10 @@ import { lexShellLine, type ShellWord } from "./shell-line-lexer.js"; * in a quoted string of an unrelated command (an `echo`) passes; only * scripting contexts look for one. Likewise a message a script only * prints: a device sign-in saying `echo 'open https://… and enter the - * code'` is the step the agent has to run and hand over. + * code'` is the step the agent has to run and hand over. The same holds + * for the string literals given to Python `print` / `sys.stdout.write` / + * `logging`, and JavaScript `console.*` / `process.stdout.write`, unless + * the output is piped on to something that could run it. */ export const BROWSER_BYPASS_RULE = "browser.use_browser_tool"; @@ -160,12 +163,124 @@ const LINE_PROBLEMS: readonly ScriptProblem[] = [ ]; /** - * `echo` / `printf` with one quoted argument and nothing taking the output + * `echo` / `printf` with only quoted arguments and nothing taking the output * (a pipe or a file could carry it to a shell). What it says is not run, * so the line-level rules do not read it. */ const PRINTED_MESSAGE = - /(^|[\s;&|(])(echo|printf)((?:\s+-[a-z]+)*)\s+(?:'[^'\n]*'|"[^"\n$`]*")(?=[ \t]*(?:[12]?>&[12][ \t]*)?(?:$|[\n;#]|&&|\|\|))/gm; + /(^|[\s;&|(])(echo|printf)((?:\s+-[a-z]+)*)(?:\s+(?:'[^'\n]*'|"[^"\n$`]*"))+(?=[ \t]*(?:[12]?>&[12][ \t]*)?(?:$|[\n;#]|&&|\|\|))/gm; + +/** + * Calls in a script that only print (ATO-1349, ATO-1353): Python `print`, + * `sys.stdout.write`, `logging.*`, JavaScript `console.*` and + * `process.stdout.write`. A sign-in hint such as + * `print("Open https://… and enter the code")` is not a browser launch. + */ +const PRINT_CALL = + /(?= text.length) return "unterminated"; + kept += text.slice(j, k + 1); + j = k + 1; + continue; + } + j += 1; + } + return "unterminated"; +} /** Commands that run the command after them; the value-taking flags of each. */ const PREFIX_COMMANDS: Readonly> = { @@ -279,7 +394,7 @@ function checkScriptFile(word: ShellWord, ctx: CommandContext, shell: boolean): function checkText(raw: string, osascriptSource: string): GuardVerdict | null { const text = raw.normalize("NFKC").toLowerCase(); - const whatRuns = text.replace(PRINTED_MESSAGE, "$1$2$3 ''"); + const whatRuns = blankPrintedLiterals(text.replace(PRINTED_MESSAGE, "$1$2$3 ''")); for (const problem of LINE_PROBLEMS) { if (problem.test.test(whatRuns)) return refuse(problem.what); } From 100396e30515d4625cabfc9700eb84b2a45ab23a Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:13:03 +0300 Subject: [PATCH 022/155] fix(approval): apply saved deny rules to safe shell commands and nested shell -c lines A command the shell guard deems safe (echo, ls) never reached the approval gate, so a saved deny rule on it did not apply. os.shell.run now asks the gate's rule decision for such a command and sends a deny to the gate, which refuses it before any level, grant or prompt. Ask and allow rules keep their current semantics. shellRuleSegments now also lists the commands a nested bash/sh/zsh -c line runs (bundled flags like -lc, -ec, recursively, bounded depth), so a deny rule on rm stops bash -c "rm -rf tmp". The nested-shell option scan is shared with the guard's dangerous layer. Refs ATO-1332 --- src/approval/README.md | 2 +- src/approval/dangerous-tool.ts | 13 ++ .../shell-command-guard/command-segments.ts | 50 ++++++ .../rules-dangerous-argv.ts | 25 +-- src/tools/os/shell/shell-command-argv.test.ts | 39 +++++ src/tools/os/shell/shell-command-argv.ts | 45 +++++- .../os/shell/shell-saved-deny-rules.test.ts | 143 ++++++++++++++++++ src/tools/os/shell/shell.ts | 33 ++-- 8 files changed, 314 insertions(+), 36 deletions(-) create mode 100644 src/tools/os/shell/shell-saved-deny-rules.test.ts diff --git a/src/approval/README.md b/src/approval/README.md index eb790bb34..73e54f222 100644 --- a/src/approval/README.md +++ b/src/approval/README.md @@ -20,7 +20,7 @@ This area owns approval policy and routing. Read [AGENTS.md](AGENTS.md) before c ## Ownership and dependencies -Gate order in `ApprovalGate.request`: the same-turn refusal (`declinedInTurn`), a matching deny rule (refused, `automatic`), a matching ask rule (skips the level and grants and prompts), the level, the session grant, a matching allow rule, then the session's refuse policy or the prompt. `resolve({ grant: "always" })` saves the allow rule built from the pending request, scoped to its project; hardline shell-guard rules fire before the gate and no rule touches them. A fan-out scope returns before the gate, so the tools ask `ruleDecisionFor` first: a deny or ask rule sends the call to the gate. A worker in a throwaway worktree matches rules against its parent's project (`SessionApprovalPolicy.ruleWorkdir`). +Gate order in `ApprovalGate.request`: the same-turn refusal (`declinedInTurn`), a matching deny rule (refused, `automatic`), a matching ask rule (skips the level and grants and prompts), the level, the session grant, a matching allow rule, then the session's refuse policy or the prompt. `resolve({ grant: "always" })` saves the allow rule built from the pending request, scoped to its project; hardline shell-guard rules fire before the gate and no rule touches them. A fan-out scope returns before the gate, so the tools ask `ruleDecisionFor` first: a deny or ask rule sends the call to the gate. A shell command the guard deems safe never asks either, so `os.shell.run` asks `ruleDecisionFor` for it too: a deny rule sends it to the gate, which refuses it (ATO-1332). Deny and ask rules match every command in `commandSegments`, including those a nested `bash|sh|zsh -c` line runs (`shellRuleSegments`); an allow rule still needs `commandArgv`. A worker in a throwaway worktree matches rules against its parent's project (`SessionApprovalPolicy.ruleWorkdir`). ApprovalRouter selects the handler for the owning session; level/grant logic decides whether a dispatch can proceed unattended or needs a decision. Frontends and channel bridges provide interaction, while tools consume the decision. Approval state is separate from offered prompt schemas and from the currently selected UI chat. diff --git a/src/approval/dangerous-tool.ts b/src/approval/dangerous-tool.ts index 223d3f511..dec38689d 100644 --- a/src/approval/dangerous-tool.ts +++ b/src/approval/dangerous-tool.ts @@ -150,6 +150,19 @@ export function savedRuleOverridesScope( return decision === "deny" || decision === "ask"; } +/** + * True when a saved deny rule covers this call. A command the shell + * guard deems safe never asks, so without this a "never run `echo`" + * rule would not stop `echo hi` (ATO-1332): such a call goes to the + * gate, which refuses it before any level, grant or prompt. + */ +export function savedRuleDenies( + approvals: ApprovalGate, + check: RuleCheck, +): boolean { + return (approvals.ruleDecisionFor?.(check) ?? null) === "deny"; +} + /** * Shared helper used by dangerous tools (shell.run, fs.write, * skill.run_script, …). Centralising it ensures every tool sends the same diff --git a/src/tools/os/shell-command-guard/command-segments.ts b/src/tools/os/shell-command-guard/command-segments.ts index 7508e68d0..a807c7255 100644 --- a/src/tools/os/shell-command-guard/command-segments.ts +++ b/src/tools/os/shell-command-guard/command-segments.ts @@ -126,3 +126,53 @@ export function isLongOption( if (word === name || word.startsWith(`${name}=`)) return true; return word.length >= minLength && name.startsWith(word); } + +/** Shells whose `-c` runs the next word as a command line. */ +export const NESTED_SHELL_PROGRAMS: readonly string[] = ["bash", "sh", "zsh", "dash", "ksh"]; + +export interface NestedShellOptions { + /** `-c` (alone or bundled, `-lc`, `-ec`) or `--command` was given. */ + runsCommand: boolean; + /** + * The command line it runs: the first word after the options, or the + * value of `--command=…`. Undefined when there is none. + */ + script?: string; +} + +/** + * Read the options of a nested shell, `args` being the words after the + * program (`bash -lc 'cd x && rm y'` → `-lc`, `cd x && rm y`). Only the + * options before the first operand count: `bash run.sh -c` passes `-c` + * to the script. + */ +export function scanNestedShellOptions(args: readonly string[]): NestedShellOptions { + let runsCommand = false; + let inline: string | undefined; + let i = 0; + for (; i < args.length; i += 1) { + const word = args[i] ?? ""; + if (word === "--") { + i += 1; + break; + } + if (!word.startsWith("-") || word === "-") break; + if (isLongOption(word, "--command")) { + runsCommand = true; + if (word.startsWith("--command=")) inline = word.slice("--command=".length); + continue; + } + if (word === "--rcfile" || word === "--init-file") { + i += 1; + continue; + } + const letters = shortOptionLetters(word, "oO"); + if (letters === null) continue; + if (letters.includes("c")) runsCommand = true; + // `-o pipefail`: the value is the next word. + if (letters.at(-1) === "o" || letters.at(-1) === "O") i += 1; + } + if (!runsCommand) return { runsCommand }; + const script = inline ?? args[i]; + return script !== undefined ? { runsCommand, script } : { runsCommand }; +} diff --git a/src/tools/os/shell-command-guard/rules-dangerous-argv.ts b/src/tools/os/shell-command-guard/rules-dangerous-argv.ts index 3710bcced..61251a603 100644 --- a/src/tools/os/shell-command-guard/rules-dangerous-argv.ts +++ b/src/tools/os/shell-command-guard/rules-dangerous-argv.ts @@ -2,7 +2,9 @@ import type { GuardVerdict, NormalisedCommand } from "./guard-types.js"; import { invocationsOf, isLongOption, + NESTED_SHELL_PROGRAMS, programName, + scanNestedShellOptions, shortOptionLetters, splitCommandSegments, } from "./command-segments.js"; @@ -377,8 +379,6 @@ export function matchHardlineRm(input: NormalisedCommand): GuardVerdict | null { return null; } -const SHELL_PROGRAMS: readonly string[] = ["bash", "sh", "zsh", "dash", "ksh"]; - /** * `bash -lc …`, `env bash -c …`, `cd x && sh -c …`: a nested shell * however it is started. Only the options before the first script @@ -387,25 +387,10 @@ const SHELL_PROGRAMS: readonly string[] = ["bash", "sh", "zsh", "dash", "ksh"]; function matchNestedShell( segments: readonly (readonly string[])[], ): ArgvCheck | null { - for (const program of SHELL_PROGRAMS) { + for (const program of NESTED_SHELL_PROGRAMS) { for (const args of invocationsOf(segments, program)) { - for (let i = 0; i < args.length; i += 1) { - const word = args[i] ?? ""; - if (!word.startsWith("-") || word === "-" || word === "--") break; - if (isLongOption(word, "--command")) { - return { id: "dangerous.shell_dash_c", reason: "nested shell command" }; - } - if (word === "--rcfile" || word === "--init-file") { - i += 1; - continue; - } - const letters = shortOptionLetters(word, "oO"); - if (letters === null) continue; - if (letters.includes("c")) { - return { id: "dangerous.shell_dash_c", reason: "nested shell command" }; - } - // `-o pipefail`: the value is the next word. - if (letters.at(-1) === "o" || letters.at(-1) === "O") i += 1; + if (scanNestedShellOptions(args).runsCommand) { + return { id: "dangerous.shell_dash_c", reason: "nested shell command" }; } } } diff --git a/src/tools/os/shell/shell-command-argv.test.ts b/src/tools/os/shell/shell-command-argv.test.ts index 3cbd7388d..a105450f2 100644 --- a/src/tools/os/shell/shell-command-argv.test.ts +++ b/src/tools/os/shell/shell-command-argv.test.ts @@ -39,6 +39,45 @@ describe("shellRuleSegments", () => { ]); }); + it("lists the commands a nested `bash -c` runs after the shell itself (ATO-1332)", () => { + expect(shellRuleSegments("bash", ["-c", "rm -rf tmp"], false)).toEqual([ + ["bash", "-c", "rm -rf tmp"], + ["rm", "-rf", "tmp"], + ]); + expect(shellRuleSegments('bash -c "rm -rf tmp"', [], true)).toEqual([ + ["bash", "-c", "rm -rf tmp"], + ["rm", "-rf", "tmp"], + ]); + }); + + it("reads bundled flags and splits the nested line into its commands", () => { + expect(shellRuleSegments("sh -lc 'cd x && rm y'", [], true)).toEqual([ + ["sh", "-lc", "cd x && rm y"], + ["cd", "x"], + ["rm", "y"], + ]); + expect(shellRuleSegments("zsh", ["-e", "-o", "pipefail", "-c", "npm publish"], false)).toContainEqual([ + "npm", + "publish", + ]); + expect(shellRuleSegments("sudo bash -ec 'npm publish'", [], true)).toContainEqual([ + "npm", + "publish", + ]); + }); + + it("unwraps shells nested in shells", () => { + expect( + shellRuleSegments(`bash -c "sh -c 'rm y'"`, [], true), + ).toContainEqual(["rm", "y"]); + }); + + it("does not read a script's own `-c` argument as a command line", () => { + expect(shellRuleSegments("bash", ["run.sh", "-c", "rm y"], false)).toEqual([ + ["bash", "run.sh", "-c", "rm y"], + ]); + }); + it("lists a wrapped command both ways", () => { expect(shellRuleSegments("FOO=1 sudo -n npm publish", [], true)).toEqual([ ["FOO=1", "sudo", "-n", "npm", "publish"], diff --git a/src/tools/os/shell/shell-command-argv.ts b/src/tools/os/shell/shell-command-argv.ts index b8ba6e920..45c7be2a2 100644 --- a/src/tools/os/shell/shell-command-argv.ts +++ b/src/tools/os/shell/shell-command-argv.ts @@ -1,5 +1,8 @@ import { + invocationsOf, + NESTED_SHELL_PROGRAMS, programName, + scanNestedShellOptions, splitCommandSegments, } from "../shell-command-guard/command-segments.js"; @@ -61,12 +64,20 @@ function withoutWrappers(words: readonly string[]): string[] { return words.slice(i); } +/** + * How many `bash -c` levels deep a deny rule still looks. A real line + * nests one or two; the bound only stops a pathological input. + */ +const MAX_NESTED_SHELL_DEPTH = 4; + /** * Every simple command a call would run, for saved deny and ask rules * (`ApprovalRequest.commandSegments`). Unlike `shellCommandArgv` this is * never withheld: a rule that only tightens is safe to apply on a best * reading of the line. A wrapped command is listed both ways, so a rule - * on `sudo` and a rule on what it runs both match. + * on `sudo` and a rule on what it runs both match. The line a nested + * shell runs (`bash -c "rm -rf x"`, `sh -lc 'cd x && rm y'`) is split + * the same way and its commands listed after the shell's own (ATO-1332). */ export function shellRuleSegments( cmd: string, @@ -76,8 +87,32 @@ export function shellRuleSegments( const segments = useShell ? splitCommandSegments([cmd, ...execArgs].join(" ")) : [[cmd, ...execArgs]]; - return segments.flatMap((words) => { - const inner = withoutWrappers(words); - return inner.length > 0 && inner.length < words.length ? [words, inner] : [words]; - }); + return segments.flatMap((words) => expandSegment(words, 0)); +} + +function expandSegment(words: string[], depth: number): string[][] { + const inner = withoutWrappers(words); + const listed = inner.length > 0 && inner.length < words.length ? [words, inner] : [words]; + if (depth >= MAX_NESTED_SHELL_DEPTH) return listed; + const nested = nestedShellScripts(words).flatMap((script) => + splitCommandSegments(script).flatMap((w) => expandSegment(w, depth + 1)), + ); + return [...listed, ...nested]; +} + +/** + * The command lines nested shells in `words` run: `bash -c ` + * anywhere in the segment, so `sudo bash -c`, `nice -n 5 sh -c` and + * `xargs sh -c` are read too, the way the guard's dangerous layer finds + * them. + */ +function nestedShellScripts(words: readonly string[]): string[] { + const scripts: string[] = []; + for (const program of NESTED_SHELL_PROGRAMS) { + for (const args of invocationsOf([words], program)) { + const { script } = scanNestedShellOptions(args); + if (script !== undefined && script.trim() !== "") scripts.push(script); + } + } + return scripts; } diff --git a/src/tools/os/shell/shell-saved-deny-rules.test.ts b/src/tools/os/shell/shell-saved-deny-rules.test.ts new file mode 100644 index 000000000..825ec0ffb --- /dev/null +++ b/src/tools/os/shell/shell-saved-deny-rules.test.ts @@ -0,0 +1,143 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdir, mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + ApprovalGate, + type ApprovalRequest, +} from "../../../approval/approval-gate.js"; +import type { ApprovalRule } from "../../../approval/approval-rules.js"; +import { ApprovalDeniedError } from "../../../approval/dangerous-tool.js"; +import type { ToolContext } from "../../tool-registry.js"; +import { buildOsShellTool } from "./shell.js"; + +/** + * ATO-1332: a saved deny rule applies to every command, not only to the + * ones the shell guard sends to the gate. A command the guard deems safe + * (`echo`) is refused on the rule, and a command run by a nested shell + * (`bash -c "rm -rf tmp"`) is matched like one on the line itself. + */ +describe.skipIf(process.platform === "win32")( + "os.shell.run with saved deny rules (ATO-1332)", + () => { + let dir: string; + let asked: ApprovalRequest[]; + let gate: ApprovalGate; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "atomic-shell-deny-rules-")); + asked = []; + gate = new ApprovalGate({ + level: 1, + emit: (req) => { + asked.push(req); + gate.resolve({ approvalId: req.approvalId, approved: true }); + }, + }); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + function withRules( + ...rules: Array> + ): void { + gate.setRules({ + list: () => + rules.map((r, i) => ({ + id: `r${i + 1}`, + createdAt: "", + source: "user" as const, + ...r, + })), + add: () => { + throw new Error("unused"); + }, + remove: () => false, + }); + } + + function ctx(): ToolContext { + return { + workingDir: dir, + sessionId: "s", + stepIndex: 0, + signal: new AbortController().signal, + }; + } + + function tool() { + return buildOsShellTool({ approvals: gate, approvalRequired: true }); + } + + async function denial( + args: Record, + ): Promise { + const err = await tool() + .run(args, ctx()) + .then( + () => null, + (e: unknown) => e, + ); + expect(err).toBeInstanceOf(ApprovalDeniedError); + return err as ApprovalDeniedError; + } + + it("a deny rule on `echo` refuses `echo hi`, which the guard deems safe", async () => { + withRules({ decision: "deny", commandPrefix: ["echo"] }); + const err = await denial({ cmd: "echo", args: ["hi"] }); + expect(err.byUser).toBe(false); + expect(err.message).toContain("blocked by your rule r1"); + expect(asked).toEqual([]); + }); + + it("a deny rule refuses a safe command even at full trust", async () => { + gate.setLevel(5); + withRules({ decision: "deny", commandPrefix: ["ls"] }); + await denial({ cmd: "ls", args: ["./"] }); + expect(asked).toEqual([]); + }); + + it('a deny rule on `rm` refuses `bash -c "rm -rf tmp"` without running it', async () => { + await mkdir(join(dir, "tmp")); + withRules({ decision: "deny", commandPrefix: ["rm"] }); + const err = await denial({ cmd: "bash", args: ["-c", "rm -rf tmp"] }); + expect(err.message).toContain("blocked by your rule r1"); + expect((await stat(join(dir, "tmp"))).isDirectory()).toBe(true); + expect(asked).toEqual([]); + }); + + it("a deny rule on `rm` refuses `sh -lc 'cd x && rm y'`", async () => { + withRules({ decision: "deny", commandPrefix: ["rm"] }); + await denial({ cmd: "sh -lc 'cd x && rm y'" }); + expect(asked).toEqual([]); + }); + + it("a safe command no deny rule names still runs unasked", async () => { + withRules({ decision: "deny", commandPrefix: ["rm"] }); + const result = await tool().run({ cmd: "echo", args: ["hi"] }, ctx()); + expect(result.status).toBe("ok"); + expect(asked).toEqual([]); + }); + + it("an ask rule does not make a safe command prompt", async () => { + withRules({ decision: "ask", commandPrefix: ["echo"] }); + const result = await tool().run({ cmd: "echo", args: ["hi"] }, ctx()); + expect(result.status).toBe("ok"); + expect(asked).toEqual([]); + }); + + it("an allow rule on `rm` does not cover the `rm` inside `bash -c`", async () => { + await mkdir(join(dir, "tmp")); + withRules({ decision: "allow", commandPrefix: ["rm"] }); + const result = await tool().run( + { cmd: "bash", args: ["-c", "rm -rf tmp"] }, + ctx(), + ); + // Asked (and approved by the test host): the allow rule did not settle it. + expect(result.status).toBe("ok"); + expect(asked).toHaveLength(1); + }); + }, +); diff --git a/src/tools/os/shell/shell.ts b/src/tools/os/shell/shell.ts index 68d9f2595..4c8005a14 100644 --- a/src/tools/os/shell/shell.ts +++ b/src/tools/os/shell/shell.ts @@ -12,6 +12,7 @@ import { withToolchainEnv } from "../../../sandbox/command-toolchain-env.js"; import { quoteCmdArg } from "../../../sandbox/shell-invocation.js"; import { requireApproval, + savedRuleDenies, savedRuleOverridesScope, type DangerousToolOptions, } from "../../../approval/dangerous-tool.js"; @@ -252,6 +253,15 @@ export function buildOsShellTool(options: OsShellToolOptions): ToolDefinition { // policy hands it back as a step for the operator (ATO-1223). const dangerous = isDangerousGuardRule(guardVerdict.rule); const commandSegments = shellRuleSegments(cmd, execArgs, useShell); + const ruleCheck = { + sessionId: ctx.sessionId, + tool: "os.shell.run", + category: "shell" as const, + commandSegments, + affectedResources: [cwd], + targetPaths, + workdir: ctx.workingDir, + }; const scopedByFanout = !dangerous && (options.approvals.fanoutScopes?.allows(ctx.sessionId, [cwd]) ?? @@ -264,16 +274,19 @@ export function buildOsShellTool(options: OsShellToolOptions): ToolDefinition { targetPaths, }) ?? false ) && - !savedRuleOverridesScope(options.approvals, { - sessionId: ctx.sessionId, - tool: "os.shell.run", - category: "shell", - commandSegments, - affectedResources: [cwd], - targetPaths, - workdir: ctx.workingDir, - }); - if (guardVerdict.action === "approval_required" && !scopedByFanout) { + !savedRuleOverridesScope(options.approvals, ruleCheck); + // A command the guard deems safe (`echo`, `ls`) runs without asking, + // but a saved deny rule still stops it: it goes to the gate, which + // refuses it on the rule before any level, grant or prompt + // (ATO-1332). Ask and allow rules keep to the gated path. + const deniedBySavedRule = + guardVerdict.action === "allow" && + options.approvalRequired && + savedRuleDenies(options.approvals, ruleCheck); + if ( + (guardVerdict.action === "approval_required" && !scopedByFanout) || + deniedBySavedRule + ) { // Shape grant unit: the bare binary name (lowercased), so `[a]` // and "Allow in this chat" cover exactly the argv[0] that would // run: `git`, not `./git` or a compound line that starts with From 3e6a8b2e18668e886bf1370e61e0bc23c3b9a40a Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:13:17 +0300 Subject: [PATCH 023/155] fix(overnight): resume a task only at the approval level it started under The approval level was set per process, not per task. An overnight run that denied actions needing approval, once interrupted, could be picked up by another process's scheduler at a higher level (and run what it had denied) or at a lower one (and wait for a person who is not there). The overnight command now stores its level on the task row, in a new nullable approval_level column added the same way as the budget columns (older binaries ignore it). The task runner leaves a row with a level for a process running at that level, logs it once, and the drain loop does not count it as a retry. ATO-1324 --- src/cli/overnight-command.test.ts | 7 ++++ src/cli/overnight-command.ts | 9 +++-- src/runtime/bootstrap.ts | 1 + .../composition/runtime-task-services.ts | 3 ++ src/tasks/task-budget.test.ts | 16 +++++++++ src/tasks/task-budget.ts | 30 ++++++++++++++++ src/tasks/task-runner.test.ts | 34 ++++++++++++++++++ src/tasks/task-runner.ts | 36 ++++++++++++++++++- src/tasks/task-store.ts | 19 ++++++++-- src/tasks/task-types.ts | 9 ++++- 10 files changed, 157 insertions(+), 7 deletions(-) diff --git a/src/cli/overnight-command.test.ts b/src/cli/overnight-command.test.ts index 7ba39e4bc..9b3eaba90 100644 --- a/src/cli/overnight-command.test.ts +++ b/src/cli/overnight-command.test.ts @@ -175,6 +175,13 @@ describe("overnightTaskInput", () => { expect(input).not.toHaveProperty("maxCostUsd"); }); + it("records the run's approval level on the row (ATO-1324)", () => { + const input = overnightTaskInput(args, "t-1", "s-1", { + agent: { task: { maxSteps: 1000, maxCostUsd: 0 } }, + }, 2); + expect(input.approvalLevel).toBe(2); + }); + it("takes agent.task.maxSteps from the loaded config", () => { const config = getConfig(); const input = overnightTaskInput(args, "t-1", "s-1", config); diff --git a/src/cli/overnight-command.ts b/src/cli/overnight-command.ts index 192302752..c85d25e94 100644 --- a/src/cli/overnight-command.ts +++ b/src/cli/overnight-command.ts @@ -159,6 +159,7 @@ export function overnightTaskInput( taskId: string, sessionId: string, config: { agent: { task: { maxSteps: number; maxCostUsd: number } } }, + approvalLevel?: number, ): TaskCreateInput { const capUsd = args.budgetUsd ?? config.agent.task.maxCostUsd; return { @@ -172,6 +173,9 @@ export function overnightTaskInput( maxDurationMs: args.durationMs, ...(capUsd > 0 ? { maxCostUsd: capUsd } : {}), ...(args.notify ? { notify: args.notify } : {}), + // Nobody is there to answer, so what this run denies or allows is + // fixed when it starts: a resume elsewhere must keep it (ATO-1324). + ...(approvalLevel !== undefined ? { approvalLevel } : {}), }; } @@ -279,12 +283,13 @@ async function runForeground(args: OvernightArgs, taskId: string): Promise void) | null = null; + const approvalLevel = resolveBootApprovalLevel(args.noApproval, config.agent.approvalLevel); const runtime = await createAgentRuntime({ // Tools resolve paths against the session's working dir, and a // session takes the runtime's: this is what makes --dir the run's // folder rather than a hint in the prompt. workingDir: args.dir, - approvalLevel: resolveBootApprovalLevel(args.noApproval, config.agent.approvalLevel), + approvalLevel, handlers: { onAgentEvent: (event, eventSessionId) => { if (sessionId === null || eventSessionId !== sessionId) return; @@ -327,7 +332,7 @@ async function runForeground(args: OvernightArgs, taskId: string): Promise telegramChannelForShutdown, logger, metrics, + approvalLevel: () => approvals.getLevel(), }); registerRuntimeFusionAndReadScope({ diff --git a/src/runtime/composition/runtime-task-services.ts b/src/runtime/composition/runtime-task-services.ts index 900ec7753..d5d95783e 100644 --- a/src/runtime/composition/runtime-task-services.ts +++ b/src/runtime/composition/runtime-task-services.ts @@ -63,6 +63,8 @@ export function createRuntimeTaskRunner(args: { resolveTelegram: () => TelegramChannel | null; logger: StructuredLogger; metrics: AgentMetrics; + /** The live approval level, so a task started under another is left alone (ATO-1324). */ + approvalLevel?: () => number; }): TaskRunner { const { config, taskStore, runTurn, sessionStore, createSession, toolRegistry, resolveTelegram, logger, metrics } = args; const taskRunner = new TaskRunner({ @@ -105,6 +107,7 @@ export function createRuntimeTaskRunner(args: { snapshotCadence: config.statusSnapshot ?? null, logger, metrics, + ...(args.approvalLevel ? { approvalLevel: args.approvalLevel } : {}), // Continue a run a crash, a kill or a restart cut off from its // session's checkpoint (ATO-1109). resume: { diff --git a/src/tasks/task-budget.test.ts b/src/tasks/task-budget.test.ts index c2cc06cb5..b07124783 100644 --- a/src/tasks/task-budget.test.ts +++ b/src/tasks/task-budget.test.ts @@ -1,11 +1,13 @@ import { describe, expect, it } from "vitest"; import { + approvalLevelAllows, formatRunTime, remainingCostUsd, remainingDurationMs, validateMaxCostUsd, validateMaxDurationMs, + validateApprovalLevel, } from "./task-budget.js"; import { TaskValidationError } from "./task-types.js"; @@ -44,4 +46,18 @@ describe("task budget rules", () => { expect(validateMaxDurationMs(60_000)).toBe(60_000); expect(() => validateMaxDurationMs(-1)).toThrow(TaskValidationError); }); + + it("keeps a recorded approval level to processes at that level (ATO-1324)", () => { + expect(validateApprovalLevel(undefined)).toBeNull(); + expect(validateApprovalLevel(3)).toBe(3); + expect(() => validateApprovalLevel(0)).toThrow(TaskValidationError); + expect(() => validateApprovalLevel(6)).toThrow(TaskValidationError); + expect(() => validateApprovalLevel(2.5)).toThrow(TaskValidationError); + expect(approvalLevelAllows({ approvalLevel: null }, 5)).toBe(true); + expect(approvalLevelAllows({}, 1)).toBe(true); + expect(approvalLevelAllows({ approvalLevel: 2 }, 2)).toBe(true); + expect(approvalLevelAllows({ approvalLevel: 2 }, 5)).toBe(false); + expect(approvalLevelAllows({ approvalLevel: 5 }, 2)).toBe(false); + expect(approvalLevelAllows({ approvalLevel: 2 }, undefined)).toBe(true); + }); }); diff --git a/src/tasks/task-budget.ts b/src/tasks/task-budget.ts index 7cc784760..ab601a5fa 100644 --- a/src/tasks/task-budget.ts +++ b/src/tasks/task-budget.ts @@ -20,6 +20,9 @@ export const TASK_BUDGET_COLUMNS = [ { name: "spent_usd", type: "REAL" }, { name: "elapsed_ms", type: "INTEGER" }, { name: "stop_cause", type: "TEXT" }, + // The approval level the task was started under (ATO-1324). Not a + // budget, but the same kind of nullable column old binaries ignore. + { name: "approval_level", type: "INTEGER" }, ] as const; /** @@ -73,6 +76,33 @@ export function validateMaxDurationMs(raw: number | null | undefined): number | return raw; } +/** `approval_level`: an integer 1..5, or `null` for "any process may run it". */ +export function validateApprovalLevel(raw: number | null | undefined): number | null { + if (raw === undefined || raw === null) return null; + if (!Number.isInteger(raw) || raw < 1 || raw > 5) { + throw new TaskValidationError( + "approvalLevel", + `approvalLevel must be an integer from 1 to 5, got ${JSON.stringify(raw)}`, + ); + } + return raw; +} + +/** + * Whether a process at `processLevel` may claim `task` (ATO-1324). A + * task that recorded the level it was started under runs only at that + * level: an interrupted overnight run that denied what needed approval + * must not be resumed by a process that would approve it, nor by one + * that would ask a person who is not there. + */ +export function approvalLevelAllows( + task: Pick, + processLevel: number | undefined, +): boolean { + const wanted = task.approvalLevel ?? null; + return wanted === null || processLevel === undefined || processLevel === wanted; +} + /** * What the next run of `task` may spend: * - `undefined`: the task has no ceiling of its own (config decides); diff --git a/src/tasks/task-runner.test.ts b/src/tasks/task-runner.test.ts index 44bf15d33..3889ad91f 100644 --- a/src/tasks/task-runner.test.ts +++ b/src/tasks/task-runner.test.ts @@ -106,6 +106,40 @@ describe("TaskRunner", () => { ]); }); + it("leaves a task started under another approval level for that level (ATO-1324)", async () => { + const { runtime, calls } = fakeRuntime(); + let level = 5; + const runner = new TaskRunner({ + store, + runtime, + sessionLoader: fakeSessionLoader(session), + defaultMaxSteps: 5, + backoff: { initialMs: 1, maxMs: 10 }, + enabled: true, + runOnCreate: false, + sleep: async () => undefined, + approvalLevel: () => level, + }); + const t = store.create({ + sessionId: session.id, + userMessage: "overnight work", + origin: "cli", + maxAttempts: 1, + approvalLevel: 2, + }); + expect(store.get(t.id)?.approvalLevel).toBe(2); + const skipped = await runner.runOne(t.id); + expect(skipped?.status).toBe("pending"); + expect(store.get(t.id)?.attempts).toBe(0); + const drained = await runner.drainPending(); + expect(drained.drained).toBe(0); + expect(calls).toEqual([]); + level = 2; + const final = await runner.runOne(t.id); + expect(final?.status).toBe("completed"); + expect(calls).toHaveLength(1); + }); + it("create() refuses an interval under tasks.minIntervalMs and accepts one at it (#548)", () => { const { runtime } = fakeRuntime(); const runner = new TaskRunner({ diff --git a/src/tasks/task-runner.ts b/src/tasks/task-runner.ts index b6cd0be63..a84cfaa7e 100644 --- a/src/tasks/task-runner.ts +++ b/src/tasks/task-runner.ts @@ -26,7 +26,7 @@ import { type TaskRunSummaryCollector, } from "./task-run-summary.js"; import { isRecurring, resolveScheduledFor } from "./task-schedule.js"; -import { formatRunTime, remainingCostUsd, remainingDurationMs } from "./task-budget.js"; +import { approvalLevelAllows, formatRunTime, remainingCostUsd, remainingDurationMs } from "./task-budget.js"; import { formatUsd } from "../llm/provider/usage-cost.js"; import { startTaskSnapshots, type TaskSnapshotSink } from "./task-snapshot.js"; import { @@ -149,6 +149,12 @@ export interface TaskRunnerOptions { snapshotSink?: TaskSnapshotSink; snapshotCadence?: { everySteps: number; everyMs: number } | null; logger?: StructuredLogger; + /** + * This process's live approval level (ATO-1324). A task that recorded + * the level it was started under is left for a process at that level. + * Omitted (tests, hosts without a gate): every task is runnable. + */ + approvalLevel?: () => number; metrics?: AgentMetrics; /** * Test seam for the inter-attempt sleep. Production uses the real @@ -231,6 +237,8 @@ export class TaskRunner { private readonly runs = new Map(); /** Set by `stop`: nothing is claimed after it. */ private stopping = false; + /** Tasks already logged as left for a process at another approval level. */ + private readonly levelSkipsLogged = new Set(); /** Summary collectors of the runs in progress, for `liveStatus`. */ private readonly live = new Map< string, @@ -349,6 +357,11 @@ export class TaskRunner { if (!task) return null; if (task.status !== "pending") return task; + // Started under another approval level (an interrupted overnight + // run): leave the row for a process at that level instead of + // resuming it with more, or fewer, actions allowed. + if (this.leftForOtherLevel(task)) return task; + const resolvedTask = this.ensureSessionBeforeClaim(task); if (!resolvedTask) { return this.options.store.get(task.id); @@ -895,6 +908,9 @@ export class TaskRunner { let current: TaskRecord | null = task; while (current && current.status === "pending") { if (signal?.aborted || this.stopping) return; + // `runOne` hands such a row back untouched; counting it as a + // retry would spin this loop on it (ATO-1324). + if (this.leftForOtherLevel(current)) break; const before = current; current = await this.runOne(current.id, signal); outcome.drained += 1; @@ -933,6 +949,24 @@ export class TaskRunner { } } + /** + * True when `task` was started under another approval level than this + * process runs at (ATO-1324). Logged once per task, not every tick. + */ + private leftForOtherLevel(task: TaskRecord): boolean { + const processLevel = this.options.approvalLevel?.(); + if (approvalLevelAllows(task, processLevel)) return false; + if (!this.levelSkipsLogged.has(task.id)) { + this.levelSkipsLogged.add(task.id); + this.options.logger?.info("task left for a process at its own approval level", { + taskId: task.id, + taskApprovalLevel: task.approvalLevel ?? null, + processApprovalLevel: processLevel ?? null, + }); + } + return true; + } + private handleFailure( task: TaskRecord, category: LlmFailureCategory, diff --git a/src/tasks/task-store.ts b/src/tasks/task-store.ts index 3c61a3427..7fcdaf266 100644 --- a/src/tasks/task-store.ts +++ b/src/tasks/task-store.ts @@ -20,6 +20,7 @@ import { ensureBudgetColumns, validateMaxCostUsd, validateMaxDurationMs, + validateApprovalLevel, } from "./task-budget.js"; import { parseScheduleRow, serializeScheduleValue } from "./task-schedule.js"; import { @@ -90,6 +91,11 @@ export interface TaskCreateInput { */ maxCostUsd?: number | null; maxDurationMs?: number | null; + /** + * Run only at this approval level (ATO-1324). Omit / `null` to let + * any process run it at its own. + */ + approvalLevel?: number | null; /** * Optional explicit id. Used by tests to make assertions; production * callers always let the store generate one. @@ -136,6 +142,7 @@ interface TaskRow { spent_usd?: number | null; elapsed_ms?: number | null; stop_cause?: string | null; + approval_level?: number | null; } interface RunningRow { @@ -211,10 +218,10 @@ export class TaskStore { this.budgetsUnavailable = ensureBudgetColumns(this.db); const withBudgets = this.budgetsUnavailable === null; const budgetColumns = withBudgets - ? ",\n max_cost_usd, max_duration_ms, spent_usd, elapsed_ms, stop_cause" + ? ",\n max_cost_usd, max_duration_ms, spent_usd, elapsed_ms, stop_cause, approval_level" : ""; const budgetValues = withBudgets - ? ",\n @max_cost_usd, @max_duration_ms, @spent_usd, @elapsed_ms, @stop_cause" + ? ",\n @max_cost_usd, @max_duration_ms, @spent_usd, @elapsed_ms, @stop_cause, @approval_level" : ""; this.ownerProbe = options.ownerProbe ?? currentTurnOwnerProbe(); this.dbIdentity = databaseIdentity(options.dbFile); @@ -414,8 +421,9 @@ export class TaskStore { const notify = validateNotify(input.notify); const maxCostUsd = validateMaxCostUsd(input.maxCostUsd); const maxDurationMs = validateMaxDurationMs(input.maxDurationMs); + const approvalLevel = validateApprovalLevel(input.approvalLevel); if ( - (maxCostUsd !== null || maxDurationMs !== null) && + (maxCostUsd !== null || maxDurationMs !== null || approvalLevel !== null) && this.budgetsUnavailable !== null ) { // Never drop a budget the caller asked for: an overnight run @@ -460,6 +468,7 @@ export class TaskStore { spent_usd: null, elapsed_ms: null, stop_cause: null, + approval_level: approvalLevel, }; this.insertStmt.run(this.bindRow(row)); return rowToRecord(row); @@ -866,6 +875,7 @@ export class TaskStore { spent_usd: row.spent_usd ?? null, elapsed_ms: row.elapsed_ms ?? null, stop_cause: row.stop_cause ?? null, + approval_level: row.approval_level ?? null, }; } const bare = { ...row }; @@ -874,6 +884,7 @@ export class TaskStore { delete bare.spent_usd; delete bare.elapsed_ms; delete bare.stop_cause; + delete bare.approval_level; return bare; } @@ -946,6 +957,7 @@ function rowToRecord(row: TaskRow): TaskRecord { spentUsd: row.spent_usd ?? null, elapsedMs: row.elapsed_ms ?? null, stopCause: row.stop_cause ?? null, + approvalLevel: row.approval_level ?? null, }; } @@ -979,6 +991,7 @@ function recordToRow(record: TaskRecord): TaskRow { spent_usd: record.spentUsd ?? null, elapsed_ms: record.elapsedMs ?? null, stop_cause: record.stopCause ?? null, + approval_level: record.approvalLevel ?? null, }; } diff --git a/src/tasks/task-types.ts b/src/tasks/task-types.ts index 5b0567f4b..d187c0e0a 100644 --- a/src/tasks/task-types.ts +++ b/src/tasks/task-types.ts @@ -160,6 +160,12 @@ export interface TaskRecord { * `null` when the agent finished on its own. */ stopCause: string | null; + /** + * The approval level (1..5) the task was started under, for a task + * that must not be resumed under another one (ATO-1324): `overnight` + * records it. `null`/absent: any process may run it at its own level. + */ + approvalLevel?: number | null; } /** @@ -193,7 +199,8 @@ export class TaskValidationError extends Error { | "schedule" | "notify" | "maxCostUsd" - | "maxDurationMs", + | "maxDurationMs" + | "approvalLevel", message: string, ) { super(message); From 84338fdf8ca854d2bfb47f38c8f5386f65b800d3 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:13:29 +0300 Subject: [PATCH 024/155] fix(fusion): count worker spend once and where the person sees it - The composer's $ readout now includes Fusion workers: a worker's throwaway session forwards its calls to the session that delegated, priced by the same rules (unpriced stays unpriced, local stays local). Escalation and judge calls, which go straight to the provider, are counted on the delegating session too. (ATO-1388) - A task re-run on the escalation model keeps the local attempt's usage on its row, so the status header and the checkpoint no longer price the re-run a second time on top of its escalation spend. (ATO-1319) - A split's subtask finds its root through the task tree's parent links, so its spend and side effects reach the run checkpoint and the resume budget. (ATO-1320) - The overnight doc and `task create --max-cost-usd` help say that Fusion workers are not in the money cap and point to llm.runMode.fusion.turnBudgetUsd. (ATO-1315) --- docs/user/overnight.md | 2 +- src/cli/task-command.ts | 5 +- src/runtime/bootstrap.ts | 10 +++- .../composition/runtime-tool-catalog.ts | 31 +++++++++++- src/session/usage-meter.test.ts | 18 +++++++ src/session/usage-meter.ts | 22 +++++++++ src/tools/fusion/escalation.test.ts | 47 +++++++++++++++++++ src/tools/fusion/escalation.ts | 5 ++ src/tools/fusion/fusion-delegate.ts | 20 ++++++-- src/tools/fusion/run-checkpoint.test.ts | 29 ++++++++++++ src/tools/fusion/run-checkpoint.ts | 37 ++++++++++++++- src/tools/fusion/worker-runner.ts | 10 ++++ 12 files changed, 224 insertions(+), 12 deletions(-) diff --git a/docs/user/overnight.md b/docs/user/overnight.md index 3d3c2bc7b..393e82ea1 100644 --- a/docs/user/overnight.md +++ b/docs/user/overnight.md @@ -32,7 +32,7 @@ Nobody is there to answer, so nobody is asked. An action that needs approval is ## Limits -- **Money.** When the cap is reached the agent writes a short summary and stops with `stopped: spent $X of $Y budget`. Spend is an estimate from the provider's price list and counts the main agent loop only (not memory, compaction or Fusion workers), so a run can end slightly over the cap. Local models count as $0. +- **Money.** When the cap is reached the agent writes a short summary and stops with `stopped: spent $X of $Y budget`. Spend is an estimate from the provider's price list and counts the main agent loop only (not memory, compaction or Fusion workers), so a run can end slightly over the cap. Local models count as $0. In Fusion mode, paid workers (a cloud worker or role model, an escalation, the judge) are not in this figure at all and do not stop the run, so with them the run can end well over the cap. Cap them per turn with `llm.runMode.fusion.turnBudgetUsd`. - **Time.** The run stops after `--hours`. If both caps are set, the first one reached ends the run. - **Loops.** The loop guard leaves long normal work alone. Re-running the tests after an edit or polling a long build is fine. A poll that keeps getting the same answer is stopped after 20 identical answers, and the guard stops a real loop sooner. diff --git a/src/cli/task-command.ts b/src/cli/task-command.ts index 9316c1003..cd2138906 100644 --- a/src/cli/task-command.ts +++ b/src/cli/task-command.ts @@ -37,7 +37,10 @@ const HELP = " channel is down or unpaired).", " --max-cost-usd stops a run once its estimated spend", " reaches that many dollars, with a closing summary;", - " local (unpriced) models count as $0. --max-duration", + " local (unpriced) models count as $0. Fusion workers", + " (cloud workers, escalation, judge) are not counted;", + " cap them with llm.runMode.fusion.turnBudgetUsd.", + " --max-duration", " takes 90s, 30m, 8h or 1h30m (a bare number is seconds).", " Both default to agent.task.* in config.", " --at takes Unix ms or strict ISO-8601. A date-time", diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index e497b92a3..a7a1323a3 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -750,7 +750,14 @@ export async function createAgentRuntime( logger, localProfile, connectedLocal, providerRegistry, ); - const { createSession, createEphemeralSession } = createRuntimeSessionFactories(workingDir, sessionStore, ensureRecorder); + const { createSession, createEphemeralSession: createWorkerSession } = createRuntimeSessionFactories(workingDir, sessionStore, ensureRecorder); + // A fusion worker's calls count on the session that spawned it, so the + // composer's `$` readout includes what the workers spent (ATO-1388). + const createEphemeralSession: typeof createWorkerSession = (meta, inheritedDir) => { + const worker = createWorkerSession(meta, inheritedDir); + sessionUsage.forward(worker.id, meta.parentSessionId); + return worker; + }; const previewPrompt = createRuntimePromptPreview(config, promptPreviewDeps); const { executeTurn, steer, runTurn, controlGoal } = createRuntimeTurnService(config, { @@ -771,6 +778,7 @@ export async function createAgentRuntime( declaredInputs, prepareLocalLink, emitAgentLoopEventFor, resolveCurrentRunMode, providerRegistry, llama, approvals, dangerous, slotManager, workingDir, logger, turnBudget: fusionTurnBudget, + sessionUsage, liveWorkers: fusionLiveWorkers, runCheckpoints: fusionRuns, steerWorker: (sessionId, text) => steeringInbox.push(sessionId, text), diff --git a/src/runtime/composition/runtime-tool-catalog.ts b/src/runtime/composition/runtime-tool-catalog.ts index 677bda242..60d66e6b1 100644 --- a/src/runtime/composition/runtime-tool-catalog.ts +++ b/src/runtime/composition/runtime-tool-catalog.ts @@ -12,6 +12,8 @@ import { resolveLlmConfig } from "../../llm/provider/index.js"; import type { ResolvedRunMode } from "../../llm/run-mode/index.js"; import { visionCapableAlternatives } from "../../llm/provider/model-vision.js"; import { resolveModelPricingFor } from "../resolve-model-pricing.js"; +import type { SessionUsageLedger } from "../../session/usage-meter.js"; +import { isLocalServerUrl } from "../../llm/provider/openai/local-server-url.js"; import { ToolRegistry } from "../../tools/tool-registry.js"; import { knownSecretsFromEnv, type KnownSecret } from "../../error-reporting/secret-detector.js"; import { isSecretEnvName } from "../../sandbox/command-env.js"; @@ -642,6 +644,8 @@ export function registerRuntimeFusionAndReadScope(input: { workingDir: string; logger: StructuredLogger; turnBudget?: FusionDelegateDeps["turnBudget"]; + /** The composer's `$` readout: escalation and judge calls land here (ATO-1388). */ + sessionUsage?: Pick; liveWorkers?: FusionDelegateDeps["liveWorkers"]; steerWorker?: FusionDelegateDeps["steerWorker"]; orchestratorTurnSpendUsd?: FusionDelegateDeps["orchestratorTurnSpendUsd"]; @@ -650,7 +654,7 @@ export function registerRuntimeFusionAndReadScope(input: { const { config, toolRegistry, runTurn, createEphemeralSession, resolveOriginalRequest, declaredInputs, prepareLocalLink, emitAgentLoopEventFor, resolveCurrentRunMode, providerRegistry, llama, approvals, dangerous, slotManager, workingDir, logger, turnBudget, - orchestratorTurnSpendUsd, liveWorkers, steerWorker, runCheckpoints } = input; + orchestratorTurnSpendUsd, liveWorkers, steerWorker, runCheckpoints, sessionUsage } = input; // The orchestrator's fan-out. Registered UNCONDITIONALLY: the tool @@ -701,7 +705,7 @@ export function registerRuntimeFusionAndReadScope(input: { // Escalation (ATO-942): one completion pinned to the configured // escalation provider — never the fallback chain, so it cannot // land on a model the operator did not pick for it. - escalationComplete: async ({ providerId, prompt, maxOutputTokens, signal }) => { + escalationComplete: async ({ providerId, prompt, maxOutputTokens, signal, sessionId }) => { const provider = providerRegistry.getProvider(providerId); if (provider === undefined) { throw new Error(`escalation provider ${providerId} is not configured`); @@ -711,6 +715,29 @@ export function registerRuntimeFusionAndReadScope(input: { maxOutputTokens, signal, }); + // These calls go straight to the provider, past the seam that + // feeds the session's readout, so they are counted here on the + // session that delegated (ATO-1388). Priced like any other call: + // no known price stays "unpriced", a local server stays local. + if (sessionUsage !== undefined && sessionId !== undefined) { + try { + const resolved = resolveLlmConfig(getConfig()); + const entry = resolved.providers.find((p) => p.id === providerId); + const model = resolveModelPricingFor(resolved, result.modelId, providerId); + sessionUsage.record(sessionId, { + providerId, + modelId: result.modelId, + ...(result.usage === undefined ? {} : { usage: result.usage }), + ...(model === undefined ? {} : { model }), + local: + entry !== undefined && + (entry.kind === "llama-server" || isLocalServerUrl(entry.baseUrl ?? entry.url)), + plan: entry?.kind === "subscription-cli", + }); + } catch { + // The readout never takes an escalation down. + } + } return { content: result.content, ...(result.usage === undefined ? {} : { usage: result.usage }), diff --git a/src/session/usage-meter.test.ts b/src/session/usage-meter.test.ts index 1ffe67f08..dd6c54c64 100644 --- a/src/session/usage-meter.test.ts +++ b/src/session/usage-meter.test.ts @@ -77,6 +77,24 @@ describe("SessionUsageLedger", () => { expect(seen.map(([id]) => id)).toEqual(["a", "a", "b"]); }); + it("counts a fusion worker's calls on the session that spawned it (ATO-1388)", () => { + const seen: string[] = []; + const ledger = new SessionUsageLedger({ now: () => NOW, onChange: (id) => seen.push(id) }); + ledger.record("s-parent", { providerId: "openrouter", modelId: "m", usage, model: priced, local: false }); + ledger.forward("s-w-1", "s-parent"); + ledger.record("s-w-1", { providerId: "openrouter", modelId: "m", usage, model: priced, local: false }); + ledger.record("s-w-1", { providerId: "openrouter", modelId: "x", usage, local: false }); + ledger.record("s-w-1", { providerId: "llama", modelId: "q", usage, local: true }); + const parent = ledger.get("s-parent"); + expect(parent?.costUsd).toBeCloseTo(9); + expect(parent?.pricedCalls).toBe(2); + expect(parent?.unpricedCalls).toBe(1); + expect(parent?.localCalls).toBe(1); + expect(formatSessionCost(parent)).toBe("$9.00+"); + expect(ledger.get("s-w-1")).toBeUndefined(); + expect(seen.every((id) => id === "s-parent")).toBe(true); + }); + it("continues from what a reopened session persisted, once", () => { const ledger = new SessionUsageLedger({ now: () => NOW }); const stored: SessionUsageState = { ...EMPTY_SESSION_USAGE, costUsd: 1, pricedCalls: 1, costSources: ["catalog"] }; diff --git a/src/session/usage-meter.ts b/src/session/usage-meter.ts index 19c63367d..3b96d83b0 100644 --- a/src/session/usage-meter.ts +++ b/src/session/usage-meter.ts @@ -176,6 +176,8 @@ export function activeLimits( export class SessionUsageLedger { private readonly sessions = new Map(); private readonly readings = new Map(); + /** Fusion worker session id → the session whose turn spawned it (ATO-1388). */ + private readonly forwards = new Map(); private readonly onChange: ((sessionId: string, state: SessionUsageState) => void) | undefined; private readonly now: () => number; private readonly maxSessions: number; @@ -199,7 +201,27 @@ export class SessionUsageLedger { this.remember(sessionId, state); } + /** + * Count a fusion worker's calls on the session that spawned it + * (ATO-1388). A worker runs in a throwaway session nobody sees and + * nothing saves, so its spend recorded there never reached the + * composer readout: the person saw less than they spent. Once linked, + * every call recorded under `workerSessionId` lands on + * `parentSessionId` with the same pricing rules (unpriced stays + * unpriced, local stays local). Kept bounded like the sessions map. + */ + forward(workerSessionId: string, parentSessionId: string): void { + if (workerSessionId === parentSessionId) return; + this.forwards.delete(workerSessionId); + this.forwards.set(workerSessionId, parentSessionId); + if (this.forwards.size > this.maxSessions) { + const oldest = this.forwards.keys().next().value; + if (oldest !== undefined) this.forwards.delete(oldest); + } + } + record(sessionId: string, input: UsageRecordInput): SessionUsageState { + sessionId = this.forwards.get(sessionId) ?? sessionId; const now = this.now(); for (const w of input.rateLimits ?? []) { this.readings.set(`${input.providerId}/${w.scope}/${w.name}`, { diff --git a/src/tools/fusion/escalation.test.ts b/src/tools/fusion/escalation.test.ts index ff3572ce8..c7d4c1dee 100644 --- a/src/tools/fusion/escalation.test.ts +++ b/src/tools/fusion/escalation.test.ts @@ -213,6 +213,53 @@ describe("advisor escalation of a stuck worker (ATO-942)", () => { }); }); +describe("the re-run's money is counted once (ATO-1319)", () => { + it("keeps the local attempt's usage on the row and counts the re-run as escalation spend", async () => { + const completed = ( + hook: TurnOptions["eventHook"], + promptTokens: number, + completionTokens: number, + ): void => + hook?.({ + type: "llm_event", + event: { + type: "llm_completed", + completion: { + content: "", + usage: { promptTokens, completionTokens, totalTokens: promptTokens + completionTokens }, + }, + }, + } as never); + const h = harness(async (_s, options) => { + if (options.providerId === "cloud") { + completed(options.eventHook, 10_000, 1_000); + return done(); + } + completed(options.eventHook, 100, 10); + for (let i = 0; i < 3; i += 1) sameStep(options.eventHook, i); + await flush(); + for (let i = 3; i < 6; i += 1) sameStep(options.eventHook, i); + return { + session: createEmptySessionState({ id: "s-x", workingDir: "/repo" }), + reason: options.signal?.aborted ? "cancelled" : "reply", + stepCount: 6, + } as RunTurnResult; + }); + const escalationSpend: number[] = []; + const [row] = await runWorkerTasks(h.deps, { + ...BASE, + tasks: TASK, + signal: new AbortController().signal, + escalation: { target: { ...TARGET, replace: true, maxUsd: 10 }, pricing: PRICING }, + onEscalationSpend: (_t, usd) => escalationSpend.push(usd), + }); + expect(row!.usage?.promptTokens).toBe(100); + expect(row!.usage?.completionTokens).toBe(10); + // advisor call(s) plus the re-run's one completion, each counted once + expect(escalationSpend.some((usd) => Math.abs(usd - (10_000 * 1 + 1_000 * 4) / 1e6) < 1e-9)).toBe(true); + }); +}); + describe("the person's stop and escalate on one worker (ATO-66)", () => { it("escalates a running worker on request and stops it alone", async () => { const live = new FusionLiveWorkers(); diff --git a/src/tools/fusion/escalation.ts b/src/tools/fusion/escalation.ts index 45357ee68..6a227c972 100644 --- a/src/tools/fusion/escalation.ts +++ b/src/tools/fusion/escalation.ts @@ -40,6 +40,11 @@ export type EscalationComplete = (input: { prompt: string; maxOutputTokens: number; signal: AbortSignal; + /** + * The session that delegated, so the call counts in its spend readout + * (ATO-1388). Set by `fusion.delegate`, never by the callers below it. + */ + sessionId?: string; }) => Promise<{ content: string; usage?: CompletionUsage }>; export function renderAdvisorPrompt(input: { diff --git a/src/tools/fusion/fusion-delegate.ts b/src/tools/fusion/fusion-delegate.ts index 9482f045e..68bf72348 100644 --- a/src/tools/fusion/fusion-delegate.ts +++ b/src/tools/fusion/fusion-delegate.ts @@ -649,6 +649,14 @@ export function buildFusionDelegateTool( const runSignal = capped ? AbortSignal.any([ctx.signal, budgetAbort.signal]) : ctx.signal; + // Escalation and judge calls name the delegating session, so they + // count in its spend readout like the workers' own calls (ATO-1388). + const sideComplete: FusionDelegateDeps["escalationComplete"] = + deps.escalationComplete === undefined + ? undefined + : (call) => deps.escalationComplete!({ ...call, sessionId: ctx.sessionId }); + const workerDeps: FusionDelegateDeps = + sideComplete === undefined ? deps : { ...deps, escalationComplete: sideComplete }; // The judge's acceptance lists (ATO-944), built before any work so // every brief carries its own, read-only. Off: no list, no suffix. @@ -719,6 +727,8 @@ export function buildFusionDelegateTool( ? undefined : new TaskTreeState(mode.taskTree, workerModel); tree?.addRoots(parsed.tasks, parsed.contract); + // A split's subtasks count on their root in the checkpoint (ATO-1320). + if (tree !== undefined) run?.useParents((id) => tree.parentOf(id)); // One of the orchestrator's own tasks whose worker proposed a split. const awaitsExpansion = (row: WorkerTaskResult): boolean => tree !== undefined && @@ -783,8 +793,8 @@ export function buildFusionDelegateTool( ): Promise => runWorkerTasks( ctx.modelMode === "cloud" - ? { ...deps, workingDir: ctx.workingDir, inheritWorkspace: true } - : deps, + ? { ...workerDeps, workingDir: ctx.workingDir, inheritWorkspace: true } + : workerDeps, { ...(originalRequest === undefined ? {} : { originalRequest }), ...(contract === undefined ? {} : { contract }), @@ -846,8 +856,8 @@ export function buildFusionDelegateTool( const onlyRole = roles.size === 1 ? [...roles][0] : "-"; return runWorkerTasks( ctx.modelMode === "cloud" - ? { ...deps, workingDir: ctx.workingDir, inheritWorkspace: true } - : deps, + ? { ...workerDeps, workingDir: ctx.workingDir, inheritWorkspace: true } + : workerDeps, { ...(originalRequest === undefined ? {} : { originalRequest }), ...(contract === undefined ? {} : { contract }), @@ -1211,7 +1221,7 @@ export function buildFusionDelegateTool( workingDir: ctx.workingDir, target: judgeTarget, pricing: judgePricing, - complete: deps.escalationComplete!, + complete: sideComplete!, signal: ctx.signal, }); if (verdict.costUsd !== null && verdict.costUsd > 0) { diff --git a/src/tools/fusion/run-checkpoint.test.ts b/src/tools/fusion/run-checkpoint.test.ts index 66754948a..bb283b55f 100644 --- a/src/tools/fusion/run-checkpoint.test.ts +++ b/src/tools/fusion/run-checkpoint.test.ts @@ -189,6 +189,35 @@ describe("fusion run checkpoints (ATO-617)", () => { expect(saved.sideEffectTools).toEqual(["os.shell.run"]); }); + it("counts a split's subtask on its root through the tree, not the id (ATO-1320)", () => { + const store = new FusionRunStore({ dir }); + const run = store.start({ + sessionId: "s-1", + workingDir: "/repo", + args: { tasks: [{ id: "a/b", title: "Root", instructions: "x" }] }, + tasks: [{ id: "a/b", title: "Root" }], + waveCount: 1, + }); + const parents = new Map([ + ["a/b/c1", "a/b"], + ["a/b/c1/d", "a/b/c1"], + ]); + run.useParents((id) => parents.get(id)); + run.noteTool("a/b/c1/d", "os.shell.run"); + run.recordTask( + row("a/b/c1", { + tools: { calls: 1, errors: 0, writes: 0, byTool: { "web.fetch": 1 } }, + }), + 0.4, + ); + const saved = store.load(run.runId)!; + expect(saved.spentUsd).toBeCloseTo(0.4); + expect(saved.tasks[0]!.sideEffects).toBe(true); + expect(saved.tasks[0]!.sideEffectTools).toContain("os.shell.run"); + // The root's own row is written when its split closes, not by a child. + expect(saved.tasks[0]!.result).toBeUndefined(); + }); + it("reads side effects off a finished row's tool counts too", () => { const store = new FusionRunStore({ dir }); const run = start(store); diff --git a/src/tools/fusion/run-checkpoint.ts b/src/tools/fusion/run-checkpoint.ts index 810e6a8f2..cadab0f23 100644 --- a/src/tools/fusion/run-checkpoint.ts +++ b/src/tools/fusion/run-checkpoint.ts @@ -342,6 +342,8 @@ export class FusionRunStore { /** One run's checkpoint as the fan-out writes it. */ export class FusionRunRecorder { private ended = false; + /** The task tree's parent links (ATO-1320); absent without a tree. */ + private parentOf: ((taskId: string) => string | undefined) | undefined; constructor( private readonly store: FusionRunStore, @@ -357,6 +359,29 @@ export class FusionRunRecorder { return structuredClone(this.checkpoint); } + /** + * How a subtask of a split finds the top-level task it belongs to + * (ATO-1320). Only the orchestrator's own tasks have a row here; a + * child's spend and side effects count on its root, found through + * the tree's parent links rather than by cutting the id at `/`, + * since a root's own id may contain one. + */ + useParents(parentOf: (taskId: string) => string | undefined): void { + this.parentOf = parentOf; + } + + /** The checkpoint row `taskId` counts on: its own, or its root's. */ + private rowFor(taskId: string): FusionRunCheckpoint["tasks"][number] | undefined { + let id: string | undefined = taskId; + for (let hops = 0; id !== undefined && hops < 64; hops += 1) { + const current: string = id; + const task = this.checkpoint.tasks.find((t) => t.id === current); + if (task !== undefined) return task; + id = this.parentOf?.(current); + } + return undefined; + } + /** * A worker started `tool`. Written only when it is the task's first * side-effecting call: a worker killed right after it is still known @@ -364,7 +389,7 @@ export class FusionRunRecorder { */ noteTool(taskId: string, tool: string): void { if (this.ended || !toolHasSideEffects(tool)) return; - const task = this.checkpoint.tasks.find((t) => t.id === taskId); + const task = this.rowFor(taskId); if (task === undefined || task.sideEffectTools.includes(tool)) return; const first = !task.sideEffects; task.sideEffects = true; @@ -375,7 +400,8 @@ export class FusionRunRecorder { /** A task finished: its row and its spend are written at once. */ recordTask(result: WorkerTaskResult, costUsd = 0): void { if (this.ended) return; - const task = this.checkpoint.tasks.find((t) => t.id === result.id); + const own = this.checkpoint.tasks.find((t) => t.id === result.id); + const task = own ?? this.rowFor(result.id); if (task === undefined) return; // The finished row's own tool counts also decide, for a worker whose // tool events were not seen (a test harness, an embedder). @@ -386,6 +412,13 @@ export class FusionRunRecorder { } } const spend = Number.isFinite(costUsd) && costUsd > 0 ? costUsd : 0; + if (own === undefined) { + // A subtask of a split (ATO-1320): its spend and side effects + // count on its root, whose own row is written when the split closes. + this.checkpoint.spentUsd += spend; + this.flush(); + return; + } task.result = { status: result.status, reply: result.reply.slice(0, CHECKPOINT_REPLY_CHARS), diff --git a/src/tools/fusion/worker-runner.ts b/src/tools/fusion/worker-runner.ts index c0d72619f..4fff3d03a 100644 --- a/src/tools/fusion/worker-runner.ts +++ b/src/tools/fusion/worker-runner.ts @@ -1507,6 +1507,12 @@ async function runOneTask( `re-running the task on ${targetLabel}: the advisor's hint did not unstick the local worker`, ); const localFindings = collector.findings(); + // The re-run's cost is counted as escalation spend as it lands + // (`onEscalationSpend`). The row keeps the local attempt's usage, so + // the status header and the checkpoint, which price a row's usage + // at the worker's rate, do not count the re-run a second time + // (ATO-1319). + const localUsage = result.usage; try { const turn = await deps.runTurn( replaceSession, @@ -1580,6 +1586,10 @@ async function runOneTask( unregisterReplace?.(); disarmWorkerSession(replaceSession.id); } + { + const { usage: _replaceUsage, ...row } = result; + result = localUsage === undefined ? row : { ...row, usage: localUsage }; + } // The local turn's own endings no longer describe the row. timedOutOutcome = false; queuedOutcome = false; From 7d61f3341dc9502b49cadc1205ccf95965f1a7dc Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:13:39 +0300 Subject: [PATCH 025/155] refactor(shell-guard): move printed-literal blanking into its own file blankPrintedLiterals, the print-call pattern, the literal scanner and the check for output that could be run now live in printed-literals.ts next to browser-bypass.ts, which imports the one function it needs. No change in behaviour. --- .../os/shell-command-guard/browser-bypass.ts | 113 +---------------- .../shell-command-guard/printed-literals.ts | 118 ++++++++++++++++++ 2 files changed, 119 insertions(+), 112 deletions(-) create mode 100644 src/tools/os/shell-command-guard/printed-literals.ts diff --git a/src/tools/os/shell-command-guard/browser-bypass.ts b/src/tools/os/shell-command-guard/browser-bypass.ts index 6d1aaeedd..5d316f8e4 100644 --- a/src/tools/os/shell-command-guard/browser-bypass.ts +++ b/src/tools/os/shell-command-guard/browser-bypass.ts @@ -4,6 +4,7 @@ import { expandHome } from "../expand-home.js"; import { BROWSER_BUNDLE_IDS } from "../screen/screen-protected.js"; import type { GuardVerdict } from "./guard-types.js"; import { basenameCommand } from "./normalise.js"; +import { blankPrintedLiterals } from "./printed-literals.js"; import { lexShellLine, type ShellWord } from "./shell-line-lexer.js"; /** @@ -170,118 +171,6 @@ const LINE_PROBLEMS: readonly ScriptProblem[] = [ const PRINTED_MESSAGE = /(^|[\s;&|(])(echo|printf)((?:\s+-[a-z]+)*)(?:\s+(?:'[^'\n]*'|"[^"\n$`]*"))+(?=[ \t]*(?:[12]?>&[12][ \t]*)?(?:$|[\n;#]|&&|\|\|))/gm; -/** - * Calls in a script that only print (ATO-1349, ATO-1353): Python `print`, - * `sys.stdout.write`, `logging.*`, JavaScript `console.*` and - * `process.stdout.write`. A sign-in hint such as - * `print("Open https://… and enter the code")` is not a browser launch. - */ -const PRINT_CALL = - /(?= text.length) return "unterminated"; - kept += text.slice(j, k + 1); - j = k + 1; - continue; - } - j += 1; - } - return "unterminated"; -} - /** Commands that run the command after them; the value-taking flags of each. */ const PREFIX_COMMANDS: Readonly> = { sudo: ["-u", "-g", "-h", "-p", "-c", "-r", "-t", "-C", "-D", "-U"], diff --git a/src/tools/os/shell-command-guard/printed-literals.ts b/src/tools/os/shell-command-guard/printed-literals.ts new file mode 100644 index 000000000..91eb9dc07 --- /dev/null +++ b/src/tools/os/shell-command-guard/printed-literals.ts @@ -0,0 +1,118 @@ +/** + * What a script only prints, taken out of the text the browser-bypass + * rules read (ATO-1349, ATO-1353). A sign-in hint such as + * `print("Open https://… and enter the code")` names a URL but opens + * nothing; a real launch (`os.system("open …")`) is left in place. + */ + +/** + * Calls in a script that only print (ATO-1349, ATO-1353): Python `print`, + * `sys.stdout.write`, `logging.*`, JavaScript `console.*` and + * `process.stdout.write`. A sign-in hint such as + * `print("Open https://… and enter the code")` is not a browser launch. + */ +const PRINT_CALL = + /(?= text.length) return "unterminated"; + kept += text.slice(j, k + 1); + j = k + 1; + continue; + } + j += 1; + } + return "unterminated"; +} From fe20f54df7cb2c19a67060ba560a2dc54de3c371 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:13:54 +0300 Subject: [PATCH 026/155] fix(llm): treat GPT-6 as an OpenAI reasoning model The reasoning rule only knew o1-o9 and gpt-5, so gpt-6 ids (Sol, Terra, Luna, Astra) went out with temperature and max_tokens, the two fields OpenAI's reasoning models refuse. Every GPT generation from 5 on now gets no temperature and max_completion_tokens. --- src/llm/provider/openai/model-params.test.ts | 3 ++- src/llm/provider/openai/model-params.ts | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/llm/provider/openai/model-params.test.ts b/src/llm/provider/openai/model-params.test.ts index 2a7e336ca..db9d3fb4a 100644 --- a/src/llm/provider/openai/model-params.test.ts +++ b/src/llm/provider/openai/model-params.test.ts @@ -12,7 +12,7 @@ afterEach(() => { }); describe("modelParamProfile", () => { - it.each(["o3", "o4-mini", "gpt-5", "openai/gpt-5-codex"])( + it.each(["o3", "o4-mini", "gpt-5", "openai/gpt-5-codex", "gpt-6", "gpt-6.1-sol", "gpt-6.2-sol", "openai/gpt-6-terra", "GPT-6-Astra"])( "forbids temperature and caps with max_completion_tokens for %s", (model) => { expect(modelParamProfile(model)).toEqual({ @@ -45,6 +45,7 @@ describe("modelParamProfile", () => { "kimi-k3-preview", "kimi-k3.10", "gpt-4o", + "gpt-4.1-mini", "olmo-3", ])("leaves temperature free for %s", (model) => { expect(modelParamProfile(model)).toEqual({ diff --git a/src/llm/provider/openai/model-params.ts b/src/llm/provider/openai/model-params.ts index 41414478b..f76cdc1e5 100644 --- a/src/llm/provider/openai/model-params.ts +++ b/src/llm/provider/openai/model-params.ts @@ -4,7 +4,7 @@ * Facts the builders need about a model id, kept next to each other so * the rule that reads the id lives in one place: * - * - OpenAI's reasoning models (`o1`, `o3`, `o4-mini`, `gpt-5`, …) reject + * - OpenAI's reasoning models (`o1`, `o3`, `o4-mini`, `gpt-5`, `gpt-6`, …) reject * `temperature` and reject `max_tokens` in favour of * `max_completion_tokens`. atag's own `request-size-rejection.ts` * quotes the second rejection verbatim and used to rely on the @@ -43,7 +43,12 @@ export interface ModelParamProfile { capField: "max_tokens" | "max_completion_tokens"; } -const OPENAI_REASONING_MODEL_RE = /^(?:o[1-9]|gpt-5)(?![a-z])/i; +/** + * `o1`…`o9` and every GPT generation from 5 on (`gpt-5.4-mini`, + * `gpt-6.2-sol`, `openai/gpt-6-terra`, ATO-1339). GPT-4.x and older take + * `temperature` and `max_tokens` and keep the historical body. + */ +const OPENAI_REASONING_MODEL_RE = /^(?:o[1-9]|gpt-(?:[5-9]|[1-9][0-9]))(?![a-z])/i; /** * `kimi-k3.1`, `kimi-k3.1-pro`, `kimi-k3.1-pro-highspeed`, with or From 1fd8eebab95fe2b5412450ea1fd4f43e91d6d36f Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:14:08 +0300 Subject: [PATCH 027/155] fix(llm): keep reasoning_content on every DeepSeek tool call and within the window - openai-native-messages: under `reasoningRequired` every assistant tool-call message carries `reasoning_content`, an empty string where the call has none. Before, the empty field was added only when some call carried reasoning, so the first step of a follow-up turn sent the answered calls without it. A new `reasoningBudget` option caps what is handed back, newest calls first. - openai-build-body: the reasoning handed back is held to the room the cloud window has left after the same messages without reasoning, the tools, the reply reserve and the safety margin. The cloud prompt does not count reasoning, so a long thinking turn failed the capacity check instead of sending less of it. - Tests: first step of a follow-up turn, the budget, a tight window, and one path from session through buildPrompt to the body sent to api.deepseek.com. Linear: ATO-1397 --- .../provider/openai/openai-build-body.test.ts | 26 ++++++++ src/llm/provider/openai/openai-build-body.ts | 61 ++++++++++++++++--- .../openai/openai-native-messages.test.ts | 32 ++++++++-- .../provider/openai/openai-native-messages.ts | 29 +++++---- src/prompt/cloud-context.test.ts | 36 ++++++++++- 5 files changed, 156 insertions(+), 28 deletions(-) diff --git a/src/llm/provider/openai/openai-build-body.test.ts b/src/llm/provider/openai/openai-build-body.test.ts index 4d7a3c051..2a066de65 100644 --- a/src/llm/provider/openai/openai-build-body.test.ts +++ b/src/llm/provider/openai/openai-build-body.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { buildOpenAiChatBody } from "./openai-build-body.js"; +import { estimateTokens } from "../../../prompt/token-budget.js"; describe("buildOpenAiChatBody", () => { it("forwards responseFormat as response_format with type=json_schema and strict=true by default", () => { @@ -761,4 +762,29 @@ describe("buildOpenAiChatBody: reasoning DeepSeek requires back (ATO-1397)", () it("keeps the field off the same cloud body otherwise", () => { expect(assistant(ask({ providerKind: "openai-compatible" }))).not.toHaveProperty("reasoning_content"); }); + + it("sends less reasoning instead of failing the capacity check when the window is tight", () => { + const long = Array.from({ length: 4000 }, (_, i) => `step${i}`).join(" "); + const tight = { + ...messages, + turns: messages.turns.map((turn) => + turn.kind === "assistant_tool_call" ? { ...turn, reasoning: long } : turn, + ), + }; + const request = { + prompt: "flat", + messages: tight, + tools, + modelMode: { mode: "cloud" as const, source: "provider" as const, providerId: "deepseek", modelId: "deepseek-v4-flash" }, + contextBudget: { window: 4000, replyReserve: 1000 }, + } as Parameters[0]; + const build = (options: Parameters[7]) => + buildOpenAiChatBody(request, "deepseek-v4-flash", false, undefined, undefined, undefined, undefined, options); + const body = build({ providerKind: "openai-compatible", reasoningRequired: true }); + const sent = assistant(body)?.reasoning_content as string; + expect(sent.length).toBeLessThan(long.length); + expect(long.endsWith(sent)).toBe(true); + // The whole reasoning would not have fitted: proof the cap did the work. + expect(estimateTokens(long)).toBeGreaterThan(4000); + }); }); diff --git a/src/llm/provider/openai/openai-build-body.ts b/src/llm/provider/openai/openai-build-body.ts index c60d5d3dd..42d0777fd 100644 --- a/src/llm/provider/openai/openai-build-body.ts +++ b/src/llm/provider/openai/openai-build-body.ts @@ -7,11 +7,15 @@ import { reasoningEffortField, resolveTemperature, } from "./model-params.js"; -import { buildNativeMessages } from "./openai-native-messages.js"; +import { + buildNativeMessages, + type NativeMessageOptions, +} from "./openai-native-messages.js"; import { toStrictOpenAiTools } from "./openai-strict-tools.js"; import { nameEscape } from "./openai-tool-call-adapter.js"; import { applyAnthropicCacheControl } from "./prompt-cache-control.js"; import { assertContextCapacity } from "../context-capacity.js"; +import { estimateTokens } from "../../../prompt/token-budget.js"; /** * Fields the caller owns unconditionally. `extraBody` is merged *under* @@ -105,17 +109,27 @@ export function buildOpenAiChatBody( // layout; a sub-call, or a service that refused the layout, sends the // flat text — which `messages` was built beside, from the same packed // conversation, so both say the same thing. + // Settled before the messages too: the reasoning DeepSeek requires back + // is held to the room the window has left after it (ATO-1397). + const cap = + filtered.maxTokens ?? filtered.maxOutputTokens ?? maxOutputTokens; + const nativeOptions: NativeMessageOptions = { + nameEscape: options.nameEscape ?? nameEscape, + // A local server renders the template, which reads the + // reasoning back where the model expects it (ATO-887). + ...(options.templateKwargs ? { preserveReasoning: true } : {}), + // DeepSeek refuses a thinking-mode tool history without it + // (ATO-1397). + ...(options.reasoningRequired + ? { preserveReasoning: true, reasoningRequired: true } + : {}), + }; const messages = resolveMessageShape(filtered, options) === "native" && filtered.messages ? buildNativeMessages(filtered.messages, { - nameEscape: options.nameEscape ?? nameEscape, - // A local server renders the template, which reads the - // reasoning back where the model expects it (ATO-887). - ...(options.templateKwargs ? { preserveReasoning: true } : {}), - // DeepSeek refuses a thinking-mode tool history without it - // (ATO-1397). + ...nativeOptions, ...(options.reasoningRequired - ? { preserveReasoning: true, reasoningRequired: true } + ? reasoningRoom(filtered, filtered.messages, nativeOptions, cap) : {}), }) : [ @@ -161,8 +175,6 @@ export function buildOpenAiChatBody( // family's own: OpenAI's reasoning models answer `max_tokens` with // "'max_tokens' is not supported with this model. Use // 'max_completion_tokens' instead." - const cap = - filtered.maxTokens ?? filtered.maxOutputTokens ?? maxOutputTokens; if (typeof cap === "number") body[profile.capField] = cap; // Reasoning effort, in the field this kind reads (`model-params.ts`); // omitted for a kind without a known one rather than guessed. @@ -283,6 +295,35 @@ export function buildOpenAiChatBody( return checkedBody(merged, request); } +/** + * The reasoning budget for a body that must carry its tool calls' + * reasoning (`reasoningRequired`): what the cloud window has left once the + * same messages without any reasoning, the tools, the reply reserve and + * the safety margin `checkedBody` checks against are counted. The cloud + * prompt's own budget counts calls and results, not reasoning, so without + * this a long thinking turn would fail the capacity check instead of + * sending less of it. Ten per cent is kept back for JSON escaping. No + * budget (unbounded) when the request has no cloud window to check. + */ +function reasoningRoom( + request: CompletionRequest, + prompt: NonNullable, + nativeOptions: NativeMessageOptions, + cap: number | undefined, +): { reasoningBudget?: number } { + const budget = + request.modelMode?.mode === "cloud" ? request.contextBudget : undefined; + if (!budget || budget.window === null) return {}; + const bare = buildNativeMessages(prompt, { ...nativeOptions, reasoningBudget: 0 }); + const used = estimateTokens( + JSON.stringify({ messages: bare, tools: request.tools, response_format: request.responseFormat }), + ); + const reserve = + typeof cap === "number" && Number.isFinite(cap) ? cap : budget.replyReserve; + const room = budget.window - used - reserve - 1024; + return { reasoningBudget: Math.max(0, Math.floor(room * 0.9)) }; +} + function checkedBody(body: Record, request: CompletionRequest): Record { if (request.modelMode?.mode === "cloud" && request.contextBudget) { const cap = body.max_completion_tokens ?? body.max_tokens; diff --git a/src/llm/provider/openai/openai-native-messages.test.ts b/src/llm/provider/openai/openai-native-messages.test.ts index e3d4821f8..5cc181dbf 100644 --- a/src/llm/provider/openai/openai-native-messages.test.ts +++ b/src/llm/provider/openai/openai-native-messages.test.ts @@ -318,7 +318,7 @@ describe("buildNativeMessages: reasoning the service requires back (ATO-1397)", expect(assistants.map((m) => m.reasoning_content)).toEqual([older, newer]); }); - it("gives a call without reasoning an empty field once the model is thinking", () => { + it("gives a call without reasoning an empty field", () => { const messages = buildNativeMessages( prompt({ turns: [ @@ -341,17 +341,41 @@ describe("buildNativeMessages: reasoning the service requires back (ATO-1397)", ); }); - it("adds nothing when no call carries reasoning (thinking off)", () => { + it("keeps the field on answered calls on the first step of a follow-up turn", () => { + // The new turn has no calls yet, so no turn carries reasoning at all. const messages = buildNativeMessages( prompt({ turns: [ - { kind: "user", text: "go" }, + { kind: "user", text: "read a" }, { kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "a" } }, result, + { kind: "assistant_reply", text: "read it" }, + { kind: "user", text: "now b" }, ], }), { ...options, reasoningRequired: true }, ); - expect(messages[2]).not.toHaveProperty("reasoning_content"); + const calls = messages.filter((m) => m.role === "assistant" && m.tool_calls !== undefined); + expect(calls).toHaveLength(1); + expect(calls[0]!.reasoning_content).toBe(""); + }); + + it("holds the reasoning to reasoningBudget, newest first, and leaves the rest empty", () => { + const older = words(600, "old"); + const newer = words(600, "new"); + const messages = buildNativeMessages( + prompt({ + turns: [ + { kind: "user", text: "go" }, + { kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "a" }, reasoning: older }, + result, + { kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "b" }, reasoning: newer }, + result, + ], + }), + { ...options, reasoningRequired: true, reasoningBudget: estimateTokens(newer) }, + ); + const calls = messages.filter((m) => m.role === "assistant"); + expect(calls.map((m) => m.reasoning_content)).toEqual(["", newer]); }); }); diff --git a/src/llm/provider/openai/openai-native-messages.ts b/src/llm/provider/openai/openai-native-messages.ts index fdc6fa3d3..4382eb756 100644 --- a/src/llm/provider/openai/openai-native-messages.ts +++ b/src/llm/provider/openai/openai-native-messages.ts @@ -50,14 +50,21 @@ export interface NativeMessageOptions { /** * The service itself requires the reasoning back (DeepSeek's thinking * mode answers a tool-call history without it with a 400, ATO-1397). - * Implies `preserveReasoning`, without the token cap: a dropped piece is - * what the service refuses, and a cloud window has room for it. Once - * any call in the request carries reasoning (the model is thinking), - * every assistant tool-call message carries the field, as an empty - * string where the transcript kept none (later calls of a batch, calls - * of answered turns), so no message lacks it. + * Implies `preserveReasoning`, without the local token cap (the body + * builder passes the room the window has left as `reasoningBudget`). + * Every assistant tool-call message carries the field: the call's own + * reasoning, else an empty string (answered turns, later calls of a + * batch, reasoning the budget left out), so no tool-call message lacks + * it — not even on the first step of a follow-up turn, when only + * answered calls are in the history. */ reasoningRequired?: boolean; + /** + * Most reasoning, in estimated tokens, handed back under + * `reasoningRequired`; newest calls first, as `capPreservedReasoning` + * does. Absent, no cap. + */ + reasoningBudget?: number; } /** @@ -132,17 +139,13 @@ export function buildNativeMessages( let orphanLines: string[] = []; const reasoningByTurn = options.reasoningRequired === true - ? capPreservedReasoning(prompt.turns, Number.POSITIVE_INFINITY) + ? capPreservedReasoning(prompt.turns, options.reasoningBudget ?? Number.POSITIVE_INFINITY) : options.preserveReasoning === true ? capPreservedReasoning(prompt.turns) : new Map(); // What a tool-call message without reasoning of its own carries: - // nothing, unless the service requires the field and the model is - // thinking in this conversation. - const missingReasoning = - options.reasoningRequired === true && reasoningByTurn.size > 0 - ? "" - : undefined; + // nothing, unless the service requires the field. + const missingReasoning = options.reasoningRequired === true ? "" : undefined; const flushOrphans = (): void => { if (orphanLines.length === 0) return; diff --git a/src/prompt/cloud-context.test.ts b/src/prompt/cloud-context.test.ts index 27520416b..f85cfe675 100644 --- a/src/prompt/cloud-context.test.ts +++ b/src/prompt/cloud-context.test.ts @@ -1,9 +1,10 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { buildPrompt } from "./build-prompt.js"; import type { BuildPromptInput } from "./build-prompt-types.js"; import { createEmptySessionState } from "../session/session-state.js"; import { userTurn, assistantToolCallTurn, toolResultTurn, assistantReplyTurn } from "../session/conversation-turn.js"; import { buildOpenAiChatBody } from "../llm/provider/openai/openai-build-body.js"; +import { OpenAiProvider } from "../llm/provider/openai/openai-provider.js"; import { buildNativeMessages } from "../llm/provider/openai/openai-native-messages.js"; import { compactionBoundaryHash } from "../session/session-compaction.js"; import { planCompaction } from "./plan-compaction.js"; @@ -149,4 +150,37 @@ describe("cloud message retention", () => { const answered = second.messages.turns.find((turn) => turn.kind === "assistant_tool_call"); expect(answered).not.toHaveProperty("reasoning"); }); + + it("sends DeepSeek the turn's reasoning_content from session to wire (ATO-1397)", async () => { + const bodies: Array> = []; + const fetchImpl = vi.fn(async (_url: unknown, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body)) as Record); + return new Response( + JSON.stringify({ model: "deepseek-v4-flash", choices: [{ message: { role: "assistant", content: "ok" }, finish_reason: "stop" }] }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + const deepseek = new OpenAiProvider({ id: "deepseek", baseUrl: "https://api.deepseek.com", apiKey: "", + defaultChatModel: "deepseek-v4-flash", fetchImpl: fetchImpl as unknown as typeof fetch }); + const tools = [{ type: "function" as const, function: { name: "os__fs__read", parameters: { type: "object", properties: {} } } }]; + const send = async (prompt: ReturnType) => { + await deepseek.complete({ prompt: prompt.text, messages: prompt.messages, tools, modelMode: mode, + ...(prompt.contextBudget ? { contextBudget: prompt.contextBudget } : {}) }); + const sent = bodies[bodies.length - 1]!.messages as Array>; + return sent.filter((m) => m.role === "assistant" && m.tool_calls !== undefined); + }; + + const input = fixture(); + input.session.turns.push( + assistantToolCallTurn({ tool: "os.fs.read", args: { path: "a" }, reasoning: "need a", at: 2 }), + toolResultTurn({ tool: "os.fs.read", status: "ok", summary: "A", at: 3 }), + ); + const step = buildPrompt(input); + expect((await send(step)).map((m) => m.reasoning_content)).toEqual(["need a"]); + + // First step of the next turn: the answered call still carries the field. + input.session.cloudContext = step.cloudContext; + input.session.turns.push(assistantReplyTurn("done", 4), userTurn("next", 5)); + expect((await send(buildPrompt(input))).map((m) => m.reasoning_content)).toEqual([""]); + }); }); From 4ddf5b4c5be27466e812209a37c2b4d95d20ca17 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:14:15 +0300 Subject: [PATCH 028/155] fix(budget): skip the session-naming call after a budget stop A turn the spend ceiling ended still fired the session namer, a paid side call (up to 512 output tokens) that the budget never counted. The promise is "stop at the cap", so the namer is now skipped when the turn stopped for money (its own budget or the provider credit). The session stays unnamed until a later turn answers within its budget. ATO-1350, ATO-1354 --- .../composition/runtime-turn-service.test.ts | 13 ++++++++++++- .../composition/runtime-turn-service.ts | 19 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/runtime/composition/runtime-turn-service.test.ts b/src/runtime/composition/runtime-turn-service.test.ts index 1e6c7fdec..3219375d1 100644 --- a/src/runtime/composition/runtime-turn-service.test.ts +++ b/src/runtime/composition/runtime-turn-service.test.ts @@ -17,7 +17,7 @@ import { TurnController } from "../turn-controller.js"; import { SteeringInbox } from "../steering-inbox.js"; import { createRuntimeTraces } from "./runtime-traces.js"; import { connectRuntimeProviders } from "./runtime-inference.js"; -import { createRuntimeTurnService, prepareRuntimeTurnState, type RuntimeTurnDependencies } from "./runtime-turn-service.js"; +import { createRuntimeTurnService, prepareRuntimeTurnState, stoppedForMoney, type RuntimeTurnDependencies } from "./runtime-turn-service.js"; import { resolveModelMode } from "../../llm/model-mode.js"; import { resolveLlmConfig } from "../../llm/provider/registry/provider-types.js"; import type { TurnProgress } from "../../session/turn-progress.js"; @@ -394,3 +394,14 @@ describe("runtime turn ownership recipes", () => { expect(fixture.state.pendingSessionNamings.size).toBe(0); }); }); + +describe("stoppedForMoney (ATO-1350)", () => { + it("is true only when the budget or the provider credit ended the turn", () => { + expect(stoppedForMoney("cost_ceiling")).toBe(true); + expect(stoppedForMoney("credit_exhausted")).toBe(true); + expect(stoppedForMoney("time_ceiling")).toBe(false); + expect(stoppedForMoney("step_ceiling")).toBe(false); + expect(stoppedForMoney("no_progress")).toBe(false); + expect(stoppedForMoney(undefined)).toBe(false); + }); +}); diff --git a/src/runtime/composition/runtime-turn-service.ts b/src/runtime/composition/runtime-turn-service.ts index f0fbfea30..48f6345bc 100644 --- a/src/runtime/composition/runtime-turn-service.ts +++ b/src/runtime/composition/runtime-turn-service.ts @@ -542,7 +542,15 @@ export function createRuntimeTurnService( // turn is already saved and already returned, and an unnamed // session simply keeps showing its prompt — which is what every // session showed before. It must never delay or fail a reply. - if (getConfig().agent.nameSessions && shouldNameSession(finished)) { + // Not after a budget stop (ATO-1350): the naming call is billed + // too, and a run that promised to stop at its cap must not make + // one more paid call. The session stays unnamed until a later + // turn answers within its budget. + if ( + getConfig().agent.nameSessions && + !stoppedForMoney(result.stopCause) && + shouldNameSession(finished) + ) { void nameSession(finished); } // `finish` ended the whole session: its kept jobs go with it. @@ -745,6 +753,15 @@ export function createRuntimeTurnService( * other llama-server entry, or external mode, is a server the operator * runs. A fixed word, never a URL. */ +/** + * Whether a turn ended because money ran out: its own budget, or the + * provider account's credit (ATO-1350). No further paid side call + * (the session namer) follows such a turn. + */ +export function stoppedForMoney(stopCause: RunTurnResult["stopCause"]): boolean { + return stopCause === "cost_ceiling" || stopCause === "credit_exhausted"; +} + export function resolveTurnLocalEngine(config: AtomicAgentConfig, providerId: string): LocalEngine | undefined { if (!providerIdIsLlamaServer(resolveLlmConfig(config), providerId)) return undefined; if (providerId !== "local-llama" || config.localModels.mode !== "managed") return "external"; From cdebe4298e0afa7ef09408f56b81232bee9dccbd Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:14:26 +0300 Subject: [PATCH 029/155] feat(http): label tool calls in a chat's history like the live stream GET /api/sessions/{id} now adds done_line to every assistant_tool_call row, built with describeToolCall from the stored name and args. While a turn runs the desktop labels a step with tool_progress.done_line, but that line was never kept, so a reopened chat fell back to its own table and labelled MCP, Composio and newer tools differently. Nothing new is stored; an older desktop ignores the field. Refs ATO-1382 --- src/http/README.md | 2 ++ src/http/route-sessions.test.ts | 30 ++++++++++++++++++++++++++++++ src/http/route-sessions.ts | 25 ++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/http/README.md b/src/http/README.md index afd6d3147..210e53515 100644 --- a/src/http/README.md +++ b/src/http/README.md @@ -29,6 +29,8 @@ Read README.md and ../runtime/docs/lifecycle.md for session/stream/steering chan [Turn phases](stream-turn-phase.ts) (ATO-1027): with `x-atomic-extensions: 1`, a chat stream carries `event: turn_phase` on every change of what the turn is doing, `{ object: "atomic.turn_phase", session_id, phase, step_index?, tool?, phase_elapsed_ms, turn_elapsed_ms, progress? }`. `phase` is `starting`, `loading_model` (the local server answered "Loading model"), `waiting_provider`, `processing_prompt` (`progress: { processed, total, cached, fraction }` when llama-server reports prompt progress), `thinking`, `generating`, `running_tool` (`tool` names the call) or `compacting`. A host counts the seconds itself from `phase_elapsed_ms`. Once per turn, when a local model has produced nothing 45 s after the turn began, `event: slow_first_token` carries `{ object: "atomic.slow_first_token", session_id, waited_ms, phase?, model_id?, lighter_model?: { id, name, file_size_gb, downloaded }, cloud_available }` ([hint](slow-first-token-hint.ts)): what the host may offer instead. Nothing is switched. +Step labels in history (ATO-1382): `GET /api/sessions/{id}` adds `done_line` to every `assistant_tool_call` row, built from the stored name and args with the same `describeToolCall` the live `tool_progress.done_line` uses, so a reopened chat reads its steps exactly as they read while the turn ran. It is computed per answer and never stored. + [Rename route](route-session-title.ts): `PATCH /api/sessions/{id}` with `{ title }` (trimmed, 1-120 characters, one line; `400` otherwise, `404` for an unknown id) stores the operator's title through `SessionStore.renameSession`. The title is stamped as user-set, so the session namer never replaces it and a turn already running cannot write an older copy back over it. `usage` on `/v1/chat/completions` (the body, the extensions-only `usage` frame) reports the tokens every LLM call of the turn reported, summed by the runtime's turn usage meter (`RunTurnResult.usage`; a local llama-server counts by its timings), and zeros when none did. A streaming request with `stream_options.include_usage: true` gets OpenAI's extra chunk before `data: [DONE]`: empty `choices` and the turn's `usage`; every chunk before it then carries `usage: null`, as OpenAI's do ([builders](openai-chunks.ts)). diff --git a/src/http/route-sessions.test.ts b/src/http/route-sessions.test.ts index 14b35c511..a0e487adf 100644 --- a/src/http/route-sessions.test.ts +++ b/src/http/route-sessions.test.ts @@ -2,6 +2,8 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import type { CompletionResult } from "../llm/llama-server-client.js"; +import { describeToolCall } from "../tools/describe-tool-call.js"; + import { startTestHarness, type Harness } from "./test-harness.js"; import { MAX_PARKED_STEERS } from "./undelivered-steers.js"; @@ -59,6 +61,34 @@ describe("/api/sessions", () => { expect(Array.isArray(body.turns)).toBe(true); }); + it("labels each tool call in the history the way the live stream did (ATO-1382)", async () => { + const session = harness.runtime.createSession(); + harness.runtime.sessionStore.save({ + ...session, + turns: [ + { kind: "user", text: "open an issue", at: 1 }, + { kind: "assistant_tool_call", tool: "mcp.github.create_issue", args: { title: "x" }, at: 2 }, + { kind: "tool_result", tool: "mcp.github.create_issue", status: "ok", summary: "ok", at: 3 }, + { kind: "assistant_reply", text: "done", at: 4 }, + ], + }); + const response = await fetch(`${harness.baseUrl}/api/sessions/${session.id}`); + const body = (await response.json()) as { + turns: Array<{ kind: string; done_line?: string }>; + }; + const expected = describeToolCall({ + name: "mcp.github.create_issue", + args: { title: "x" }, + ok: true, + }).line; + expect(body.turns[1]?.done_line).toBe(expected); + expect(body.turns[0]?.done_line).toBeUndefined(); + expect(body.turns[2]?.done_line).toBeUndefined(); + // Built for the answer only; the stored row stays as it was. + const stored = harness.runtime.sessionStore.load(session.id); + expect(stored?.turns[1]).not.toHaveProperty("done_line"); + }); + it("returns 404 for an unknown session id", async () => { const response = await fetch(`${harness.baseUrl}/api/sessions/missing`); expect(response.status).toBe(404); diff --git a/src/http/route-sessions.ts b/src/http/route-sessions.ts index c766d6666..b90524d03 100644 --- a/src/http/route-sessions.ts +++ b/src/http/route-sessions.ts @@ -1,5 +1,7 @@ import { MAX_PENDING_STEERS } from "../runtime/steering-inbox.js"; +import type { SessionState } from "../session/session-state.js"; import { readSessionTitle } from "../session/session-title.js"; +import { describeToolCall } from "../tools/describe-tool-call.js"; import { openaiError } from "./openai-errors.js"; import { readJsonBody, @@ -57,6 +59,12 @@ export function createListSessionsHandler(): HttpHandler { /** * `GET /api/sessions/{id}` — return the full `SessionState`, including * the transcript. 404 if the session is not persisted. + * + * Each `assistant_tool_call` row also carries `done_line` (ATO-1382): + * the call in words as the live stream's `tool_progress.done_line` reads + * once done ("Used github: create issue"). Built here from the stored + * name and args, never stored, so a reopened chat labels its steps the + * way they read while the turn ran. */ export function createGetSessionHandler(): HttpHandler { return async (_req, res, ctx) => { @@ -70,7 +78,22 @@ export function createGetSessionHandler(): HttpHandler { sendError(res, 404, openaiError(`session not found: ${id}`)); return; } - sendJson(res, 200, state); + sendJson(res, 200, withToolCallDoneLines(state)); + }; +} + +/** The state with `done_line` on every tool call row, for the history view. */ +export function withToolCallDoneLines(state: SessionState): SessionState { + return { + ...state, + turns: state.turns.map((turn) => + turn.kind === "assistant_tool_call" + ? { + ...turn, + done_line: describeToolCall({ name: turn.tool, args: turn.args, ok: true }).line, + } + : turn, + ), }; } From c25518f12afeaa253b9126450c22f06fd562a6dd Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:15:06 +0300 Subject: [PATCH 030/155] fix(fusion): handle a stopped or swapped worker like any other cut - A local worker the person stopped, or one swapped for the escalation model, now marks the local leg as possibly wedged, so the next local worker asks the server first instead of waiting out a queue budget. The cut is read before the re-run resets its flags. (ATO-1317) - The re-run on the escalation model reports its tool calls as they start, so a crash mid-re-run leaves its side effects in the checkpoint and the task is not offered as safe to repeat. (ATO-1316) - The fan-out pool starts nothing more once the turn is stopped, also when no budget cap is set. (ATO-1334) --- src/tools/fusion/escalation.test.ts | 31 +++++++++++++ src/tools/fusion/local-leg-health.test.ts | 17 +++++++ src/tools/fusion/worker-runner.test.ts | 18 ++++++++ src/tools/fusion/worker-runner.ts | 56 ++++++++++++++++++----- 4 files changed, 111 insertions(+), 11 deletions(-) diff --git a/src/tools/fusion/escalation.test.ts b/src/tools/fusion/escalation.test.ts index ff3572ce8..469730dbb 100644 --- a/src/tools/fusion/escalation.test.ts +++ b/src/tools/fusion/escalation.test.ts @@ -213,6 +213,37 @@ describe("advisor escalation of a stuck worker (ATO-942)", () => { }); }); +describe("the re-run's side effects reach the checkpoint at once (ATO-1316)", () => { + it("reports the re-run's tool calls as they start", async () => { + const h = harness(async (_s, options) => { + if (options.providerId === "cloud") { + options.eventHook?.({ + type: "llm_event", + event: { type: "tool_call_parsed", call: { tool: "os.shell.run", args: {} } }, + } as never); + return done(); + } + for (let i = 0; i < 3; i += 1) sameStep(options.eventHook, i); + await flush(); + for (let i = 3; i < 6; i += 1) sameStep(options.eventHook, i); + return { + session: createEmptySessionState({ id: "s-x", workingDir: "/repo" }), + reason: options.signal?.aborted ? "cancelled" : "reply", + stepCount: 6, + } as RunTurnResult; + }); + const tools: Array<[string, string]> = []; + await runWorkerTasks(h.deps, { + ...BASE, + tasks: TASK, + signal: new AbortController().signal, + escalation: { target: { ...TARGET, replace: true }, pricing: PRICING }, + onWorkerTool: (taskId, tool) => tools.push([taskId, tool]), + }); + expect(tools).toContainEqual(["t0", "os.shell.run"]); + }); +}); + describe("the person's stop and escalate on one worker (ATO-66)", () => { it("escalates a running worker on request and stops it alone", async () => { const live = new FusionLiveWorkers(); diff --git a/src/tools/fusion/local-leg-health.test.ts b/src/tools/fusion/local-leg-health.test.ts index 95afd881a..a9a1a79d3 100644 --- a/src/tools/fusion/local-leg-health.test.ts +++ b/src/tools/fusion/local-leg-health.test.ts @@ -5,6 +5,7 @@ import type { FusionWorkerMeta } from "../../session/fusion-worker-session.js"; import { createEmptySessionState } from "../../session/session-state.js"; import { FUSION_WORKER_ID_PREFIX } from "../../session/fusion-worker-session.js"; import { runWorkerTasks, type WorkerRunnerDeps } from "./worker-runner.js"; +import { FusionLiveWorkers } from "./escalation.js"; import { LocalLegHealth, LOCAL_LEG_UNRESPONSIVE_ERROR, @@ -167,4 +168,20 @@ describe("dispatch after a cut-off local worker (ATO-917)", () => { expect(probes).toBe(0); expect(deps.turns).toBe(1); }); + + it("marks the leg when the person stops a local worker (ATO-1317)", async () => { + const health = new LocalLegHealth(); + const live = new FusionLiveWorkers(); + const deps = harness(servedThenHung, { localLegHealth: health, liveWorkers: live }); + const run = runWorkerTasks(deps, { + ...BASE, + tasks: TASK, + signal: new AbortController().signal, + }); + await new Promise((r) => setTimeout(r, 0)); + expect(live.stop("s-parent", "t0")).toBe(true); + await run; + expect(health.isSuspect()).toBe(true); + expect(health.cause()).toBe("task t0 was stopped by the person"); + }); }); diff --git a/src/tools/fusion/worker-runner.test.ts b/src/tools/fusion/worker-runner.test.ts index eb89113e6..f29fe4c85 100644 --- a/src/tools/fusion/worker-runner.test.ts +++ b/src/tools/fusion/worker-runner.test.ts @@ -93,6 +93,24 @@ const BASE = { }; describe("runWorkerTasks", () => { + it("starts no further task after the turn is stopped, with no budget cap set (ATO-1334)", async () => { + const stop = new AbortController(); + const { deps, calls } = harness(async () => { + stop.abort(new Error("stopped")); + return turnResult({ reason: "cancelled" }); + }); + const results = await runWorkerTasks(deps, { + ...BASE, + tasks: tasks(3), + maxWorkers: 1, + signal: stop.signal, + }); + expect(calls).toHaveLength(1); + expect(results.map((r) => r.status)).toEqual(["cancelled", "cancelled", "cancelled"]); + expect(results[1]!.error).toBe("not started: the turn was stopped"); + expect(results[2]!.error).toBe("not started: the turn was stopped"); + }); + it("pins each worker turn to the local leg with the fusion origin and the worker filter", async () => { const { deps, calls } = harness(async ({ options }) => { options.eventHook?.({ diff --git a/src/tools/fusion/worker-runner.ts b/src/tools/fusion/worker-runner.ts index c0d72619f..31d77365f 100644 --- a/src/tools/fusion/worker-runner.ts +++ b/src/tools/fusion/worker-runner.ts @@ -663,7 +663,11 @@ export async function runWorkerTasks( const width = Math.max(1, Math.min(options.maxWorkers, options.tasks.length)); const runners = Array.from({ length: width }, async () => { for (let i = claim(); i < options.tasks.length; i = claim()) { - const stop = options.shouldStart?.(); + // A stopped turn starts nothing more, capped or not (ATO-1334): + // `shouldStart` is only wired when a budget cap is set. + const stop = options.signal.aborted + ? "the turn was stopped" + : options.shouldStart?.(); results[i] = stop === undefined ? await runOneTask(deps, options, options.tasks[i]!) @@ -1469,6 +1473,30 @@ async function runOneTask( disarmWorkerSession(session.id); } + // How the LOCAL turn ended, read before the re-run below resets these + // flags (ATO-1317). A turn cut by the person's stop or by the swap to + // the escalation model was cut as surely as one cut by its clock, and + // may have left the server just as wedged; without these two the next + // local worker waited out a whole queue budget before anyone asked. + const localCut = + replaceRequested || + stopByPerson.signal.aborted || + timedOutOutcome || + queuedOutcome || + stoppedByHandBack || + options.signal.aborted; + const localCutCause = timedOutOutcome + ? "timed out" + : queuedOutcome + ? "never served" + : stoppedByHandBack + ? "handed back" + : replaceRequested + ? "replaced by the escalation model" + : stopByPerson.signal.aborted + ? "stopped by the person" + : "cancelled"; + // The advisor's hint did not help and the operator allowed the // replacement path (`escalationReplace`): the task runs again on the // escalation model, with what the local worker found, under the @@ -1526,6 +1554,20 @@ async function runOneTask( eventHook: (event) => { markNoteRead(replaceSession.id, event); replaceCollector.observe(event); + // The re-run's side effects reach the checkpoint as they + // start, like the local worker's do (ATO-1316): a crash in + // the middle of the re-run must not offer the task as safe + // to repeat. + if ( + event.type === "llm_event" && + event.event.type === "tool_call_parsed" + ) { + try { + options.onWorkerTool?.(task.id, event.event.call.tool); + } catch { + // An observer never takes the worker down. + } + } if ( event.type === "llm_event" && event.event.type === "llm_completed" && @@ -1605,16 +1647,8 @@ async function runOneTask( // A local worker that ended by an abort — its clock, its queue wait, // its hand-back or the operator — may have left the server wedged // (ATO-917); the next local dispatch asks the server first. - if ( - options.localLeg === true && - (timedOutOutcome || - queuedOutcome || - stoppedByHandBack || - options.signal.aborted) - ) { - deps.localLegHealth?.noteCut( - `task ${task.id} was ${timedOutOutcome ? "timed out" : queuedOutcome ? "never served" : stoppedByHandBack ? "handed back" : "cancelled"}`, - ); + if (options.localLeg === true && localCut) { + deps.localLegHealth?.noteCut(`task ${task.id} was ${localCutCause}`); } // How long the server took to say anything. The one number that tells From 64315037e1641ed1b71e100d65cd67aec5d75ce9 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:15:28 +0300 Subject: [PATCH 031/155] fix(openrouter): rank new model generations with their family The live catalog's bonuses named single versions (gpt-5., qwen3.7, qwen3.6, kimi-k2.6), so a new release such as GPT-6, Qwen 3.8/4 or Kimi K3 got no bonus and sat below the model it replaces. Each family bonus now covers its generation and every later one, ties go to the newer row by OpenRouter's created date, and :batch variants leave the picker. Scoring moves to its own file with tests. --- .../fetch-openrouter-chat-catalog.ts | 74 +----------- .../openrouter/openrouter-chat-score.test.ts | 56 +++++++++ .../openrouter/openrouter-chat-score.ts | 107 ++++++++++++++++++ 3 files changed, 167 insertions(+), 70 deletions(-) create mode 100644 src/llm/provider/openrouter/openrouter-chat-score.test.ts create mode 100644 src/llm/provider/openrouter/openrouter-chat-score.ts diff --git a/src/llm/provider/openrouter/fetch-openrouter-chat-catalog.ts b/src/llm/provider/openrouter/fetch-openrouter-chat-catalog.ts index 584f5bf48..65636e075 100644 --- a/src/llm/provider/openrouter/fetch-openrouter-chat-catalog.ts +++ b/src/llm/provider/openrouter/fetch-openrouter-chat-catalog.ts @@ -7,6 +7,7 @@ import { OPENROUTER_CHAT_MODEL_ORDER, OPENROUTER_MODELS_CATALOG, } from "./openrouter-models-catalog.js"; +import { compareChatRows, pricePerMillion, scoreChat } from "./openrouter-chat-score.js"; const MODELS_URL = "https://openrouter.ai/api/v1/models"; const FETCH_TIMEOUT_MS = 20_000; @@ -16,6 +17,8 @@ type OpenRouterApiModel = { id?: string; name?: string; context_length?: number; + /** Unix seconds the model was listed; the tiebreak in `compareChatRows`. */ + created?: number; pricing?: { prompt?: string; completion?: string }; supported_parameters?: string[]; architecture?: { input_modalities?: string[] }; @@ -32,75 +35,6 @@ let cached: { fetchedAt: number; picks: readonly OpenRouterChatPick[] } | null = /** Stated image support per id from the cached fetch; see `getAdvertisedOpenRouterVision`. */ let advertisedVision: ReadonlyMap = new Map(); -function pricePerMillion(tokenPrice: string | undefined): number { - const n = parseFloat(tokenPrice ?? "0"); - if (!Number.isFinite(n) || n < 0) return 0; - return Math.round(n * 1_000_000 * 100) / 100; -} - -/** - * Tool support as advertised by the API. - * - * A missing or empty `supported_parameters` means the response says - * nothing, not that the model lacks tools. Treating silence as a "no" - * is what emptied the aimlapi catalog when that provider dropped its - * capability field, so this reader keeps the two cases apart. - */ -function readAdvertisedTools(m: OpenRouterApiModel): boolean | undefined { - if (!Array.isArray(m.supported_parameters)) return undefined; - if (m.supported_parameters.length === 0) return undefined; - return m.supported_parameters.includes("tools"); -} - -function hasTools(m: OpenRouterApiModel): boolean { - return readAdvertisedTools(m) ?? true; -} - -/** - * Ranking, not gatekeeping. - * - * This function used to return -1 for every `anthropic/*` id and - * everything matching `/gemini/i`, which removed ~40 currently served - * models — the whole Claude 5 and Gemini 3.x lines — from the picker - * with no way for an operator to get them back. Nothing in the runtime - * needs that: both families speak the same OpenAI-shaped - * `/v1/chat/completions` OpenRouter exposes for everything else, and - * `native_tools` transport is what the picker already requires via - * `hasTools`. The exclusions are gone; the families are scored instead, - * so the models this agent is tuned for still sort to the top. - * - * A negative score is now reserved for rows that genuinely cannot be - * used: non-chat surfaces (embeddings, rerank, TTS) and models that - * explicitly advertise no tool support. - */ -function scoreChat(m: OpenRouterApiModel): number { - const id = m.id ?? ""; - if (!id) return -1; - if (/qwen3\.5/i.test(id)) return -1; - if (/embed|rerank|moderation|ocr|tts|transcribe/i.test(id)) return -1; - if (!hasTools(m)) return -1; - let s = 10; - const ctx = m.context_length ?? 0; - if (ctx >= 1_000_000) s += 8; - else if (ctx >= 200_000) s += 5; - if (/qwen3\.7|qwen3\.6/i.test(id)) s += 20; - if (/gpt-5\./i.test(id)) s += 15; - if (/claude-(opus|sonnet|fable|haiku)-5|claude-opus-4\.8/i.test(id)) s += 18; - else if (id.startsWith("anthropic/")) s += 6; - if (/gemini-3\./i.test(id)) s += 14; - else if (/gemini/i.test(id)) s += 4; - if (/deepseek.*v4|deepseek.*v3/i.test(id)) s += 10; - if (/kimi-k2\.6/i.test(id)) s += 12; - else if (/kimi-k2/i.test(id)) s += 8; - if (/z-ai\/glm-5|z-ai\/glm-4\.7-flash/i.test(id)) s += 11; - else if (/z-ai\/glm/i.test(id)) s += 7; - if (/4o-mini|3\.5-turbo/i.test(id)) s -= 8; - if (/thinking|reasoner|r1/i.test(id)) s -= 5; - const pin = pricePerMillion(m.pricing?.prompt); - if (pin > 0 && pin < 1) s += 3; - return s; -} - function apiRowToEntry(m: OpenRouterApiModel): ModelCatalogEntry { const id = m.id!; // Unlike tools, silence deliberately reads as "no vision", and that is @@ -232,7 +166,7 @@ async function fetchAndCacheCatalog(): Promise { ); const ranked = rows .filter((m) => scoreChat(m) >= 0) - .sort((a, b) => scoreChat(b) - scoreChat(a)); + .sort(compareChatRows); const seen = new Set(); const picks: OpenRouterChatPick[] = []; diff --git a/src/llm/provider/openrouter/openrouter-chat-score.test.ts b/src/llm/provider/openrouter/openrouter-chat-score.test.ts new file mode 100644 index 000000000..08be992f1 --- /dev/null +++ b/src/llm/provider/openrouter/openrouter-chat-score.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; + +import { compareChatRows, scoreChat } from "./openrouter-chat-score.js"; + +const row = (id: string, created?: number) => ({ + id, + context_length: 400_000, + ...(created !== undefined ? { created } : {}), +}); + +describe("scoreChat: family bonuses cover later generations (ATO-1340)", () => { + it.each([ + ["openai/gpt-6.2-sol", "openai/gpt-5.4"], + ["qwen/qwen4-235b", "qwen/qwen3.7-35b"], + ["qwen/qwen3.8-35b", "qwen/qwen3.6-35b-a3b"], + ["moonshotai/kimi-k3", "moonshotai/kimi-k2.6"], + ["anthropic/claude-fable-5.2", "anthropic/claude-sonnet-5"], + ["google/gemini-4-pro", "google/gemini-3.1-pro"], + ])("scores %s no lower than %s", (newer, older) => { + expect(scoreChat(row(newer))).toBeGreaterThanOrEqual(scoreChat(row(older))); + }); + + it("keeps older generations below their family's bonus", () => { + expect(scoreChat(row("openai/gpt-4.1"))).toBeLessThan(scoreChat(row("openai/gpt-6"))); + expect(scoreChat(row("moonshotai/kimi-k2.5"))).toBeLessThan(scoreChat(row("moonshotai/kimi-k3"))); + expect(scoreChat(row("anthropic/claude-sonnet-4"))).toBeLessThan( + scoreChat(row("anthropic/claude-sonnet-5.5")), + ); + }); + + it("drops batch-only variants and keeps the regular id", () => { + expect(scoreChat(row("openai/gpt-6.2-sol:batch"))).toBe(-1); + expect(scoreChat(row("openai/gpt-6.2-sol"))).toBeGreaterThan(0); + }); +}); + +describe("compareChatRows", () => { + it("puts the newer of two equally scored rows first", () => { + const sorted = [row("openai/gpt-5.4", 1_760_000_000), row("openai/gpt-6.2-sol", 1_790_000_000)] + .sort(compareChatRows) + .map((r) => r.id); + expect(sorted).toEqual(["openai/gpt-6.2-sol", "openai/gpt-5.4"]); + }); + + it("still lets a higher score win over a newer date", () => { + const sorted = [row("vendor/new-thing", 1_790_000_000), row("openai/gpt-5.4", 1_700_000_000)] + .sort(compareChatRows) + .map((r) => r.id); + expect(sorted).toEqual(["openai/gpt-5.4", "vendor/new-thing"]); + }); + + it("falls back to the id when nothing else differs", () => { + const sorted = [row("vendor/b"), row("vendor/a")].sort(compareChatRows).map((r) => r.id); + expect(sorted).toEqual(["vendor/a", "vendor/b"]); + }); +}); diff --git a/src/llm/provider/openrouter/openrouter-chat-score.ts b/src/llm/provider/openrouter/openrouter-chat-score.ts new file mode 100644 index 000000000..a9fdcedcb --- /dev/null +++ b/src/llm/provider/openrouter/openrouter-chat-score.ts @@ -0,0 +1,107 @@ +/** + * Ranking, not gatekeeping, for OpenRouter's live chat catalog. + * + * This used to return -1 for every `anthropic/*` id and everything + * matching `/gemini/i`, which removed ~40 currently served models — the + * whole Claude 5 and Gemini 3.x lines — from the picker with no way for + * an operator to get them back. Nothing in the runtime needs that: both + * families speak the same OpenAI-shaped `/v1/chat/completions` + * OpenRouter exposes for everything else, and `native_tools` transport + * is what the picker already requires via `hasTools`. The families are + * scored instead, so the models this agent is tuned for sort to the top. + * + * Family bonuses name a generation *and everything after it* + * (`gpt-5` and later, `qwen3.6` and later, `kimi-k3`…), never one + * version: pinned to `gpt-5.` / `qwen3.7` / `kimi-k2.6`, a new release + * (GPT-6, Qwen 4, Kimi K3) got no bonus and sank below the model it + * replaces (ATO-1340). Within a tie the newer row (OpenRouter's + * `created`) comes first. + * + * A negative score is reserved for rows that genuinely cannot be used: + * non-chat surfaces (embeddings, rerank, TTS), batch-only variants, and + * models that explicitly advertise no tool support. + */ + +export type OpenRouterScoredRow = { + id?: string; + context_length?: number; + created?: number; + pricing?: { prompt?: string; completion?: string }; + supported_parameters?: string[]; +}; + +/** A generation number at or past `from`: `[5-9]` or any two digits. */ +const GPT_FROM_5 = /gpt-(?:[5-9]|[1-9][0-9])(?![0-9])/i; +const QWEN_FROM_3_6 = /qwen(?:3\.(?:[6-9]|[1-9][0-9])|(?:[4-9]|[1-9][0-9])(?![0-9]))/i; +const CLAUDE_FROM_5 = + /claude-(?:opus|sonnet|fable|haiku)-(?:[5-9]|[1-9][0-9])(?![0-9])|claude-opus-4\.8/i; +const GEMINI_FROM_3 = /gemini-(?:[3-9]|[1-9][0-9])(?![0-9])/i; +const DEEPSEEK_FROM_V3 = /deepseek.*v(?:[3-9]|[1-9][0-9])(?![0-9])/i; +const KIMI_FROM_K2_6 = /kimi-k(?:2\.(?:[6-9]|[1-9][0-9])|(?:[3-9]|[1-9][0-9])(?![0-9]))/i; +const GLM_FROM_5 = /z-ai\/glm-(?:[5-9]|[1-9][0-9])(?![0-9])|z-ai\/glm-4\.7-flash/i; + +/** USD per million tokens from OpenRouter's per-token string. */ +export function pricePerMillion(tokenPrice: string | undefined): number { + const n = parseFloat(tokenPrice ?? "0"); + if (!Number.isFinite(n) || n < 0) return 0; + return Math.round(n * 1_000_000 * 100) / 100; +} + +/** + * Tool support as advertised by the API. + * + * A missing or empty `supported_parameters` means the response says + * nothing, not that the model lacks tools. Treating silence as a "no" + * is what emptied the aimlapi catalog when that provider dropped its + * capability field, so this reader keeps the two cases apart. + */ +function hasTools(m: OpenRouterScoredRow): boolean { + if (!Array.isArray(m.supported_parameters)) return true; + if (m.supported_parameters.length === 0) return true; + return m.supported_parameters.includes("tools"); +} + +export function scoreChat(m: OpenRouterScoredRow): number { + const id = m.id ?? ""; + if (!id) return -1; + if (/qwen3\.5/i.test(id)) return -1; + if (/embed|rerank|moderation|ocr|tts|transcribe/i.test(id)) return -1; + // Batch-only routes answer later, not in a turn. + if (/:batch$/i.test(id)) return -1; + if (!hasTools(m)) return -1; + let s = 10; + const ctx = m.context_length ?? 0; + if (ctx >= 1_000_000) s += 8; + else if (ctx >= 200_000) s += 5; + if (QWEN_FROM_3_6.test(id)) s += 20; + if (GPT_FROM_5.test(id)) s += 15; + if (CLAUDE_FROM_5.test(id)) s += 18; + else if (id.startsWith("anthropic/")) s += 6; + if (GEMINI_FROM_3.test(id)) s += 14; + else if (/gemini/i.test(id)) s += 4; + if (DEEPSEEK_FROM_V3.test(id)) s += 10; + if (KIMI_FROM_K2_6.test(id)) s += 12; + else if (/kimi-k2/i.test(id)) s += 8; + if (GLM_FROM_5.test(id)) s += 11; + else if (/z-ai\/glm/i.test(id)) s += 7; + if (/4o-mini|3\.5-turbo/i.test(id)) s -= 8; + if (/thinking|reasoner|r1/i.test(id)) s -= 5; + const pin = pricePerMillion(m.pricing?.prompt); + if (pin > 0 && pin < 1) s += 3; + return s; +} + +/** + * Picker order: higher score first, then the newer row, then the id so + * the order is stable across fetches. + */ +export function compareChatRows( + a: OpenRouterScoredRow, + b: OpenRouterScoredRow, +): number { + const byScore = scoreChat(b) - scoreChat(a); + if (byScore !== 0) return byScore; + const byCreated = (b.created ?? 0) - (a.created ?? 0); + if (byCreated !== 0) return byCreated; + return (a.id ?? "").localeCompare(b.id ?? ""); +} From 0dabaa5a39199c6121c7d0a4d4bae82f10a46bd3 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:15:47 +0300 Subject: [PATCH 032/155] fix(llm): send boolean tool subschemas as objects and drop a tool the server rejects LM Studio and llama.cpp refuse the whole chat request when one tool's JSON Schema holds a boolean subschema ("evidence": true), answering 'JSON schema error at #/...: schema must be an object' (ATO-1399). - normalize boolean subschemas in every OpenAI-compatible tools array (true -> {}, false -> dropped property/branch or { not: {} }); additionalProperties and friends keep their boolean values, and tools with nothing to rewrite stay byte-identical - treat a schema error as non-retryable instead of three blind 5xx tries - when the error's JSON pointer names exactly one tool, resend once without it (unary and stream) and log a warning naming the tool - give the leftover case its own user-facing sentence --- src/llm/provider/openai/openai-build-body.ts | 12 +- src/llm/provider/openai/openai-http.ts | 13 ++ src/llm/provider/openai/openai-provider.ts | 79 ++++++- .../openai/openai-tool-schema-retry.test.ts | 205 +++++++++++++++++ .../openai/tool-schema-booleans.test.ts | 167 ++++++++++++++ .../provider/openai/tool-schema-booleans.ts | 215 ++++++++++++++++++ .../openai/tool-schema-rejection.test.ts | 86 +++++++ .../provider/openai/tool-schema-rejection.ts | 139 +++++++++++ 8 files changed, 910 insertions(+), 6 deletions(-) create mode 100644 src/llm/provider/openai/openai-tool-schema-retry.test.ts create mode 100644 src/llm/provider/openai/tool-schema-booleans.test.ts create mode 100644 src/llm/provider/openai/tool-schema-booleans.ts create mode 100644 src/llm/provider/openai/tool-schema-rejection.test.ts create mode 100644 src/llm/provider/openai/tool-schema-rejection.ts diff --git a/src/llm/provider/openai/openai-build-body.ts b/src/llm/provider/openai/openai-build-body.ts index 39b4404fd..a627e82fc 100644 --- a/src/llm/provider/openai/openai-build-body.ts +++ b/src/llm/provider/openai/openai-build-body.ts @@ -9,6 +9,7 @@ import { } from "./model-params.js"; import { buildNativeMessages } from "./openai-native-messages.js"; import { toStrictOpenAiTools } from "./openai-strict-tools.js"; +import { normalizeToolSchemaBooleans } from "./tool-schema-booleans.js"; import { nameEscape } from "./openai-tool-call-adapter.js"; import { applyAnthropicCacheControl } from "./prompt-cache-control.js"; import { assertContextCapacity } from "../context-capacity.js"; @@ -185,9 +186,14 @@ export function buildOpenAiChatBody( // is a reserved key: the loop below restores `body.tools` over the // merge, so the transformed array is what survives. Off by default, // and off it must leave this line byte-identical to what it was. - const emittedTools = strictTools - ? toStrictOpenAiTools(filtered.tools) - : filtered.tools; + // + // Boolean subschemas (`"evidence": true`) are spelled as objects + // first, for every service: llama.cpp and LM Studio refuse the whole + // request over one, and the rewrite keeps or loosens the meaning + // (`tool-schema-booleans.ts`). Tools without one come back as the + // same array, so the body stays byte-identical for them. + const tools = normalizeToolSchemaBooleans(filtered.tools); + const emittedTools = strictTools ? toStrictOpenAiTools(tools) : tools; body.tools = emittedTools; // Structured Outputs and parallel function calls do not compose: // OpenAI documents that a parallel call generated under strict mode diff --git a/src/llm/provider/openai/openai-http.ts b/src/llm/provider/openai/openai-http.ts index 71bf0d47e..0354ded00 100644 --- a/src/llm/provider/openai/openai-http.ts +++ b/src/llm/provider/openai/openai-http.ts @@ -22,6 +22,7 @@ import { LOCAL_BAD_REQUEST_SENTENCE, localBadRequestKind, } from "./local-bad-request.js"; +import { isToolSchemaErrorText } from "./tool-schema-rejection.js"; export type OpenAiHttpDeps = { baseUrl: string; @@ -253,6 +254,14 @@ export function humanizeOpenAiHttpError(err: OpenAiHttpError): string { if (err.status === 429) { return `${who} is rate-limiting this key (429). Tried ${OPENAI_MAX_ATTEMPTS} times — wait a minute and retry.`; } + if (isToolSchemaErrorText(err.message)) { + // Not retried and not "server trouble": the server could not use a + // tool's JSON schema and no single tool could be left out (ATO-1399). + return ( + `${who} could not use the JSON schema of one of the agent's tools (${err.status}). ` + + `Turn off the MCP server or plugin that added the tool, or update the model server.` + ); + } if (err.status >= 500) { return `${who} is having server trouble (${err.status}). Tried ${OPENAI_MAX_ATTEMPTS} times — this is on the provider, not your setup.`; } @@ -953,6 +962,10 @@ function isRetryableOpenAiError(err: unknown): boolean { // up, and three fast retries only multiply the refusals (42 per // worker, once). if (isCreditExhausted(err)) return false; + // A server that cannot use a tool's JSON Schema (llama.cpp, LM Studio) + // answers it with a 500, and the same body gets the same 500. The + // provider drops the tool and resends instead (ATO-1399). + if (isToolSchemaErrorText(err.message)) return false; return err.status >= 500 || err.status === 429 || err.status === 408; } diff --git a/src/llm/provider/openai/openai-provider.ts b/src/llm/provider/openai/openai-provider.ts index ee9b3bc1e..9bf378ef3 100644 --- a/src/llm/provider/openai/openai-provider.ts +++ b/src/llm/provider/openai/openai-provider.ts @@ -60,6 +60,8 @@ import { } from "./qwen-tagged-tool-response-adapter.js"; import type { CreditLimitLogger } from "./plan-credit-limit-retry.js"; import { sendWithStructuredOutputFallback } from "./structured-output-fallback.js"; +import { normalizeToolSchemaBooleans } from "./tool-schema-booleans.js"; +import { findToolBehindSchemaError } from "./tool-schema-rejection.js"; import { isImageRejection, markModelCannotSee, @@ -276,6 +278,20 @@ export class OpenAiProvider implements LlmProvider { } async complete(request: CompletionRequest): Promise { + try { + return await this.completeOnce(request); + } catch (err) { + // One resend without the tool whose schema the server could not + // use (ATO-1399); a second refusal goes back as the server wrote it. + const reduced = this.withoutRejectedTool(request, err); + if (!reduced) throw err; + return await this.completeOnce(reduced); + } + } + + private async completeOnce( + request: CompletionRequest, + ): Promise { const model = this.defaultChatModel; if (!model) { throw new Error( @@ -423,6 +439,46 @@ export class OpenAiProvider implements LlmProvider { return true; } + /** + * The request without the one tool a schema error names + * (`tool-schema-rejection.ts`), or null when the error is something + * else or names no single tool. The warning says which tool went and + * why: the turn carries on without it rather than failing outright. + */ + private withoutRejectedTool( + request: CompletionRequest, + err: unknown, + ): CompletionRequest | null { + if (request.signal?.aborted) return null; + if (!(err instanceof OpenAiHttpError) || err.status === null) return null; + const tools = request.tools; + if (!tools || tools.length === 0) return null; + const text = [err.message, err.body?.message ?? ""].join("\n"); + const culprit = findToolBehindSchemaError( + text, + normalizeToolSchemaBooleans(tools), + tools, + ); + if (!culprit) return null; + const tool = this.toolCallAdapter.nameUnescape(culprit.name); + this.http.logger?.warn( + `llm: "${this.id}" could not use the JSON schema of tool "${tool}" (at ${culprit.pointer}); resending this request without that tool`, + { + provider: this.id, + model: this.defaultChatModel, + sessionId: request.sessionId ?? null, + status: err.status, + tool, + pointer: culprit.pointer, + detail: err.message.slice(0, 240), + }, + ); + return { + ...request, + tools: tools.filter((_, index) => index !== culprit.index), + }; + } + private markFlat(request: CompletionRequest, err: unknown): void { this.flatSessions.add(sessionKey(request)); this.http.logger?.warn( @@ -447,7 +503,10 @@ export class OpenAiProvider implements LlmProvider { ); } let shape = this.shapeFor(request); - let body = this.buildBody(request, true, shape); + // The request the body is built from: the caller's, until a schema + // error names a tool to leave out (`withoutRejectedTool`). + let activeRequest = request; + let body = this.buildBody(activeRequest, true, shape); const path = `${this.chatPathPrefix}/chat/completions`; let accumulated = ""; let accumulatedReasoning = ""; @@ -464,6 +523,8 @@ export class OpenAiProvider implements LlmProvider { let committed = false; // Whether the one resend for a named temperature has been spent. let temperatureRetried = false; + // Whether the one resend without a rejected tool has been spent. + let toolDropped = false; // One completion, one budget. Passed into every open so a reopen // draws from the same pot the opens do — the alternative is two // nested loops of `OPENAI_MAX_ATTEMPTS` and 9 requests per turn. @@ -596,6 +657,18 @@ export class OpenAiProvider implements LlmProvider { // stopped. `signal.reason` is abort-shaped by construction. if (request.signal?.aborted) throw cancellationError(request.signal, err); + // The server could not use one tool's JSON schema (ATO-1399): a + // 4xx/5xx from the open, before any output, so the same request + // goes out once more without that tool. + if (!committed && !toolDropped) { + const reduced = this.withoutRejectedTool(activeRequest, err); + if (reduced) { + toolDropped = true; + activeRequest = reduced; + body = this.buildBody(activeRequest, true, shape); + continue; + } + } // The service named the one temperature it takes: resend once // with it. Same window as the flat resend below — a 400 comes // from the open, before any output — and only once per @@ -606,7 +679,7 @@ export class OpenAiProvider implements LlmProvider { this.learnAllowedTemperature(request, err) ) { temperatureRetried = true; - body = this.buildBody(request, true, shape); + body = this.buildBody(activeRequest, true, shape); continue; } // The service refused the message layout, before any byte of @@ -617,7 +690,7 @@ export class OpenAiProvider implements LlmProvider { if (!committed && this.shouldFallBackToFlat(request, shape, err)) { this.markFlat(request, err); shape = "flat"; - body = this.buildBody(request, true, shape); + body = this.buildBody(activeRequest, true, shape); continue; } // An error event inside the stream (OpenRouter's `504 Upstream diff --git a/src/llm/provider/openai/openai-tool-schema-retry.test.ts b/src/llm/provider/openai/openai-tool-schema-retry.test.ts new file mode 100644 index 000000000..f505975f9 --- /dev/null +++ b/src/llm/provider/openai/openai-tool-schema-retry.test.ts @@ -0,0 +1,205 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { CompletionResult, StreamChunk } from "../completion-types.js"; +import { humanizeOpenAiHttpError, OpenAiHttpError } from "./openai-http.js"; +import { OpenAiProvider } from "./openai-provider.js"; + +/** + * ATO-1399: a local server that cannot use one tool's JSON schema + * refuses the whole request. The request goes out once more without + * that tool, with a warning naming it; anything unmatched fails as it + * did before. + */ + +const MODEL = "local-model"; + +const SCHEMA_500 = JSON.stringify({ + error: + "JSON schema error at #/$defs/RecordSource/oneOf/1/properties/data: schema must be an object", +}); + +function json(body: string, status: number): Response { + return new Response(body, { + status, + headers: { "content-type": "application/json" }, + }); +} + +function okUnary(): Response { + return json( + JSON.stringify({ + model: MODEL, + choices: [ + { message: { role: "assistant", content: "ok" }, finish_reason: "stop" }, + ], + }), + 200, + ); +} + +function okStream(): Response { + return new Response( + 'data: {"choices":[{"delta":{"content":"ok"},"finish_reason":null}]}\n\n' + + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n' + + "data: [DONE]\n\n", + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); +} + +function scripted(responses: ReadonlyArray<() => Response>) { + const bodies: Record[] = []; + const fetchImpl = vi.fn(async (_url: unknown, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Record); + const next = responses[bodies.length - 1] ?? responses[responses.length - 1]; + return next!(); + }); + return { fetchImpl, bodies }; +} + +function setup(responses: ReadonlyArray<() => Response>) { + const { fetchImpl, bodies } = scripted(responses); + const warn = vi.fn(); + const provider = new OpenAiProvider({ + id: "lmstudio", + baseUrl: "http://127.0.0.1:1234", + apiKey: "", + defaultChatModel: MODEL, + fetchImpl: fetchImpl as typeof fetch, + logger: { warn }, + }); + return { provider, fetchImpl, bodies, warn }; +} + +/** + * A schema the boolean rewrite leaves alone but this fake server still + * refuses: `data` is a string where a subschema belongs. + */ +const BAD_TOOL = { + type: "function", + function: { + name: "plugin__records__lookup", + parameters: { + type: "object", + $defs: { + RecordSource: { + oneOf: [{ type: "object" }, { properties: { data: "anything" } }], + }, + }, + }, + }, +}; + +const GOOD_TOOL = { + type: "function", + function: { + name: "os__fs__read", + parameters: { type: "object", properties: { path: { type: "string" } } }, + }, +}; + +function names(body: Record): string[] { + return ((body.tools as any[]) ?? []).map((t) => t.function.name); +} + +async function drain( + stream: AsyncGenerator, +): Promise { + for (;;) { + const next = await stream.next(); + if (next.done) return next.value; + } +} + +describe("OpenAiProvider — drop a tool the server cannot use (ATO-1399)", () => { + it("resends a unary completion once without the named tool", async () => { + const { provider, fetchImpl, bodies, warn } = setup([ + () => json(SCHEMA_500, 500), + okUnary, + ]); + const result = await provider.complete({ + prompt: "hi", + tools: [GOOD_TOOL, BAD_TOOL], + }); + + expect(result.content).toBe("ok"); + // No blind 5xx retries of the same body before the drop. + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(names(bodies[0]!)).toEqual(["os__fs__read", "plugin__records__lookup"]); + expect(names(bodies[1]!)).toEqual(["os__fs__read"]); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]![0]).toMatch(/plugin\.records\.lookup/); + expect(warn.mock.calls[0]![1]).toMatchObject({ + tool: "plugin.records.lookup", + status: 500, + }); + }); + + it("resends a streamed completion once without the named tool", async () => { + const { provider, fetchImpl, bodies } = setup([ + () => json(SCHEMA_500, 500), + okStream, + ]); + const result = await drain( + provider.completeStream({ prompt: "hi", tools: [GOOD_TOOL, BAD_TOOL] }), + ); + + expect(result.content).toBe("ok"); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(names(bodies[1]!)).toEqual(["os__fs__read"]); + }); + + it("surfaces the original error when no single tool matches", async () => { + const { provider, fetchImpl, warn } = setup([ + () => json(SCHEMA_500, 500), + okUnary, + ]); + const err = await provider + .complete({ prompt: "hi", tools: [GOOD_TOOL] }) + .catch((e: unknown) => e); + expect(err).toBeInstanceOf(OpenAiHttpError); + expect((err as Error).message).toMatch(/JSON schema error/); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(warn).not.toHaveBeenCalled(); + // Named for what it is, not as retried "server trouble". + expect(humanizeOpenAiHttpError(err as OpenAiHttpError)).toMatch( + /could not use the JSON schema/, + ); + }); + + it("drops at most one tool per completion", async () => { + const { provider, fetchImpl } = setup([() => json(SCHEMA_500, 500)]); + await expect( + drain( + provider.completeStream({ + prompt: "hi", + tools: [GOOD_TOOL, BAD_TOOL], + }), + ), + ).rejects.toThrow(/JSON schema error/); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it("sends a boolean subschema as an object, so the first send succeeds", async () => { + const { provider, fetchImpl, bodies } = setup([okUnary]); + await provider.complete({ + prompt: "hi", + tools: [ + { + type: "function", + function: { + name: "plugin__records__lookup", + parameters: { + type: "object", + properties: { evidence: true }, + }, + }, + }, + ], + }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect((bodies[0]!.tools as any[])[0].function.parameters).toEqual({ + type: "object", + properties: { evidence: {} }, + }); + }); +}); diff --git a/src/llm/provider/openai/tool-schema-booleans.test.ts b/src/llm/provider/openai/tool-schema-booleans.test.ts new file mode 100644 index 000000000..cb83a595b --- /dev/null +++ b/src/llm/provider/openai/tool-schema-booleans.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; + +import { buildOpenAiChatBody } from "./openai-build-body.js"; +import { + normalizeSchemaBooleans, + normalizeToolSchemaBooleans, +} from "./tool-schema-booleans.js"; + +/** The schema shape from the ATO-1399 report, trimmed to what matters. */ +function reportedSchema(): Record { + return { + type: "object", + properties: { source: { $ref: "#/$defs/RecordSource" } }, + required: ["source"], + $defs: { + RecordSource: { + oneOf: [ + { type: "object", properties: { url: { type: "string" } } }, + { + type: "object", + properties: { kind: { type: "string" }, evidence: true }, + required: ["kind", "evidence"], + }, + ], + }, + }, + }; +} + +describe("normalizeSchemaBooleans (ATO-1399)", () => { + it("rewrites $defs/RecordSource/oneOf/1/properties/evidence: true to {}", () => { + const input = reportedSchema(); + const snapshot = structuredClone(input); + const out = normalizeSchemaBooleans(input) as any; + + expect(out.$defs.RecordSource.oneOf[1].properties.evidence).toEqual({}); + expect(out.$defs.RecordSource.oneOf[1].required).toEqual([ + "kind", + "evidence", + ]); + // Untouched branches keep their identity; the input is not mutated. + expect(out.$defs.RecordSource.oneOf[0]).toBe( + (input as any).$defs.RecordSource.oneOf[0], + ); + expect(input).toEqual(snapshot); + }); + + it("returns the same reference when there is nothing to rewrite", () => { + const schema = { + type: "object", + properties: { a: { type: "array", items: { type: "string" } } }, + additionalProperties: false, + }; + expect(normalizeSchemaBooleans(schema)).toBe(schema); + }); + + it("drops a false property and its required entry", () => { + const out = normalizeSchemaBooleans({ + type: "object", + properties: { a: { type: "string" }, b: false }, + required: ["a", "b"], + }); + expect(out).toEqual({ + type: "object", + properties: { a: { type: "string" } }, + required: ["a"], + }); + }); + + it("keeps boolean additionalProperties and recurses an object one", () => { + const out = normalizeSchemaBooleans({ + type: "object", + properties: { + open: { type: "object", additionalProperties: true }, + map: { type: "object", additionalProperties: { anyOf: [true] } }, + }, + additionalProperties: false, + }) as any; + expect(out.additionalProperties).toBe(false); + expect(out.properties.open.additionalProperties).toBe(true); + expect(out.properties.map.additionalProperties).toEqual({ anyOf: [{}] }); + }); + + it("covers items, prefixItems, compositions, not and conditionals", () => { + const out = normalizeSchemaBooleans({ + type: "object", + properties: { + anything: { type: "array", items: true }, + empty: { type: "array", items: false }, + tuple: { type: "array", prefixItems: [true, { type: "string" }] }, + legacyTuple: { type: "array", items: [false] }, + either: { anyOf: [false, { type: "string" }] }, + never: { oneOf: [false] }, + both: { allOf: [true, { type: "string" }] }, + notFalse: { type: "string", not: false }, + notTrue: { not: true }, + cond: { if: true, then: false, else: { type: "string" } }, + }, + patternProperties: { "^x-": false, "^y-": true }, + dependentSchemas: { a: true, b: false, c: { required: ["a"] } }, + definitions: { Never: false, Any: true }, + }) as any; + const p = out.properties; + expect(p.anything).toEqual({ type: "array" }); + expect(p.empty).toEqual({ type: "array", maxItems: 0 }); + expect(p.tuple.prefixItems).toEqual([{}, { type: "string" }]); + expect(p.legacyTuple.items).toEqual([{ not: {} }]); + expect(p.either.anyOf).toEqual([{ type: "string" }]); + expect(p.never.oneOf).toEqual([{ not: {} }]); + expect(p.both.allOf).toEqual([{ type: "string" }]); + expect(p.notFalse).toEqual({ type: "string" }); + expect(p.notTrue).toEqual({ not: {} }); + expect(p.cond).toEqual({ if: {}, then: { not: {} }, else: { type: "string" } }); + expect(out.patternProperties).toEqual({ "^y-": {} }); + expect(out.dependentSchemas).toEqual({ c: { required: ["a"] } }); + expect(out.definitions).toEqual({ Never: { not: {} }, Any: {} }); + }); + + it("turns a boolean root into an object schema", () => { + expect(normalizeSchemaBooleans(true)).toEqual({ type: "object" }); + }); +}); + +describe("normalizeToolSchemaBooleans", () => { + it("rewrites only the tool that needs it and keeps the array otherwise", () => { + const clean = { + type: "function", + function: { name: "clean", parameters: { type: "object" } }, + }; + const bad = { + type: "function", + function: { name: "plugin__lookup", parameters: reportedSchema() }, + }; + expect(normalizeToolSchemaBooleans([clean])).toEqual([clean]); + const same = [clean]; + expect(normalizeToolSchemaBooleans(same)).toBe(same); + + const out = normalizeToolSchemaBooleans([clean, bad]); + expect(out[0]).toBe(clean); + expect(out[1]).not.toBe(bad); + expect( + (out[1] as any).function.parameters.$defs.RecordSource.oneOf[1] + .properties.evidence, + ).toEqual({}); + }); + + it("is applied to the tools of every chat body", () => { + const body = buildOpenAiChatBody( + { + prompt: "hi", + tools: [ + { + type: "function", + function: { name: "plugin__lookup", parameters: reportedSchema() }, + }, + ], + }, + "local-model", + false, + ); + const tools = body.tools as any[]; + expect( + tools[0].function.parameters.$defs.RecordSource.oneOf[1].properties + .evidence, + ).toEqual({}); + }); +}); diff --git a/src/llm/provider/openai/tool-schema-booleans.ts b/src/llm/provider/openai/tool-schema-booleans.ts new file mode 100644 index 000000000..975865753 --- /dev/null +++ b/src/llm/provider/openai/tool-schema-booleans.ts @@ -0,0 +1,215 @@ +/** + * Boolean subschemas out of tool `parameters` (ATO-1399). + * + * JSON Schema lets any subschema be a bare `true` ("anything") or + * `false` ("nothing"), and MCP servers do send them: the report was a + * plugin tool with `$defs.RecordSource.oneOf[1].properties.evidence: + * true`. llama.cpp's schema-to-grammar converter and LM Studio's + * validator both demand an object at every subschema position and + * answer the **whole request** with an error ("JSON schema error at + * #/$defs/RecordSource/oneOf/1/properties/evidence: schema must be an + * object"), so one third-party tool took the chat down with it. + * + * The rewrite, applied recursively to every subschema position: + * + * - `true` becomes `{}`: the same schema, spelled as an object. + * Where the keyword itself is optional (`items`, an `allOf` + * member) it is dropped instead. + * - `false` under `properties` / `patternProperties` / + * `dependentSchemas` drops the entry (and the name from + * `required`); under `anyOf` / `oneOf` drops the branch. As + * `items` it becomes `maxItems: 0`; `not: false` is dropped. + * - any other `false` becomes `{ not: {} }`. + * + * Every `false` rewrite either keeps the meaning or **loosens** it (a + * forbidden key becomes merely undeclared). That is safe for the reason + * `openai-strict-tools.ts` gives for its stripped bounds: the schema + * steers the model's decoding, and the tool's own validator still + * checks every call. + * + * Kept as booleans: `additionalProperties`, `additionalItems`, + * `unevaluatedProperties`, `unevaluatedItems`. There a boolean is the + * keyword's documented value and llama.cpp reads it as such. + * + * Copy-on-write: the input is never mutated, and a schema with nothing + * to rewrite comes back as the same reference, so a request whose tools + * carry no boolean subschema is byte-identical to what it was. + */ + +type Schema = Record; + +/** Past this depth a schema is returned as it is rather than recursed. */ +const MAX_DEPTH = 64; + +/** Keywords whose value is one subschema. */ +const SINGLE_SCHEMA_KEYWORDS = [ + "contains", + "propertyNames", + "if", + "then", + "else", +] as const; + +/** Keywords whose boolean value is legal and kept; objects are recursed. */ +const BOOLEAN_OK_KEYWORDS = [ + "additionalProperties", + "additionalItems", + "unevaluatedProperties", + "unevaluatedItems", +] as const; + +function isPlainObject(value: unknown): value is Schema { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** A subschema in a position where a boolean has no shorter spelling. */ +function asObject(value: unknown, depth: number): unknown { + if (value === true) return {}; + if (value === false) return { not: {} }; + return normalizeNode(value, depth); +} + +/** A name -> schema map; `false` entries are dropped and reported. */ +function normalizeMap(map: Schema, depth: number, dropped: string[]): Schema { + let out: Schema | undefined; + for (const [key, child] of Object.entries(map)) { + const next = child === false ? undefined : asObject(child, depth); + if (next === child) continue; + out ??= { ...map }; + if (next === undefined) { + delete out[key]; + dropped.push(key); + } else { + out[key] = next; + } + } + return out ?? map; +} + +/** `anyOf` / `oneOf` / `allOf` / `prefixItems` / tuple `items`. */ +function normalizeList( + list: readonly unknown[], + depth: number, + drop: (value: unknown) => boolean, +): readonly unknown[] { + let changed = false; + const out: unknown[] = []; + for (const child of list) { + if (drop(child)) { + changed = true; + continue; + } + const next = asObject(child, depth); + if (next !== child) changed = true; + out.push(next); + } + return changed ? out : list; +} + +function normalizeNode(node: unknown, depth: number): unknown { + if (!isPlainObject(node) || depth > MAX_DEPTH) return node; + const next = depth + 1; + let out: Schema | undefined; + const set = (key: string, value: unknown) => { + if (value === node[key]) return; + out ??= { ...node }; + if (value === undefined) delete out[key]; + else out[key] = value; + }; + + if (isPlainObject(node.properties)) { + const dropped: string[] = []; + set("properties", normalizeMap(node.properties, next, dropped)); + if (dropped.length > 0 && Array.isArray(node.required)) { + const gone = new Set(dropped); + set("required", node.required.filter((name) => !gone.has(name))); + } + } + if (isPlainObject(node.patternProperties)) { + set("patternProperties", normalizeMap(node.patternProperties, next, [])); + } + if (isPlainObject(node.dependentSchemas)) { + // `true` adds no constraint and `false` forbids the key: dropping the + // entry keeps the first and loosens the second. + const deps = node.dependentSchemas; + const kept = Object.fromEntries( + Object.entries(deps).filter(([, child]) => typeof child !== "boolean"), + ); + const base = + Object.keys(kept).length === Object.keys(deps).length ? deps : kept; + set("dependentSchemas", normalizeMap(base, next, [])); + } + for (const key of ["$defs", "definitions"] as const) { + const defs = node[key]; + if (!isPlainObject(defs)) continue; + // A `$ref` may point at any entry, so none is dropped. + let copy: Schema | undefined; + for (const [name, def] of Object.entries(defs)) { + const value = asObject(def, next); + if (value === def) continue; + copy ??= { ...defs }; + copy[name] = value; + } + if (copy) set(key, copy); + } + for (const key of ["anyOf", "oneOf"] as const) { + const list = node[key]; + if (!Array.isArray(list)) continue; + const kept = normalizeList(list, next, (v) => v === false); + set(key, kept.length > 0 ? kept : [{ not: {} }]); + } + if (Array.isArray(node.allOf)) { + const kept = normalizeList(node.allOf, next, (v) => v === true); + set("allOf", kept.length > 0 ? kept : undefined); + } + if (Array.isArray(node.prefixItems)) { + set("prefixItems", normalizeList(node.prefixItems, next, () => false)); + } + if (node.items === true) { + set("items", undefined); + } else if (node.items === false) { + set("items", undefined); + set("maxItems", 0); + } else if (Array.isArray(node.items)) { + set("items", normalizeList(node.items, next, () => false)); + } else if (isPlainObject(node.items)) { + set("items", normalizeNode(node.items, next)); + } + if (node.not === false) set("not", undefined); + else if ("not" in node) set("not", asObject(node.not, next)); + for (const key of SINGLE_SCHEMA_KEYWORDS) { + if (key in node) set(key, asObject(node[key], next)); + } + for (const key of BOOLEAN_OK_KEYWORDS) { + if (isPlainObject(node[key])) set(key, normalizeNode(node[key], next)); + } + return out ?? node; +} + +/** + * One function's `parameters`, with every boolean subschema rewritten. + * A boolean root (never a valid function schema) becomes an open object. + */ +export function normalizeSchemaBooleans(schema: unknown): unknown { + if (typeof schema === "boolean") return { type: "object" }; + return normalizeNode(schema, 0); +} + +/** + * Every OpenAI function tool with its `parameters` normalized. Returns + * the input array itself when no tool needed a change. + */ +export function normalizeToolSchemaBooleans>( + tools: ReadonlyArray, +): ReadonlyArray { + let out: T[] | undefined; + tools.forEach((tool, index) => { + const fn = tool.function; + if (!isPlainObject(fn) || !("parameters" in fn)) return; + const parameters = normalizeSchemaBooleans(fn.parameters); + if (parameters === fn.parameters) return; + out ??= [...tools]; + out[index] = { ...tool, function: { ...fn, parameters } }; + }); + return out ?? tools; +} diff --git a/src/llm/provider/openai/tool-schema-rejection.test.ts b/src/llm/provider/openai/tool-schema-rejection.test.ts new file mode 100644 index 000000000..f56f17476 --- /dev/null +++ b/src/llm/provider/openai/tool-schema-rejection.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vitest"; + +import { + findToolBehindSchemaError, + isToolSchemaErrorText, +} from "./tool-schema-rejection.js"; + +const LM_STUDIO_500 = + "openai provider 500: {\"error\":\"JSON schema error at #/$defs/RecordSource/oneOf/1/properties/evidence: schema must be an object\"}"; + +function tool(name: string, parameters: unknown): Record { + return { type: "function", function: { name, parameters } }; +} + +const plugin = tool("plugin__lookup", { + type: "object", + $defs: { + RecordSource: { + oneOf: [ + { type: "object" }, + { type: "object", properties: { evidence: true } }, + ], + }, + }, +}); + +const other = tool("os__fs__read", { + type: "object", + properties: { path: { type: "string" } }, +}); + +describe("tool schema rejection (ATO-1399)", () => { + it("recognises the llama.cpp / LM Studio wording", () => { + expect(isToolSchemaErrorText(LM_STUDIO_500)).toBe(true); + expect(isToolSchemaErrorText("openai provider 500: upstream died")).toBe( + false, + ); + }); + + it("names the one tool whose schema holds a non-object at the pointer", () => { + expect( + findToolBehindSchemaError(LM_STUDIO_500, [other, plugin]), + ).toEqual({ + index: 1, + name: "plugin__lookup", + pointer: "/$defs/RecordSource/oneOf/1/properties/evidence", + }); + }); + + it("matches on the pre-rewrite schema when the sent one is already an object", () => { + const sent = tool("plugin__lookup", { + type: "object", + $defs: { + RecordSource: { + oneOf: [{ type: "object" }, { properties: { evidence: {} } }], + }, + }, + }); + expect( + findToolBehindSchemaError(LM_STUDIO_500, [other, sent], [other, plugin]) + ?.index, + ).toBe(1); + }); + + it("reads a pointer into the request body", () => { + const text = "JSON schema error at #/tools/0/function/parameters/properties/x: bad"; + expect(findToolBehindSchemaError(text, [other, plugin])?.name).toBe( + "os__fs__read", + ); + }); + + it("names nothing when the pointer is ambiguous, missing or unmatched", () => { + const twin = tool("plugin__twin", (plugin.function as any).parameters); + expect(findToolBehindSchemaError(LM_STUDIO_500, [plugin, twin])).toBeNull(); + expect( + findToolBehindSchemaError("JSON schema error: schema must be an object", [plugin]), + ).toBeNull(); + expect(findToolBehindSchemaError(LM_STUDIO_500, [other])).toBeNull(); + expect(findToolBehindSchemaError("http 500", [plugin])).toBeNull(); + }); + + it("never names reply or finish", () => { + const reply = tool("reply", (plugin.function as any).parameters); + expect(findToolBehindSchemaError(LM_STUDIO_500, [reply])).toBeNull(); + }); +}); diff --git a/src/llm/provider/openai/tool-schema-rejection.ts b/src/llm/provider/openai/tool-schema-rejection.ts new file mode 100644 index 000000000..4db278386 --- /dev/null +++ b/src/llm/provider/openai/tool-schema-rejection.ts @@ -0,0 +1,139 @@ +/** + * A server that refused the request over one tool's JSON Schema + * (ATO-1399), and which tool that was. + * + * llama.cpp and LM Studio validate every function's `parameters` and + * answer one schema they cannot turn into a grammar with an error for + * the whole request, often a 500: "JSON schema error at + * #/$defs/RecordSource/oneOf/1/properties/evidence: schema must be an + * object". `tool-schema-booleans.ts` removes the known cause before the + * send; this is the net for anything else a third-party tool carries. + * + * The answer is deterministic, so it is neither retried as-is (see + * `isRetryableOpenAiError`) nor guessed at: a tool is named only when + * the error's JSON pointer resolves inside its schema, and for "must be + * an object" only where the pointer lands on something that is not an + * object. No unique match means no drop, and the original error goes + * back to the caller. `reply` and `finish` are never dropped. + * + * A leaf with no imports, so the HTTP client and the provider can both + * read it without a cycle. + */ + +/** The wording llama.cpp / LM Studio use for an unusable tool schema. */ +const SCHEMA_ERROR = /JSON schema error|schema must be/i; + +/** "must be an object": the pointer names a non-object subschema. */ +const MUST_BE_OBJECT = /must be an object/i; + +/** The pointer after `at #` (or a bare `#/...`), up to `:` or whitespace. */ +const POINTER = /#(\/[^\s:'",]*)/; + +/** A pointer into the request body rather than into one tool's schema. */ +const BODY_POINTER = /^\/tools\/(\d+)\/function\/parameters(\/.*)?$/; + +/** The turn's own terminal tools; without them a turn cannot end. */ +const PROTECTED_TOOLS: ReadonlySet = new Set(["reply", "finish"]); + +/** Whether an error message is a server refusing a tool's JSON Schema. */ +export function isToolSchemaErrorText(text: string): boolean { + return SCHEMA_ERROR.test(text); +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function decodeSegment(segment: string): string { + let text = segment; + try { + text = decodeURIComponent(segment); + } catch { + // Not percent-encoded after all; use it as written. + } + return text.replace(/~1/g, "/").replace(/~0/g, "~"); +} + +/** Walks a JSON pointer; `{ found: false }` when any step is missing. */ +function resolvePointer( + root: unknown, + segments: readonly string[], +): { found: true; value: unknown } | { found: false } { + let node = root; + for (const segment of segments) { + if (Array.isArray(node)) { + if (!/^\d+$/.test(segment)) return { found: false }; + const index = Number(segment); + if (index >= node.length) return { found: false }; + node = node[index]; + } else if (isPlainObject(node)) { + if (!Object.hasOwn(node, segment)) return { found: false }; + node = node[segment]; + } else { + return { found: false }; + } + } + return { found: true, value: node }; +} + +function functionOf(tool: Record): { + name: string; + parameters: unknown; +} | null { + const fn = tool.function; + if (!isPlainObject(fn) || typeof fn.name !== "string") return null; + return { name: fn.name, parameters: fn.parameters }; +} + +export interface ToolSchemaCulprit { + /** Index into the tools array the error was matched against. */ + index: number; + /** The function name as it went on the wire (escaped). */ + name: string; + /** The JSON pointer the server named, for the warning. */ + pointer: string; +} + +/** + * The one tool a schema error is about, or null when the text is not a + * schema error or no single tool can be named from it. `schemas` are + * the tools as sent; `alternates` (same order) are the tools before any + * rewrite, consulted too because a pointer the server computed on one + * may still resolve only on the other. + */ +export function findToolBehindSchemaError( + text: string, + tools: ReadonlyArray>, + alternates: ReadonlyArray> = [], +): ToolSchemaCulprit | null { + if (!isToolSchemaErrorText(text)) return null; + const match = POINTER.exec(text); + if (!match) return null; + const pointer = match[1]!; + const body = BODY_POINTER.exec(pointer); + if (body) { + const index = Number(body[1]); + const fn = tools[index] ? functionOf(tools[index]!) : null; + if (!fn || PROTECTED_TOOLS.has(fn.name)) return null; + return { index, name: fn.name, pointer }; + } + const segments = pointer.split("/").slice(1).map(decodeSegment); + const needsNonObject = MUST_BE_OBJECT.test(text); + const hits: number[] = []; + tools.forEach((tool, index) => { + const fn = functionOf(tool); + if (!fn || PROTECTED_TOOLS.has(fn.name)) return; + const candidates = [fn.parameters]; + const alt = alternates[index] ? functionOf(alternates[index]!) : null; + if (alt && alt.name === fn.name) candidates.push(alt.parameters); + const matches = candidates.some((schema) => { + const resolved = resolvePointer(schema, segments); + if (!resolved.found) return false; + return needsNonObject ? !isPlainObject(resolved.value) : true; + }); + if (matches) hits.push(index); + }); + if (hits.length !== 1) return null; + const index = hits[0]!; + return { index, name: functionOf(tools[index]!)!.name, pointer }; +} From a6906ed025df0c7bc946212940ad33cc243a38e4 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:15:55 +0300 Subject: [PATCH 033/155] Reserve reply room for the reasoning budget on small windows On a 16K window the reply reserve is 4096 tokens. A local reasoning model thinks inside that reserve, so a reasoningBudgetTokens above about 3000 used it up and the tool call was cut. The reserve now holds at least the bounded reasoning budget plus 1024 tokens for the tool call, still capped by completionMaxTokens and half the window. It applies only when the profile reasons and thinking is not off. At load time the runtime warns when the budget still does not fit. Automatic compaction that finds no cut on a model without prefix reuse now says so once per session and window, like the existing "window too small" notice, instead of trimming turns silently. ATO-1295 --- src/prompt/build-prompt-types.ts | 7 ++ src/prompt/build-prompt.ts | 10 ++ src/prompt/docs/assembly.md | 2 +- src/prompt/token-budget.test.ts | 57 ++++++++++++ src/prompt/token-budget.ts | 86 +++++++++++++++-- .../composition/runtime-local-profile.ts | 38 +++++++- .../context-compaction-no-slice.test.ts | 92 +++++++++++++++++++ src/runtime/context-compaction.ts | 20 +++- 8 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 src/runtime/context-compaction-no-slice.test.ts diff --git a/src/prompt/build-prompt-types.ts b/src/prompt/build-prompt-types.ts index 2640831e0..28480bfc4 100644 --- a/src/prompt/build-prompt-types.ts +++ b/src/prompt/build-prompt-types.ts @@ -147,6 +147,13 @@ export interface BuildPromptInput { sessionSectionsMaxTokens?: number; worldSnapshotMaxTokens?: number; completionMaxTokens?: number; + /** + * Overrides `localModels.reasoningBudgetTokens` for this build. Only + * counted when the profile reasons and thinking is not switched off: + * the reply reserve then keeps room for the tool call after the think + * block (ATO-1295). + */ + reasoningBudgetTokens?: number; transientNotice?: string; /** * The serving route changed since the session's previous turn diff --git a/src/prompt/build-prompt.ts b/src/prompt/build-prompt.ts index 8d6f1f6a5..067957a91 100644 --- a/src/prompt/build-prompt.ts +++ b/src/prompt/build-prompt.ts @@ -131,6 +131,15 @@ export function buildPrompt(input: BuildPromptInput): BuiltPrompt { input.sessionSectionsMaxTokens ?? config.agent.sessionSectionsMaxTokens; const completionMaxTokens = input.completionMaxTokens ?? config.localModels.completionMaxTokens; + // A reasoning model thinks inside the reply reserve before its tool + // call; a bounded think block is reserved for on top (ATO-1295). Not + // when the profile has no reasoning channel or thinking is off. + const reasoningBudgetTokens = + input.profile !== undefined && + input.profile.reasoningStyle !== "none" && + (input.thinking ?? config.localModels.thinking) !== "off" + ? (input.reasoningBudgetTokens ?? config.localModels.reasoningBudgetTokens) + : 0; // Under auto the conversation share is pinned to the schema default // rather than left to `defaultBudget`'s `tokenBudget * 0.35`. @@ -380,6 +389,7 @@ export function buildPrompt(input: BuildPromptInput): BuiltPrompt { proceduresTokens, loadedToolsTokens, completionMaxTokens, + reasoningBudgetTokens, }); // One option set for every pack of this build: the `### request` diff --git a/src/prompt/docs/assembly.md b/src/prompt/docs/assembly.md index 8826678b4..224930b9b 100644 --- a/src/prompt/docs/assembly.md +++ b/src/prompt/docs/assembly.md @@ -25,7 +25,7 @@ Empty optional sections are omitted. Notice composition and structured messages ## Local budget and packing -Configured caps, context-window detection, reserved completion capacity and section budgets jointly bound the prompt. The reply reserve of the flat assembly (`effectiveReplyReserve`) is `localModels.completionMaxTokens`, at most half a known window, and on any route with a known window under 32k at most what leaves the prompt 16k (never below a quarter of the window or 1024): 8k reserves 2048, 16k 4096, 24k 8192, 32k and up are unchanged. With half of a 16k window reserved the default scaffold left the transcript only the 512-token floor. The reserve is the budget's figure, not `n_predict`. ConversationMaxTokens=0 means auto; the configured fallback when a window is unknown is not an arbitrary tiny token share. The packer retains its cut between overflows and cuts deeper when prefix reuse is unavailable; for the same reason [the compaction planner](../plan-compaction.ts) keeps a smaller tail there. Profile, lessons, procedures and loaded sections consume room that must not be counted twice. +Configured caps, context-window detection, reserved completion capacity and section budgets jointly bound the prompt. The reply reserve of the flat assembly (`effectiveReplyReserve`) is `localModels.completionMaxTokens`, at most half a known window, and on any route with a known window under 32k at most what leaves the prompt 16k (never below a quarter of the window or 1024): 8k reserves 2048, 16k 4096, 24k 8192, 32k and up are unchanged. With half of a 16k window reserved the default scaffold left the transcript only the 512-token floor. A local reasoning model that thinks on the prompt with a bounded `localModels.reasoningBudgetTokens` gets at least that budget plus 1024 tokens for the tool call, within the cap and half the window (ATO-1295): on 16k a 4000 budget reserves 5024 instead of 4096, and the runtime warns at load time when even that cannot fit. The reserve is the budget's figure, not `n_predict`. ConversationMaxTokens=0 means auto; the configured fallback when a window is unknown is not an arbitrary tiny token share. The packer retains its cut between overflows and cuts deeper when prefix reuse is unavailable; for the same reason [the compaction planner](../plan-compaction.ts) keeps a smaller tail there. Profile, lessons, procedures and loaded sections consume room that must not be counted twice. Read the implementation for exact defaults; historical ~2.5k targets are not a universal current window ceiling. diff --git a/src/prompt/token-budget.test.ts b/src/prompt/token-budget.test.ts index 9f61273b7..bc304293f 100644 --- a/src/prompt/token-budget.test.ts +++ b/src/prompt/token-budget.test.ts @@ -6,7 +6,9 @@ import { defaultBudget, effectiveReplyReserve, estimateTokens, + MIN_REPLY_AFTER_REASONING_TOKENS, minUsableContextWindow, + reasoningBudgetExceedsReplyReserve, AGENT_FIXED_PROMPT_TOKENS_COMPACT, SESSION_SECTIONS_BUDGET_SHARE, SESSION_SECTIONS_CAP_AUTO, @@ -254,6 +256,61 @@ describe("effectiveReplyReserve", () => { expect(effectiveReplyReserve(96_000, 131_072)).toBe(65_536); expect(effectiveReplyReserve(8192, 32_768)).toBe(8192); }); + + /** + * ATO-1295: a reasoning model thinks inside the reply reserve. On a 16k + * window the reserve was 4096, so a reasoning budget above about 3000 + * left the tool call no room and it was cut. + */ + describe("with a bounded reasoning budget", () => { + it("raises a 16k window's reserve to the budget plus room for the tool call", () => { + expect(MIN_REPLY_AFTER_REASONING_TOKENS).toBe(1024); + expect(effectiveReplyReserve(16_384, 16_384, 4000)).toBe(5024); + expect(effectiveReplyReserve(96_000, 16_384, 3500)).toBe(4524); + }); + + it("keeps the reserve when the budget already fits it", () => { + expect(effectiveReplyReserve(16_384, 16_384, 2000)).toBe(4096); + expect(effectiveReplyReserve(16_384, 32_768, 4000)).toBe(16_384); + }); + + it("never goes past half the window or the configured cap", () => { + expect(effectiveReplyReserve(16_384, 16_384, 12_000)).toBe(8192); + expect(effectiveReplyReserve(4500, 16_384, 4000)).toBe(4500); + }); + + it("changes nothing for an unbounded budget, an unknown window or the no-cap sentinel", () => { + expect(effectiveReplyReserve(16_384, 16_384, 0)).toBe(4096); + expect(effectiveReplyReserve(16_384, null, 4000)).toBe(16_384); + expect(effectiveReplyReserve(0, 16_384, 4000)).toBe(0); + }); + + it("flags a budget that still does not fit after the reserve was raised", () => { + expect(reasoningBudgetExceedsReplyReserve(16_384, 16_384, 4000)).toBe(false); + // Half of 16k is 8192: a 7500 budget plus 1024 cannot be reserved. + expect(reasoningBudgetExceedsReplyReserve(16_384, 16_384, 7500)).toBe(true); + // A configured cap below budget + room cuts the reply on any window. + expect(reasoningBudgetExceedsReplyReserve(4096, 131_072, 4000)).toBe(true); + expect(reasoningBudgetExceedsReplyReserve(16_384, 16_384, 0)).toBe(false); + expect(reasoningBudgetExceedsReplyReserve(0, 16_384, 4000)).toBe(false); + expect(reasoningBudgetExceedsReplyReserve(0, 16_384, 7500)).toBe(true); + }); + + it("lowers the conversation cap by the raised reserve", () => { + const base = { + configuredCap: 100_000, + autoFill: true, + contextWindow: 16_384, + stablePrefixTokens: 4000, + sessionTokens: 0, + worldSnapshotTokens: 0, + completionMaxTokens: 16_384, + }; + const plain = computeEffectiveConversationCap(base); + const reasoning = computeEffectiveConversationCap({ ...base, reasoningBudgetTokens: 4000 }); + expect(plain - reasoning).toBe(5024 - 4096); + }); + }); }); describe("computeEffectiveConversationCap with a reply cap past the window", () => { diff --git a/src/prompt/token-budget.ts b/src/prompt/token-budget.ts index 86bc0396c..1424e52ae 100644 --- a/src/prompt/token-budget.ts +++ b/src/prompt/token-budget.ts @@ -182,6 +182,13 @@ export interface EffectiveConversationCapInput { */ loadedToolsTokens?: number; completionMaxTokens: number; + /** + * Bounded think block of a local reasoning model that thinks on this + * prompt, so the reply reserve keeps room for the tool call after it + * (see {@link effectiveReplyReserve}). Omitted or `0`: no reasoning to + * account for. + */ + reasoningBudgetTokens?: number; /** * `agent.conversationMaxTokens` was left at {@link CONVERSATION_CAP_AUTO}: * the transcript takes whatever the window leaves rather than being @@ -257,6 +264,16 @@ export const SMALL_WINDOW_REPLY_SHARE = 0.25; /** The reply reserve never shrinks below this on a small window (unless the cap or half the window is lower). */ export const MIN_SMALL_WINDOW_REPLY_RESERVE = 1024; +/** + * Room a bounded think block must leave for the tool call after it + * (ATO-1295). A reasoning model spends its reply reserve on thinking + * first; when `localModels.reasoningBudgetTokens` is close to the + * reserve (4096 on a 16k window) the think block eats it and the tool + * call is cut. The reserve is raised to the reasoning budget plus this + * much, within the same caps. + */ +export const MIN_REPLY_AFTER_REASONING_TOKENS = 1024; + /** * The reply reservation the prompt budget actually uses on a given * window: `completionMaxTokens`, never more than @@ -269,6 +286,15 @@ export const MIN_SMALL_WINDOW_REPLY_RESERVE = 1024; * (nothing to compare it against), and a non-positive cap (`0`, "no * client-side cap") reserves nothing, as before. * + * `reasoningBudgetTokens`: the bounded think block of a local reasoning + * model (`localModels.reasoningBudgetTokens`, only when the model really + * thinks on this prompt; `0` or omitted for an unbounded budget, a plain + * model or thinking switched off). The reserve then holds at least that + * budget plus {@link MIN_REPLY_AFTER_REASONING_TOKENS} for the tool + * call, never more than `completionMaxTokens` or half the window + * (ATO-1295). Without it a 16k window reserved 4096 tokens, and a + * reasoning budget above about 3000 left the tool call no room. + * * This is the budget's reservation, not `n_predict`: the request still * carries the configured cap, and a local reply may run past this figure * into whatever the prompt left free, up to the end of the window. @@ -276,20 +302,60 @@ export const MIN_SMALL_WINDOW_REPLY_RESERVE = 1024; export function effectiveReplyReserve( completionMaxTokens: number, contextWindow: number | null | undefined, + reasoningBudgetTokens = 0, ): number { if (!Number.isFinite(completionMaxTokens) || completionMaxTokens <= 0) { return 0; } if (!contextWindow || contextWindow <= 0) return completionMaxTokens; + const halfWindow = Math.floor(contextWindow * REPLY_RESERVE_MAX_WINDOW_SHARE); const smallWindowCap = Math.max( MIN_SMALL_WINDOW_REPLY_RESERVE, Math.floor(contextWindow * SMALL_WINDOW_REPLY_SHARE), contextWindow - SMALL_WINDOW_PROMPT_ROOM, ); - return Math.min( + const reserve = Math.min(completionMaxTokens, halfWindow, smallWindowCap); + if (!Number.isFinite(reasoningBudgetTokens) || reasoningBudgetTokens <= 0) { + return reserve; + } + const withReasoning = Math.min( completionMaxTokens, - Math.floor(contextWindow * REPLY_RESERVE_MAX_WINDOW_SHARE), - smallWindowCap, + halfWindow, + Math.floor(reasoningBudgetTokens) + MIN_REPLY_AFTER_REASONING_TOKENS, + ); + return Math.max(reserve, withReasoning); +} + +/** + * True when a bounded reasoning budget plus room for the tool call does + * not fit the reply reserve this window can give, even after + * `effectiveReplyReserve` raised it: the think block can still eat the + * reply. The runtime warns about it at load time (ATO-1295). + */ +export function reasoningBudgetExceedsReplyReserve( + completionMaxTokens: number, + contextWindow: number | null | undefined, + reasoningBudgetTokens: number, +): boolean { + if (!Number.isFinite(reasoningBudgetTokens) || reasoningBudgetTokens <= 0) { + return false; + } + if (!Number.isFinite(completionMaxTokens) || completionMaxTokens <= 0) { + // No client-side cap: nothing in the request cuts the reply, but a + // known window still bounds what the budget can reserve. + if (!contextWindow || contextWindow <= 0) return false; + return ( + Math.floor(reasoningBudgetTokens) + MIN_REPLY_AFTER_REASONING_TOKENS > + Math.floor(contextWindow * REPLY_RESERVE_MAX_WINDOW_SHARE) + ); + } + return ( + Math.floor(reasoningBudgetTokens) + MIN_REPLY_AFTER_REASONING_TOKENS > + effectiveReplyReserve( + completionMaxTokens, + contextWindow, + reasoningBudgetTokens, + ) ); } @@ -314,12 +380,16 @@ export function effectiveReplyReserve( export function minUsableContextWindow( completionMaxTokens: number, contextWindow?: number | null, - options?: { compact?: boolean }, + options?: { compact?: boolean; reasoningBudgetTokens?: number }, ): number { return ( (options?.compact ? AGENT_FIXED_PROMPT_TOKENS_COMPACT : AGENT_FIXED_PROMPT_TOKENS) + (contextWindow - ? effectiveReplyReserve(completionMaxTokens, contextWindow) + ? effectiveReplyReserve( + completionMaxTokens, + contextWindow, + options?.reasoningBudgetTokens, + ) : completionMaxTokens) + CONVERSATION_CAP_SAFETY_MARGIN ); @@ -399,7 +469,11 @@ export function computeEffectiveConversationCap( (input.lessonsTokens ?? 0) - (input.proceduresTokens ?? 0) - (input.loadedToolsTokens ?? 0) - - effectiveReplyReserve(input.completionMaxTokens, input.contextWindow) - + effectiveReplyReserve( + input.completionMaxTokens, + input.contextWindow, + input.reasoningBudgetTokens, + ) - CONVERSATION_CAP_SAFETY_MARGIN; if (input.autoFill) return Math.max(CONVERSATION_CAP_FLOOR, available); return Math.max( diff --git a/src/runtime/composition/runtime-local-profile.ts b/src/runtime/composition/runtime-local-profile.ts index 255eb3629..e63c1989f 100644 --- a/src/runtime/composition/runtime-local-profile.ts +++ b/src/runtime/composition/runtime-local-profile.ts @@ -18,7 +18,12 @@ import { resolveLlmConfig } from "../../llm/provider/index.js"; import { activeTextProviderIsLlamaServer } from "../../llm/provider/registry/active-text-provider.js"; import { DeferredLocalBackendProbes } from "../../llm/local-backend-gate.js"; import { readLaunchRecord, readRunningPid, readThroughputRecord } from "../../local-llm/index.js"; -import { minUsableContextWindow } from "../../prompt/token-budget.js"; +import { + effectiveReplyReserve, + MIN_REPLY_AFTER_REASONING_TOKENS, + minUsableContextWindow, + reasoningBudgetExceedsReplyReserve, +} from "../../prompt/token-budget.js"; type RuntimeInferenceOptions = Pick; @@ -132,11 +137,40 @@ export async function prepareRuntimeLocalProfile( config.agent.prompt?.compact ?? "auto", candidate.contextWindow, ).compact; + // The bounded think block a reasoning model spends inside the reply + // reserve (ATO-1295) — the same rule `buildPrompt` applies. + const reasoningBudgetTokens = + candidate.reasoningStyle !== "none" && + config.localModels.thinking !== "off" + ? config.localModels.reasoningBudgetTokens + : 0; const minUsableCtx = minUsableContextWindow( config.localModels.completionMaxTokens, candidate.contextWindow, - { compact: compactPrompt }, + { compact: compactPrompt, reasoningBudgetTokens }, ); + if ( + candidate.contextWindow && + reasoningBudgetExceedsReplyReserve( + config.localModels.completionMaxTokens, + candidate.contextWindow, + reasoningBudgetTokens, + ) + ) { + logger.warn("reasoning budget does not leave room for the reply", { + contextWindow: candidate.contextWindow, + reasoningBudgetTokens, + replyReserve: effectiveReplyReserve( + config.localModels.completionMaxTokens, + candidate.contextWindow, + reasoningBudgetTokens, + ), + completionMaxTokens: config.localModels.completionMaxTokens, + minReplyAfterReasoning: MIN_REPLY_AFTER_REASONING_TOKENS, + hint: + "lower localModels.reasoningBudgetTokens, raise localModels.completionMaxTokens, or use a larger context window; otherwise thinking can use up the reply and cut the tool call", + }); + } if (candidate.contextWindow && candidate.contextWindow < minUsableCtx) { logger.warn("context window too small for the agent prompt", { contextWindow: candidate.contextWindow, diff --git a/src/runtime/context-compaction-no-slice.test.ts b/src/runtime/context-compaction-no-slice.test.ts new file mode 100644 index 000000000..106e3c5eb --- /dev/null +++ b/src/runtime/context-compaction-no-slice.test.ts @@ -0,0 +1,92 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resetConfigCache } from "../config/index.js"; +import { createCompactionDefaults } from "../config/agent/compaction-config.js"; +import { createEmptySessionState, recordTurn, type SessionState } from "../session/session-state.js"; +import { assistantToolCallTurn, toolResultTurn, userTurn } from "../session/conversation-turn.js"; +import type { BuildPromptInput } from "../prompt/build-prompt-types.js"; +import { PLAIN_INSTRUCT_PROFILE, type PrefixReuse } from "../llm/model-profile.js"; +import type { CompactionEvent } from "../agent/compaction-control.js"; +import { TurnController } from "./turn-controller.js"; +import { createContextCompaction, noSliceFitsMessage } from "./context-compaction.js"; + +// ATO-1295: the planner's "no cut fits" outcome is forced here, so the +// test is about what the runtime says about it, not about building a +// transcript that happens to defeat every cut. +vi.mock("../prompt/plan-compaction.js", async (importOriginal) => { + const original = await importOriginal(); + return { ...original, decideCompaction: vi.fn(() => ({ plan: null, skip: "no_reduction" })) }; +}); + +let dir: string; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "context-compaction-no-slice-")); + vi.stubEnv("ATOMIC_AGENT_STATE_DIR", dir); + resetConfigCache(); +}); +afterEach(() => { vi.unstubAllEnvs(); resetConfigCache(); rmSync(dir, { recursive: true, force: true }); }); + +function history(): SessionState { + let state = recordTurn(createEmptySessionState({ id: "chat", workingDir: dir }), userTurn("read the files")); + for (let i = 0; i < 4; i++) { + state = recordTurn(state, assistantToolCallTurn({ tool: "os.fs.read", args: { path: `/file-${i}` } })); + state = recordTurn(state, toolResultTurn({ tool: "os.fs.read", status: "ok", summary: `result ${i}` })); + } + return state; +} + +function input(session: SessionState, prefixReuse: PrefixReuse, contextWindow = 16_384): BuildPromptInput { + return { session, toolDescriptors: [], skillCatalog: [], profile: { ...PLAIN_INSTRUCT_PROFILE, prefixReuse }, + contextWindow, conversationMaxTokens: 4000, conversationMaxPairs: 200, + capabilities: { platform: "darwin", arch: "arm64", browserChannel: "chrome", workingDir: dir, hasClipboard: false, hasWmctrl: false, hasNotifications: false } }; +} + +function fixture(state: SessionState) { + const events: CompactionEvent[] = []; + const warn = vi.fn(); + const complete = vi.fn(); + const service = createContextCompaction({ + config: () => createCompactionDefaults(), sessionStore: { load: (id) => id === state.id ? state : null }, + turnController: new TurnController(), complete, warn, persist: vi.fn(), + route: (_id: string, providerId?: string) => ({ providerId: providerId ?? "current", transport: "grammar" as const, serverTemplate: true }), + emit: (event) => events.push(event), sideCallSlotId: () => 3, promptInput: (s) => input(s, "none"), + }); + return { ...service, events, warn, complete }; +} + +describe("automatic compaction with no cut that fits (ATO-1295)", () => { + it("says so once per session and window on a model without prefix reuse", () => { + const state = history(); + const f = fixture(state); + const signal = new AbortController().signal; + expect(f.control.beforeStep(input(state, "none"), { signal })).toBeUndefined(); + expect(f.control.beforeStep(input(state, "none"), { signal })).toBeUndefined(); + const notices = f.events.filter((event) => event.type === "compaction_failed"); + expect(notices).toEqual([{ type: "compaction_failed", sessionId: state.id, + result: { status: "noop", reason: "threshold", message: noSliceFitsMessage(16_384) }, + usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 }, calls: 0 }]); + expect(f.warn).toHaveBeenCalledTimes(1); + expect(f.complete).not.toHaveBeenCalled(); + // Another window is told again, once. + f.control.beforeStep(input(state, "none", 24_576), { signal }); + f.control.beforeStep(input(state, "none", 24_576), { signal }); + expect(f.events.filter((event) => event.type === "compaction_failed")).toHaveLength(2); + }); + + it("stays quiet on a model with partial prefix reuse", () => { + const state = history(); + const f = fixture(state); + const signal = new AbortController().signal; + f.control.beforeStep(input(state, "partial"), { signal }); + expect(f.events.filter((event) => event.type === "compaction_failed")).toEqual([]); + }); + + it("names the window and the fix", () => { + expect(noSliceFitsMessage(16_384)).toContain("(16384 tokens)"); + expect(noSliceFitsMessage(16_384)).toContain("older turns are trimmed instead of summarized"); + expect(noSliceFitsMessage(16_384)).toContain("raise the context size"); + expect(noSliceFitsMessage(null)).not.toContain("tokens)"); + }); +}); diff --git a/src/runtime/context-compaction.ts b/src/runtime/context-compaction.ts index 1f5abccb1..1ab085dee 100644 --- a/src/runtime/context-compaction.ts +++ b/src/runtime/context-compaction.ts @@ -78,6 +78,19 @@ export function skipMessage(skip: CompactionSkip, contextWindow?: number | null) return SKIP_MESSAGES[skip]; } +/** + * What automatic compaction says, once per session and window, when no + * cut fits on a model without prefix reuse (ATO-1295). Before this the + * skip was silent and trimming dropped turns with no word; only a window + * too small for any summary was named. Kept to the no-reuse case, where a + * small window makes this the usual outcome rather than a passing one. + */ +export function noSliceFitsMessage(contextWindow?: number | null): string { + const window = contextWindow && contextWindow > 0 ? ` (${contextWindow} tokens)` : ""; + return `Automatic compaction found no safe cut that fits the model's context window${window}: ` + + "the newest steps alone fill the history it may keep, so older turns are trimmed instead of summarized; raise the context size to keep a summary."; +} + /** A control inbox, separate from user messages and from the FIFO's occupied lock. */ export function createContextCompaction(deps: ContextCompactionDependencies) { const accepting = new Map(); @@ -99,8 +112,8 @@ export function createContextCompaction(deps: ContextCompactionDependencies) { * silently. A `compaction_failed` event with a `noop` result reaches the * TUI, traces and HTTP streams like any other compaction outcome. */ - function noticeTooSmall(id: string, contextWindow: number | null, reason: CompactionResult["reason"], message: string) { - const key = JSON.stringify([id, contextWindow]); + function noticeTooSmall(id: string, contextWindow: number | null, reason: CompactionResult["reason"], message: string, kind: CompactionSkip = "window_too_small") { + const key = JSON.stringify([id, contextWindow, kind]); if (tooSmallNoticed.has(key)) return; tooSmallNoticed.add(key); deps.warn?.(id, message); @@ -150,6 +163,9 @@ export function createContextCompaction(deps: ContextCompactionDependencies) { const message = skipMessage(skip ?? "no_reduction", prompt.contextWindow); if (request) request.settle({ status: "noop", reason, message }); else if (skip === "window_too_small") noticeTooSmall(id, prompt.contextWindow, reason, message); + else if (skip === "no_reduction" && prefixReuse === "none") { + noticeTooSmall(id, prompt.contextWindow, reason, noSliceFitsMessage(prompt.contextWindow), "no_reduction"); + } return undefined; } const controller = request?.controller ?? new AbortController(); From a11483091e98b6569e78acfd75d3df7c9266fa84 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:16:17 +0300 Subject: [PATCH 034/155] fix(overnight): say when the Telegram report could not be sent overnight --notify telegram queues its report in memory until its own Telegram channel is up. When the desktop app or the TUI holds telegram.lock the channel never comes up in the overnight process, which then exits and the report is gone without a word. Before exiting, overnight now gives a still-connecting channel up to 15 s, and if the report is still waiting prints why it was not sent (naming the other agent holding Telegram) and where to read the result. Sending through the lock holder is left as a follow-up. ATO-1358 --- .../telegram/telegram-channel.test.ts | 5 +++ src/channels/telegram/telegram-channel.ts | 9 ++++++ src/cli/overnight-command.ts | 32 +++++++++++++++++++ src/cli/overnight-progress.test.ts | 24 ++++++++++++++ src/cli/overnight-progress.ts | 28 ++++++++++++++++ 5 files changed, 98 insertions(+) diff --git a/src/channels/telegram/telegram-channel.test.ts b/src/channels/telegram/telegram-channel.test.ts index ba218f1a2..52025eddc 100644 --- a/src/channels/telegram/telegram-channel.test.ts +++ b/src/channels/telegram/telegram-channel.test.ts @@ -1244,11 +1244,16 @@ describe("TelegramChannel live-control surface", () => { const delivery = await channel.sendTaskReport(makeReport()); expect(delivery).toBe("queued"); expect(state.sendMessageCalls).toHaveLength(0); + // What `overnight` reads before it exits (ATO-1358). + const queuedId = makeReport().taskId; + expect(channel.hasPendingTaskReport(queuedId)).toBe(true); + expect(channel.hasPendingTaskReport("t-other")).toBe(false); await channel.start(); await vi.waitFor(() => { expect(state.sendMessageCalls).toHaveLength(1); }); + expect(channel.hasPendingTaskReport(queuedId)).toBe(false); expect(state.sendMessageCalls[0]!.chatId).toBe(42); expect(state.sendMessageCalls[0]!.text).toContain( "✅ Scheduled task completed", diff --git a/src/channels/telegram/telegram-channel.ts b/src/channels/telegram/telegram-channel.ts index 467fdcaef..1fd1b2f4d 100644 --- a/src/channels/telegram/telegram-channel.ts +++ b/src/channels/telegram/telegram-channel.ts @@ -1146,6 +1146,15 @@ export class TelegramChannel { }; } + /** + * Whether a report for `taskId` is still waiting for the channel to + * come up. A short-lived process (`overnight`) reads it before it + * exits, since the in-memory queue goes with it (ATO-1358). + */ + hasPendingTaskReport(taskId: string): boolean { + return this.pendingTaskReports.some((report) => report.taskId === taskId); + } + /** FIFO-append with oldest-first eviction at `TASK_REPORT_QUEUE_LIMIT`. */ private queueTaskReport(report: TaskReport): void { this.pendingTaskReports.push(report); diff --git a/src/cli/overnight-command.ts b/src/cli/overnight-command.ts index 192302752..e2ea03a60 100644 --- a/src/cli/overnight-command.ts +++ b/src/cli/overnight-command.ts @@ -38,6 +38,7 @@ import { formatOvernightSummary, formatProgressLine, overnightExitCode, + formatTelegramUndelivered, replyFromEvent, } from "./overnight-progress.js"; @@ -344,6 +345,7 @@ async function runForeground(args: OvernightArgs, taskId: string): Promise>; +/** How long a channel still connecting may take to send the report. */ +const TELEGRAM_FLUSH_WAIT_MS = 15_000; + +/** + * The Telegram report waits in this process's memory until the channel + * is up, and this process exits next (ATO-1358). When the desktop app + * or the TUI holds the connection the channel never comes up here, and + * the report would vanish without a word: say so instead. + */ +async function warnIfReportStuck(runtime: OvernightRuntime, taskId: string): Promise { + // The report is handed to the channel from the runner's report hook. + await new Promise((resolve) => setImmediate(resolve)); + const channel = runtime.telegramChannel; + if (!channel) { + process.stderr.write(formatTelegramUndelivered("Telegram is not set up for this agent", taskId)); + return; + } + const deadline = Date.now() + TELEGRAM_FLUSH_WAIT_MS; + while ( + channel.hasPendingTaskReport(taskId) && + channel.state() === "starting" && + Date.now() < deadline + ) { + await new Promise((resolve) => setTimeout(resolve, 500)); + } + if (channel.hasPendingTaskReport(taskId)) { + process.stderr.write(formatTelegramUndelivered(channel.lastError(), taskId)); + } +} + /** * Run the task here, or wait for whoever claimed it. Another process's * scheduler may pick the fresh pending row up first; `runOne` then hands diff --git a/src/cli/overnight-progress.test.ts b/src/cli/overnight-progress.test.ts index 100bb609b..384c7119a 100644 --- a/src/cli/overnight-progress.test.ts +++ b/src/cli/overnight-progress.test.ts @@ -6,6 +6,7 @@ import { formatElapsed, formatOvernightSummary, formatProgressLine, + formatTelegramUndelivered, overnightExitCode, replyFromEvent, } from "./overnight-progress.js"; @@ -133,3 +134,26 @@ describe("overnight progress", () => { expect(overnightExitCode("cancelled")).toBe(1); }); }); + +describe("formatTelegramUndelivered (ATO-1358)", () => { + it("names the other agent holding Telegram", () => { + const line = formatTelegramUndelivered( + "channel-locked: already running in another atomic-agent (pid 4242)", + "t-9", + ); + expect(line).toContain("telegram report not sent"); + expect(line).toContain("pid 4242"); + expect(line).toContain("the desktop app or the TUI"); + expect(line).toContain("atomic-agent task show t-9"); + expect(line).not.toContain("channel-locked:"); + }); + + it("falls back to the channel's own error", () => { + expect(formatTelegramUndelivered("getMe failed", "t-1")).toContain( + "the Telegram channel did not come up (getMe failed)", + ); + expect(formatTelegramUndelivered(null, "t-1")).toContain( + "the Telegram channel did not come up.", + ); + }); +}); diff --git a/src/cli/overnight-progress.ts b/src/cli/overnight-progress.ts index 6dc6c19f1..c1eafbf41 100644 --- a/src/cli/overnight-progress.ts +++ b/src/cli/overnight-progress.ts @@ -4,6 +4,10 @@ * closing summary. Pure, so the wording is pinned by unit tests. */ import type { AgentLoopEvent } from "../agent/index.js"; +import { + describeChannelLockConflict, + isChannelLockConflict, +} from "../channels/channel-lock-error.js"; import { formatUsd } from "../llm/provider/usage-cost.js"; import type { TaskRecord } from "../tasks/index.js"; @@ -115,6 +119,30 @@ export function formatDetachedNotice(taskId: string, logPath: string, dir: strin ].join("\n"); } +/** + * The line `overnight --notify telegram` prints when its report never + * left (ATO-1358). The report waits in this process's memory for the + * channel, and this process is about to exit. The usual cause: the + * desktop app or the TUI holds the Telegram connection (`telegram.lock`). + */ +export function formatTelegramUndelivered(lastError: string | null, taskId: string): string { + let why: string; + if (isChannelLockConflict(lastError)) { + // "already running in another atomic-agent (pid 123)" + const held = describeChannelLockConflict(lastError ?? ""); + why = `Telegram is ${held || "already running in another atomic-agent"} ` + + "(the desktop app or the TUI), so this run could not use it"; + } else { + why = lastError + ? `the Telegram channel did not come up (${lastError})` + : "the Telegram channel did not come up"; + } + return ( + ` telegram report not sent: ${why}. ` + + `The result is above and in \`atomic-agent task show ${taskId}\`.\n` + ); +} + export function overnightExitCode(status: TaskRecord["status"]): number { return status === "completed" ? 0 : 1; } From 36d061f967f2ffb64187ca42d1e3533ac7ce1c71 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:16:38 +0300 Subject: [PATCH 035/155] fix(openrouter): run Fable 5.2 at medium effort by default The agent sends no reasoning effort unless a Fusion worker sets one, so Fable 5.2 on OpenRouter ran at the API default, high. On a measured task that cost about four times medium for the same tests passed. A turn that asks for no effort now sends medium for Fable 5.2 and later on OpenRouter; an explicit effort, extraBody or model params still win. --- src/llm/provider/openai/model-params.test.ts | 17 ++++++++++++++ src/llm/provider/openai/model-params.ts | 23 +++++++++++++++++++ .../provider/openai/openai-build-body.test.ts | 20 ++++++++++++++++ src/llm/provider/openai/openai-build-body.ts | 18 ++++++++++----- 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/src/llm/provider/openai/model-params.test.ts b/src/llm/provider/openai/model-params.test.ts index 2a7e336ca..95d61795e 100644 --- a/src/llm/provider/openai/model-params.test.ts +++ b/src/llm/provider/openai/model-params.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { + defaultReasoningEffort, forgetLearnedModelParams, modelParamProfile, rememberAllowedTemperature, @@ -85,3 +86,19 @@ describe("resolveTemperature", () => { expect(resolveTemperature(profile, undefined, 0.2)).toBe(1); }); }); + +describe("defaultReasoningEffort (ATO-1368)", () => { + it.each(["anthropic/claude-fable-5.2", "claude-fable-5-2", "anthropic/claude-fable-5.3", "claude-fable-6"])( + "runs %s at medium unless asked otherwise", + (model) => { + expect(defaultReasoningEffort(model)).toBe("medium"); + }, + ); + + it.each(["anthropic/claude-fable-5", "anthropic/claude-fable-5.1", "anthropic/claude-opus-5.5", "openai/gpt-6.2-sol"])( + "leaves the service default for %s", + (model) => { + expect(defaultReasoningEffort(model)).toBeUndefined(); + }, + ); +}); diff --git a/src/llm/provider/openai/model-params.ts b/src/llm/provider/openai/model-params.ts index 41414478b..498861575 100644 --- a/src/llm/provider/openai/model-params.ts +++ b/src/llm/provider/openai/model-params.ts @@ -123,6 +123,29 @@ export function forgetLearnedModelParams(): void { learnedTemperatures.clear(); } +/** + * Claude Fable 5.2 and later (`anthropic/claude-fable-5.2`, + * `claude-fable-5-2`). The API's own default effort for it is `high`, + * which on a measured task cost about four times `medium` for the same + * result; Claude Code and the Claude apps run it at `medium` (ATO-1368). + */ +const FABLE_MEDIUM_DEFAULT_RE = + /^claude-fable-(?:5[.-](?:[2-9]|[1-9][0-9])|(?:[6-9]|[1-9][0-9]))(?![0-9])/i; + +/** + * The effort to send when the turn asked for none, or `undefined` to + * leave the service's default. Only for a model whose service default + * is known to overspend; every other model keeps its own default. + */ +export function defaultReasoningEffort( + modelId: string, +): "low" | "medium" | "high" | undefined { + const bare = modelId.includes("/") + ? modelId.slice(modelId.lastIndexOf("/") + 1) + : modelId; + return FABLE_MEDIUM_DEFAULT_RE.test(bare) ? "medium" : undefined; +} + /** * How `reasoningEffort` is spelled for a provider kind. OpenRouter takes a * `reasoning` object; OpenAI-compatible services take the flat diff --git a/src/llm/provider/openai/openai-build-body.test.ts b/src/llm/provider/openai/openai-build-body.test.ts index 3198bb0f8..c26c4f018 100644 --- a/src/llm/provider/openai/openai-build-body.test.ts +++ b/src/llm/provider/openai/openai-build-body.test.ts @@ -567,6 +567,26 @@ describe("buildOpenAiChatBody — per-model parameters", () => { }); expect(plain).not.toHaveProperty("reasoning"); }); + + // ATO-1368: Fable 5.2's API default is the expensive `high`. + it("runs Fable 5.2 on OpenRouter at medium when the turn asked for no effort", () => { + const fable = "anthropic/claude-fable-5.2"; + const or = { providerKind: "openrouter" }; + const build = (request: { prompt: string; reasoningEffort?: "low" | "medium" | "high" }, extraBody?: Record) => + buildOpenAiChatBody(request, fable, false, extraBody, undefined, undefined, undefined, or); + expect(build({ prompt: "hi" }).reasoning).toEqual({ effort: "medium" }); + // The turn's own choice wins, and so does the operator's passthrough. + expect(build({ prompt: "hi", reasoningEffort: "high" }).reasoning).toEqual({ effort: "high" }); + expect(build({ prompt: "hi" }, { reasoning: { effort: "low" } }).reasoning).toEqual({ effort: "low" }); + // Other models and other kinds keep the service default. + expect( + buildOpenAiChatBody({ prompt: "hi" }, "anthropic/claude-opus-5.5", false, undefined, undefined, undefined, undefined, or), + ).not.toHaveProperty("reasoning"); + const direct = buildOpenAiChatBody({ prompt: "hi" }, "claude-fable-5-2", false, undefined, undefined, undefined, undefined, { + providerKind: "openai-compatible", + }); + expect(direct).not.toHaveProperty("reasoning_effort"); + }); }); describe("buildOpenAiChatBody — the native message layout", () => { diff --git a/src/llm/provider/openai/openai-build-body.ts b/src/llm/provider/openai/openai-build-body.ts index 39b4404fd..07068bd03 100644 --- a/src/llm/provider/openai/openai-build-body.ts +++ b/src/llm/provider/openai/openai-build-body.ts @@ -3,6 +3,7 @@ import { hasStrictFunctionTools } from "../adapters/tool-call-adapter.js"; import { ensureJsonMention } from "./ensure-json-mention.js"; import { filterCloudCompletionRequest } from "./sampling-filter.js"; import { + defaultReasoningEffort, modelParamProfile, reasoningEffortField, resolveTemperature, @@ -152,12 +153,17 @@ export function buildOpenAiChatBody( filtered.maxTokens ?? filtered.maxOutputTokens ?? maxOutputTokens; if (typeof cap === "number") body[profile.capField] = cap; // Reasoning effort, in the field this kind reads (`model-params.ts`); - // omitted for a kind without a known one rather than guessed. - if (filtered.reasoningEffort !== undefined) { - Object.assign( - body, - reasoningEffortField(options.providerKind, filtered.reasoningEffort), - ); + // omitted for a kind without a known one rather than guessed. A turn + // that asked for none gets the model's own default where the service's + // overspends (Fable on OpenRouter, ATO-1368); `extraBody` and model + // params are merged later and still win. + const effort = + filtered.reasoningEffort ?? + (options.providerKind === "openrouter" + ? defaultReasoningEffort(defaultChatModel) + : undefined); + if (effort !== undefined) { + Object.assign(body, reasoningEffortField(options.providerKind, effort)); } // The template's thinking switch, for a request that asked for none // (context compaction). Only a local server gets it: OpenAI and other From c32dd2b55a78d81474cf91b35c8f6ae617f0f69d Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:16:55 +0300 Subject: [PATCH 036/155] fix(fusion): keep local server health per server With mixLocalModels two local servers can serve one fan-out, but the health mark set after a cut-off worker was shared, and the only probe asked the main server. A wedge on one server could fail a worker sent to the other, and an answer from the main server cleared the other's mark. Marks are now filed under the leg's provider id, and a leg is asked only through its own server's probe. The runtime has a probe for the main local server only, so a second server's leg is not checked rather than checked against the wrong one. (ATO-1318) --- .../composition/runtime-tool-catalog.ts | 14 +++++ src/tools/fusion/local-leg-health.test.ts | 55 ++++++++++++++++++- src/tools/fusion/local-leg-health.ts | 46 ++++++++++------ src/tools/fusion/worker-runner.ts | 17 +++++- 4 files changed, 112 insertions(+), 20 deletions(-) diff --git a/src/runtime/composition/runtime-tool-catalog.ts b/src/runtime/composition/runtime-tool-catalog.ts index 677bda242..0b0e98a01 100644 --- a/src/runtime/composition/runtime-tool-catalog.ts +++ b/src/runtime/composition/runtime-tool-catalog.ts @@ -692,6 +692,20 @@ export function registerRuntimeFusionAndReadScope(input: { // Asked before a local worker is sent after one was cut off // (ATO-917): a server that stopped answering fails the task now. probeLocalHealth: () => llama.probeHealth(), + // Per local server (ATO-1318): this client only ever asks the main + // local server (`localModels.url`), so it is the probe for a leg on + // that server alone. A second local server has no probe here and + // its leg is not checked, rather than checked against the wrong one. + probeLocalHealthFor: (providerId) => { + const entry = resolveLlmConfig(getConfig()).providers.find((p) => p.id === providerId); + if (entry === undefined || entry.kind !== "llama-server") return undefined; + const main = getConfig().localModels.url; + const same = (a: string, b: string): boolean => + a.trim().replace(/\/+$/, "") === b.trim().replace(/\/+$/, ""); + return entry.url === undefined || same(entry.url, main) + ? () => llama.probeHealth() + : undefined; + }, // The turn's $ / time caps (ATO-947), reset at each fusion turn's // start by the loop; read only when a cap is configured. ...(turnBudget === undefined ? {} : { turnBudget }), diff --git a/src/tools/fusion/local-leg-health.test.ts b/src/tools/fusion/local-leg-health.test.ts index 95afd881a..c9a084ed4 100644 --- a/src/tools/fusion/local-leg-health.test.ts +++ b/src/tools/fusion/local-leg-health.test.ts @@ -132,7 +132,7 @@ describe("dispatch after a cut-off local worker (ATO-917)", () => { it("dispatches as before once the server answers, and clears the mark", async () => { const health = new LocalLegHealth(); - health.noteCut("task t9 was cancelled"); + health.noteCut("task t9 was cancelled", "local-llama"); const deps = harness(okTurn, { localLegHealth: health, probeLocalHealth: async () => "answered", @@ -167,4 +167,57 @@ describe("dispatch after a cut-off local worker (ATO-917)", () => { expect(probes).toBe(0); expect(deps.turns).toBe(1); }); + + it("keeps each local server's mark apart and asks only the leg's own server (ATO-1318)", async () => { + const health = new LocalLegHealth(); + health.noteCut("task t9 was timed out", "second-llama"); + const asked: string[] = []; + const deps = harness(okTurn, { + localLegHealth: health, + probeLocalHealthFor: (providerId) => + providerId === "local-llama" + ? async () => { + asked.push(providerId); + return "unanswered"; + } + : undefined, + }); + // The main server was never cut: no question, the worker goes. + const [main] = await runWorkerTasks(deps, { + ...BASE, + tasks: TASK, + signal: new AbortController().signal, + }); + expect(main!.status).not.toBe("failed"); + expect(asked).toEqual([]); + // The cut server has no probe of its own: not checked against the + // main one, and its mark stays. + const [second] = await runWorkerTasks(deps, { + ...BASE, + providerId: "second-llama", + tasks: TASK, + signal: new AbortController().signal, + }); + expect(second!.status).not.toBe("failed"); + expect(asked).toEqual([]); + expect(health.isSuspect("second-llama")).toBe(true); + expect(health.isSuspect("local-llama")).toBe(false); + }); + + it("files a cut under the worker's own server", async () => { + vi.useFakeTimers(); + const health = new LocalLegHealth(); + const deps = harness(servedThenHung, { localLegHealth: health }); + const run = runWorkerTasks(deps, { + ...BASE, + providerId: "second-llama", + tasks: TASK, + signal: new AbortController().signal, + }); + await vi.advanceTimersByTimeAsync(46 * MIN); + await run; + expect(health.isSuspect("second-llama")).toBe(true); + expect(health.isSuspect("local-llama")).toBe(false); + expect(health.cause("second-llama")).toMatch(/task t0 was timed out/); + }); }); diff --git a/src/tools/fusion/local-leg-health.ts b/src/tools/fusion/local-leg-health.ts index b07b4fa0c..1167757dc 100644 --- a/src/tools/fusion/local-leg-health.ts +++ b/src/tools/fusion/local-leg-health.ts @@ -31,36 +31,49 @@ export const LOCAL_LEG_UNRESPONSIVE_ERROR = export const LOCAL_LEG_UNRESPONSIVE_HINT = "the local server is not answering /health since a worker on it was cancelled or timed out mid-generation; restart the local server (Manage → Local models, or restart the app) before delegating again, or do this part yourself"; +/** The key a mark is filed under when the caller names no server. */ +export const DEFAULT_LOCAL_LEG = "local"; + +/** + * Kept per local server (ATO-1318). With `mixLocalModels` two local + * servers can serve one fan-out; one shared mark let a wedge on one + * fail a worker sent to the other, or a healthy answer from one clear + * the other's mark. Each leg is filed under its provider id, and only + * a leg with a probe of its own is ever asked. + */ export class LocalLegHealth { - private suspect = false; - private suspectSince: string | null = null; + private readonly suspects = new Map(); - /** A worker on the local leg ended by an abort; check before the next dispatch. */ - noteCut(reason: string): void { - if (!this.suspect) this.suspectSince = reason; - this.suspect = true; + /** A worker on this local leg ended by an abort; check before the next dispatch there. */ + noteCut(reason: string, leg: string = DEFAULT_LOCAL_LEG): void { + if (!this.suspects.has(leg)) this.suspects.set(leg, reason); } - isSuspect(): boolean { - return this.suspect; + /** Whether `leg` is marked; with no leg named, whether any is. */ + isSuspect(leg?: string): boolean { + return leg === undefined ? this.suspects.size > 0 : this.suspects.has(leg); } - /** Why the leg was marked, for the row's note. */ - cause(): string | null { - return this.suspectSince; + /** Why the leg was marked, for the row's note; with no leg named, the first mark. */ + cause(leg?: string): string | null { + if (leg !== undefined) return this.suspects.get(leg) ?? null; + const first = this.suspects.values().next(); + return first.done === true ? null : first.value; } /** - * Ask the server before dispatching. Resolves `true` when the worker may - * be sent: nothing was cut, there is no probe to ask with, or the server - * answered (which clears the mark). Never throws. + * Ask the leg's server before dispatching. Resolves `true` when the + * worker may be sent: nothing on this leg was cut, there is no probe + * for this leg to ask with, or the server answered (which clears this + * leg's mark only). Never throws. */ async readyForDispatch( probe: (() => Promise) | undefined, sleep: (ms: number) => Promise = defaultSleep, signal?: AbortSignal, + leg: string = DEFAULT_LOCAL_LEG, ): Promise { - if (!this.suspect || probe === undefined) return true; + if (!this.suspects.has(leg) || probe === undefined) return true; for (let attempt = 0; attempt < LOCAL_LEG_HEALTH_ATTEMPTS; attempt += 1) { if (signal?.aborted) return true; if (attempt > 0) await sleep(LOCAL_LEG_HEALTH_RETRY_MS); @@ -71,8 +84,7 @@ export class LocalLegHealth { answer = "unanswered"; } if (answer === "answered") { - this.suspect = false; - this.suspectSince = null; + this.suspects.delete(leg); return true; } } diff --git a/src/tools/fusion/worker-runner.ts b/src/tools/fusion/worker-runner.ts index c0d72619f..3fbd489f6 100644 --- a/src/tools/fusion/worker-runner.ts +++ b/src/tools/fusion/worker-runner.ts @@ -483,6 +483,15 @@ export interface WorkerRunnerDeps { localLegHealth?: LocalLegHealth; /** `LlamaServerClient.probeHealth`, the question the check above asks. */ probeLocalHealth?: () => Promise; + /** + * The probe for one local leg's own server (ATO-1318), or `undefined` + * when nothing can ask that server; then the check is skipped for it. + * When set it replaces `probeLocalHealth`, which only ever asks the + * main local server. + */ + probeLocalHealthFor?: ( + providerId: string, + ) => (() => Promise) | undefined; /** * One completion on the escalation provider (ATO-942). Absent, nothing * is ever escalated, manual or automatic. @@ -719,12 +728,15 @@ async function runOneTask( options.localLeg === true && deps.localLegHealth !== undefined && !(await deps.localLegHealth.readyForDispatch( - deps.probeLocalHealth, + deps.probeLocalHealthFor !== undefined + ? deps.probeLocalHealthFor(options.providerId) + : deps.probeLocalHealth, undefined, options.signal, + options.providerId, )) ) { - const cause = deps.localLegHealth.cause(); + const cause = deps.localLegHealth.cause(options.providerId); const refused: WorkerTaskResult = { ...collector.finish({ id: task.id, @@ -1614,6 +1626,7 @@ async function runOneTask( ) { deps.localLegHealth?.noteCut( `task ${task.id} was ${timedOutOutcome ? "timed out" : queuedOutcome ? "never served" : stoppedByHandBack ? "handed back" : "cancelled"}`, + options.providerId, ); } From 5f1ad2edada1a3fc1e8c8fd4e840e439df61a2e1 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:17:05 +0300 Subject: [PATCH 037/155] Keep compacted turns out of the session row Every write of a session (each step's checkpoint, every turn's end) serialized the whole transcript into one JSON string, even the part a compaction had already summarized. On long sessions and overnight runs that string grows without bound, and so does the peak memory of each write. Turns covered by a compaction checkpoint now move once into a new session_turn_archive table, one row per turn. The row's payload keeps only the turns after them, plus an archivedTurns marker with the count and the first user message (for the session list). SessionStore.load, listRecent and listByWorkingDir put the archived turns back in front, so the in-memory transcript and every absolute index into it (coveredThrough, macroTurnStarts, pack start) are unchanged. - Archive rows are append-only and written in the same transaction as the payload. A write whose transcript no longer starts with the archived turns (an older copy, a replaced history) drops the archive and stores the transcript in full, so a load always returns what was saved. - Rows written before this change load as before and are migrated on their next write. Sessions without a checkpoint are stored exactly as before. - listChatLines and the summary page read across the archive. - delete, the retention pass and the desktop importer handle the archive. If the table cannot be created (read-only file, lock held), the store keeps storing transcripts in full. Refs ATO-1271 --- .../atomic-desktop/atomic-desktop-importer.ts | 16 +- src/runtime/docs/compaction.md | 5 + src/session/session-retention.ts | 7 +- src/session/session-store.ts | 246 ++++++++++++++---- .../session-store.turn-archive.test.ts | 230 ++++++++++++++++ src/session/session-summary-page.ts | 27 +- src/session/turn-archive.ts | 242 +++++++++++++++++ 7 files changed, 713 insertions(+), 60 deletions(-) create mode 100644 src/session/session-store.turn-archive.test.ts create mode 100644 src/session/turn-archive.ts diff --git a/src/import/atomic-desktop/atomic-desktop-importer.ts b/src/import/atomic-desktop/atomic-desktop-importer.ts index 1c963ca0a..6e298a3a0 100644 --- a/src/import/atomic-desktop/atomic-desktop-importer.ts +++ b/src/import/atomic-desktop/atomic-desktop-importer.ts @@ -20,6 +20,11 @@ import type { SessionState, SessionStatus } from "../../session/session-state.js import { TaskStore } from "../../tasks/index.js"; import { splitMemoryNote } from "../split-note.js"; import { snapshotSqlite } from "./sqlite-snapshot.js"; +import { + TURN_ARCHIVE_TABLE, + prepareTurnArchiveStatements, + restoreArchivedTurns, +} from "../../session/turn-archive.js"; /** * The databases of an Atomic Agent state dir this importer reads. The @@ -227,6 +232,11 @@ function importSessions( batch = []; }; let unreadable = 0; + // A source written by a binary that archives compacted turns + // (ATO-1271) keeps them beside the payload: read them back in. + const archive = hasTable(db, TURN_ARCHIVE_TABLE) + ? prepareTurnArchiveStatements(db) + : null; const rows = db .prepare( `SELECT id, working_dir, status, payload, created_at, updated_at @@ -236,7 +246,11 @@ function importSessions( for (const row of rows) { let state: SessionState; try { - state = normalizeSessionState(JSON.parse(row.payload)); + const raw: unknown = JSON.parse(row.payload); + if (typeof raw === "object" && raw !== null && !Array.isArray(raw)) { + restoreArchivedTurns(archive, row.id, raw as Record); + } + state = normalizeSessionState(raw); } catch { unreadable += 1; counts.skipped += 1; diff --git a/src/runtime/docs/compaction.md b/src/runtime/docs/compaction.md index 908cb7e36..b4e1fbd0f 100644 --- a/src/runtime/docs/compaction.md +++ b/src/runtime/docs/compaction.md @@ -46,6 +46,11 @@ checkpoint has `model: null` and the optional `kind: "extractive"`; one made fro rolling summary has `kind: "rolling"`. Loading rejects unsupported versions, invalid boundaries and cuts through tool/result pairs; `compactionWarning` carries the diagnostic. Transcript records are never rewritten. +In storage, the records a checkpoint covers move once to the +`session_turn_archive` table and the row's payload keeps only the rest, with an +`archivedTurns` count (`src/session/turn-archive.ts`). `SessionStore.load` +puts them back, so the in-memory transcript and every index into it are +unchanged; only the per-write serialization stops growing with the session. The optional `SessionState.rollingSummary` (see Rolling summary) is never part of the prompt; loading drops it silently when stale. diff --git a/src/session/session-retention.ts b/src/session/session-retention.ts index 21c7f4cc8..cd3175caa 100644 --- a/src/session/session-retention.ts +++ b/src/session/session-retention.ts @@ -4,6 +4,7 @@ import { resolve, sep } from "node:path"; import type Database from "better-sqlite3"; import { traceFilePath } from "../tracing/trace/trace-sink.js"; import type { SessionStatus } from "./session-state.js"; +import { deleteArchivedTurns } from "./turn-archive.js"; /** * One bounded retention pass over the `sessions` table, run at startup. @@ -236,7 +237,11 @@ function removeIds( result: PruneSessionsResult, ): void { const placeholders = ids.map(() => "?").join(", "); - db.prepare(`DELETE FROM sessions WHERE id IN (${placeholders})`).run(...ids); + db.transaction(() => { + db.prepare(`DELETE FROM sessions WHERE id IN (${placeholders})`).run(...ids); + // Their compacted turns (`turn-archive.ts`) go with them. + deleteArchivedTurns(db, ids); + })(); result.deleted += ids.length; if (tracesDir === null) return; for (const id of ids) { diff --git a/src/session/session-store.ts b/src/session/session-store.ts index 8f30e6101..0b53cf549 100644 --- a/src/session/session-store.ts +++ b/src/session/session-store.ts @@ -44,6 +44,16 @@ import { type TurnProgress, type TurnProgressStatements, } from "./turn-progress.js"; +import { + ARCHIVED_TURNS_KEY, + TURN_ARCHIVE_TABLE, + archivedTurnsMarker, + ensureTurnArchiveTable, + prepareTurnArchiveStatements, + restoreArchivedTurns, + syncTurnArchive, + type TurnArchiveStatements, +} from "./turn-archive.js"; // `turn_owner` names the process running a turn on the row (see // `beginTurn`) and is null otherwise — a shutdown's stand-in keeps it @@ -86,13 +96,53 @@ SELECT json_extract(t.value, '$.kind') AS kind, AND json_type(s.payload, '$.turns') = 'array' THEN s.payload ELSE '{}' END, '$.turns') AS t - WHERE s.id = ? + WHERE s.id = @id AND t.type = 'object' AND json_extract(t.value, '$.kind') IN ('user', 'assistant_reply') AND json_type(t.value, '$.text') = 'text' AND json_extract(t.value, '$.progressNote') IS NOT 1 ORDER BY t.key ASC`; +/** + * {@link CHAT_LINES_SQL} with the turns kept in the archive + * (`turn-archive.ts`) read first: only the rows the payload's marker + * counts, then the payload's own `turns`. Each arm keeps its guards in + * its own `WHERE`, in the order the single-table query has them. An + * archived row is always one whole JSON document, so `json_extract` on + * it cannot raise; the `CASE` is there for a row written by hand. + */ +const CHAT_LINES_WITH_ARCHIVE_SQL = ` +SELECT kind, text FROM ( + SELECT 0 AS part, a.idx AS ord, + json_extract(CASE WHEN json_valid(a.turn) THEN a.turn ELSE '{}' END, '$.kind') AS kind, + json_extract(CASE WHEN json_valid(a.turn) THEN a.turn ELSE '{}' END, '$.text') AS text + FROM sessions AS s + JOIN ${TURN_ARCHIVE_TABLE} AS a ON a.session_id = s.id + WHERE s.id = @id + AND json_valid(s.payload) + AND a.idx < COALESCE(json_extract(s.payload, '$.${ARCHIVED_TURNS_KEY}.count'), 0) + AND json_valid(a.turn) + AND json_type(a.turn) = 'object' + AND json_extract(a.turn, '$.kind') IN ('user', 'assistant_reply') + AND json_type(a.turn, '$.text') = 'text' + AND json_extract(a.turn, '$.progressNote') IS NOT 1 + UNION ALL + SELECT 1 AS part, t.key AS ord, + json_extract(t.value, '$.kind') AS kind, + json_extract(t.value, '$.text') AS text + FROM sessions AS s, + json_each(CASE WHEN json_valid(s.payload) + AND json_type(s.payload, '$.turns') = 'array' + THEN s.payload ELSE '{}' END, + '$.turns') AS t + WHERE s.id = @id + AND t.type = 'object' + AND json_extract(t.value, '$.kind') IN ('user', 'assistant_reply') + AND json_type(t.value, '$.text') = 'text' + AND json_extract(t.value, '$.progressNote') IS NOT 1 +) + ORDER BY part ASC, ord ASC`; + /** One line of `listChatLines`. */ export interface SessionChatLine { kind: "user" | "assistant_reply"; @@ -211,12 +261,16 @@ interface TurnMarkStatements { recover: Database.Statement; } -/** A row as the readers select it: the status column and the payload. */ +/** A row as the readers select it: the id and status columns and the payload. */ interface StoredPayloadRow { + id?: unknown; status?: unknown; payload: string; } +/** Thrown inside a write transaction to roll it back; never escapes. */ +const ROLLBACK = Symbol("rollback"); + /** Narrow projection returned by `listRecentWorkingDirs`. */ export interface RecentWorkingDirRow { workingDir: string; @@ -224,9 +278,11 @@ export interface RecentWorkingDirRow { } /** - * Durable session persistence. We serialise the whole SessionState as JSON - * because it is small (<10 KB) and we optimise for developer iteration - * over schema stability. A columnar migration can come later. + * Durable session persistence. The SessionState is serialised as JSON, + * optimising for developer iteration over schema stability, except for the + * turns a context compaction covers: those move to `session_turn_archive` + * once, so a long session's every write does not re-serialise its whole + * history (`turn-archive.ts`, ATO-1271). */ export class SessionStore { private readonly db: Database.Database; @@ -260,6 +316,12 @@ export class SessionStore { * did before them. */ private readonly progress: TurnProgressStatements | null; + /** + * The archive of compacted turns (`turn-archive.ts`, ATO-1271), or + * `null` when the table could not be created: transcripts are then + * stored in full, as before. + */ + private readonly archive: TurnArchiveStatements | null; /** The database file's real path, as marks record it; `undefined` in memory. */ private readonly dbIdentity: string | undefined; /** @@ -293,6 +355,10 @@ export class SessionStore { // Checkpoints are guarded on the turn's mark, so they need both. const withProgress = withMarks && ensureTurnProgressColumn(this.db) === null; + this.archive = + ensureTurnArchiveTable(this.db) === null + ? prepareTurnArchiveStatements(this.db) + : null; this.turnOwnerProbe = options.turnOwnerProbe ?? currentTurnOwnerProbe(); this.dbIdentity = databaseIdentity(file); this.insertStmt = this.db.prepare( @@ -315,7 +381,7 @@ export class SessionStore { // Every reader takes `status` from the column, beside the payload: // see `readPayload`. this.selectStmt = this.db.prepare( - `SELECT status, payload FROM sessions WHERE id = ?`, + `SELECT id, status, payload FROM sessions WHERE id = ?`, ); // Projected in SQL rather than parsed in JS: `save` runs this on // every write, and a transcript payload is the one thing in this @@ -331,10 +397,10 @@ export class SessionStore { `SELECT status, ${turnOwnerColumn} AS turnOwner FROM sessions WHERE id = ?`, ); this.listByWorkingDirStmt = this.db.prepare( - `SELECT status, payload FROM sessions WHERE working_dir = ? ORDER BY updated_at DESC LIMIT ?`, + `SELECT id, status, payload FROM sessions WHERE working_dir = ? ORDER BY updated_at DESC LIMIT ?`, ); this.listRecentStmt = this.db.prepare( - `SELECT status, payload FROM sessions ORDER BY updated_at DESC LIMIT ?`, + `SELECT id, status, payload FROM sessions ORDER BY updated_at DESC LIMIT ?`, ); this.listRecentDirsStmt = this.db.prepare( `SELECT working_dir AS workingDir, updated_at AS updatedAt @@ -347,7 +413,9 @@ export class SessionStore { this.summaryFirstPageStmt = this.db.prepare(SUMMARY_FIRST_PAGE_SQL); this.summaryNextPageStmt = this.db.prepare(SUMMARY_NEXT_PAGE_SQL); this.countUnreadableStmt = this.db.prepare(COUNT_UNREADABLE_SQL); - this.chatLinesStmt = this.db.prepare(CHAT_LINES_SQL); + this.chatLinesStmt = this.db.prepare( + this.archive === null ? CHAT_LINES_SQL : CHAT_LINES_WITH_ARCHIVE_SQL, + ); this.deleteStmt = this.db.prepare(`DELETE FROM sessions WHERE id = ?`); this.finishTurnStmt = withMarks ? this.db.prepare( @@ -506,17 +574,25 @@ export class SessionStore { // namer's title written since is only in the row, and replacing the // payload without it would lose it for good (`finishTurn` reads the // row's title to keep it). - const row = this.serialize( - withStoredTitle(state, this.storedRow(state.id)), - ); - const result = this.progress.checkpoint.run({ - id: state.id, - owner, - payload: row.payload, - updated_at: row.updated_at, - progress: serializeTurnProgress(progress), - }) as { changes: number }; - return result.changes > 0; + const progressStatements = this.progress; + // One transaction with the archive rows the write adds, rolled back + // when the guard finds the row no longer this turn's: archive rows + // written for a payload that was never stored would not match the + // payload that is. + return this.inWriteTransaction(() => { + const row = this.serialize( + withStoredTitle(state, this.storedRow(state.id)), + ); + const result = progressStatements.checkpoint.run({ + id: state.id, + owner, + payload: row.payload, + updated_at: row.updated_at, + progress: serializeTurnProgress(progress), + }) as { changes: number }; + if (result.changes === 0) throw ROLLBACK; + return true; + }, false); } /** @@ -680,19 +756,38 @@ export class SessionStore { * only `save` is held to `statusForSave`. */ private write(state: SessionState, endsTurn: boolean): void { - const stored = this.storedRow(state.id); - const status = endsTurn - ? state.status - : statusForSave(state.status, stored); - let next: SessionState = - status === state.status ? state : { ...state, status }; - if (stored === undefined) { - this.insertStmt.run(this.serialize(next)); - return; + // One transaction: the archive rows the write adds and the payload + // that counts them land together or not at all. + this.inWriteTransaction(() => { + const stored = this.storedRow(state.id); + const status = endsTurn + ? state.status + : statusForSave(state.status, stored); + let next: SessionState = + status === state.status ? state : { ...state, status }; + if (stored === undefined) { + this.insertStmt.run(this.serialize(next)); + return; + } + next = withStoredTitle(next, stored); + const update = endsTurn ? this.finishTurnStmt : this.updateStmt; + update.run(this.serialize(next)); + }, undefined); + } + + /** + * Run `fn` in one `BEGIN IMMEDIATE` transaction, so the write lock is + * waited for (busy timeout) up front rather than failing on an upgrade + * from read to write. `fn` throws {@link ROLLBACK} to undo its writes + * and return `onRollback`; any other error rolls back and propagates. + */ + private inWriteTransaction(fn: () => T, onRollback: T): T { + try { + return this.db.transaction(fn).immediate(); + } catch (err) { + if (err === ROLLBACK) return onRollback; + throw err; } - next = withStoredTitle(next, stored); - const update = endsTurn ? this.finishTurnStmt : this.updateStmt; - update.run(this.serialize(next)); } /** @@ -756,8 +851,10 @@ export class SessionStore { (batch: readonly SessionState[]): string[] => { const inserted: string[] = []; for (const state of batch) { + // Stored in full: archiving here would write rows for a copy + // the `OR IGNORE` may then drop. The next write archives it. const result = this.insertIfAbsentStmt.run( - this.serialize(endLiveTurn(state)), + this.serialize(endLiveTurn(state), { archive: false }), ) as { changes: number }; if (result.changes > 0) inserted.push(state.id); } @@ -787,17 +884,23 @@ export class SessionStore { } load(id: string): SessionState | null { - const row = this.selectStmt.get(id) as StoredPayloadRow | undefined; - if (!row) return null; - return this.readPayload(row); + // One read transaction, so the payload and the archived turns it + // counts come from the same snapshot of the file. + return this.db.transaction((): SessionState | null => { + const row = this.selectStmt.get(id) as StoredPayloadRow | undefined; + if (!row) return null; + return this.readPayload(row); + })(); } listByWorkingDir(workingDir: string, limit = 25): SessionState[] { - const rows = this.listByWorkingDirStmt.all( - workingDir, - limit, - ) as StoredPayloadRow[]; - return this.readPayloads(rows); + return this.db.transaction((): SessionState[] => { + const rows = this.listByWorkingDirStmt.all( + workingDir, + limit, + ) as StoredPayloadRow[]; + return this.readPayloads(rows); + })(); } /** @@ -806,8 +909,10 @@ export class SessionStore { * ongoing threads from any project root. */ listRecent(limit = 25): SessionState[] { - const rows = this.listRecentStmt.all(limit) as StoredPayloadRow[]; - return this.readPayloads(rows); + return this.db.transaction((): SessionState[] => { + const rows = this.listRecentStmt.all(limit) as StoredPayloadRow[]; + return this.readPayloads(rows); + })(); } /** @@ -844,7 +949,7 @@ export class SessionStore { * an unknown id or a payload that will not parse. */ listChatLines(id: string): SessionChatLine[] { - return this.chatLinesStmt.all(id) as SessionChatLine[]; + return this.chatLinesStmt.all({ id }) as SessionChatLine[]; } /** Rows whose payload is not valid JSON — the ones every reader skips. */ @@ -870,7 +975,10 @@ export class SessionStore { } delete(id: string): void { - this.deleteStmt.run(id); + this.db.transaction(() => { + this.deleteStmt.run(id); + this.archive?.clear.run(id); + })(); } /** @@ -903,7 +1011,17 @@ export class SessionStore { private readPayload(row: StoredPayloadRow): SessionState | null { let state: SessionState; try { - state = normalizeSessionState(JSON.parse(row.payload)); + const raw: unknown = JSON.parse(row.payload); + if (typeof raw === "object" && raw !== null && !Array.isArray(raw)) { + // The archived prefix goes back in front before the checkpoint + // is checked against the transcript (ATO-1271). + const record = raw as Record; + const id = typeof row.id === "string" ? row.id : record.id; + if (typeof id === "string") { + restoreArchivedTurns(this.archive, id, record); + } + } + state = normalizeSessionState(raw); } catch { this.unreadableSkips += 1; return null; @@ -922,13 +1040,43 @@ export class SessionStore { return states; } - private serialize(state: SessionState): Record { - const persistable = stripEphemeral(state); + /** + * The row for `state`. With the archive on, the turns a compaction + * covers are written to it (once each) and left out of the payload, + * which then carries only the rest and a marker counting them + * (`turn-archive.ts`, ATO-1271). Writes archive rows, so callers run it + * inside the transaction that writes the row. + */ + private serialize( + state: SessionState, + options: { archive?: boolean } = {}, + ): Record { + const persistable = stripEphemeral(state) as SessionState & { + [ARCHIVED_TURNS_KEY]?: unknown; + }; + const archived = + this.archive !== null && options.archive !== false + ? syncTurnArchive(this.archive, persistable) + : 0; + // A marker never lives on a loaded state; drop one a copy may carry + // from elsewhere (an imported payload) so it cannot miscount. + const { [ARCHIVED_TURNS_KEY]: _marker, ...rest } = persistable; + const payload = + archived === 0 + ? JSON.stringify(rest) + : JSON.stringify({ + ...rest, + turns: persistable.turns.slice(archived), + [ARCHIVED_TURNS_KEY]: archivedTurnsMarker( + persistable.turns, + archived, + ), + }); return { id: persistable.id, working_dir: persistable.workingDir, status: persistable.status, - payload: JSON.stringify(persistable), + payload, created_at: persistable.createdAt, updated_at: persistable.updatedAt, }; diff --git a/src/session/session-store.turn-archive.test.ts b/src/session/session-store.turn-archive.test.ts new file mode 100644 index 000000000..5e8113868 --- /dev/null +++ b/src/session/session-store.turn-archive.test.ts @@ -0,0 +1,230 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Database as DatabaseCtor } from "../native/load-better-sqlite3.js"; +import { + assistantReplyTurn, + userTurn, + type ConversationTurn, +} from "./conversation-turn.js"; +import { compactionBoundaryHash, type SessionCompaction } from "./session-compaction.js"; +import { pruneSessions } from "./session-retention.js"; +import { createEmptySessionState, recordTurn, type SessionState } from "./session-state.js"; +import { SessionStore } from "./session-store.js"; +import { ARCHIVED_TURNS_KEY, TURN_ARCHIVE_TABLE } from "./turn-archive.js"; + +/** + * ATO-1271: turns a compaction covers move out of the session row into + * `session_turn_archive`, so a write no longer stringifies the whole + * transcript. Loads still return the full transcript with its original + * indices. + */ + +function conversation(pairs: number, at = 1): ConversationTurn[] { + const turns: ConversationTurn[] = []; + for (let i = 0; i < pairs; i++) { + turns.push(userTurn(`question ${i}`, at + i * 2)); + turns.push(assistantReplyTurn(`answer ${i}`, at + i * 2 + 1)); + } + return turns; +} + +function compactedAt(turns: readonly ConversationTurn[], through: number): SessionCompaction { + return { + version: 1, + summary: "Earlier: questions and answers.", + coveredThrough: through, + boundaryHash: compactionBoundaryHash(turns[through - 1]!), + createdAt: 1, + reason: "threshold", + model: null, + kind: "extractive", + tokensBefore: 100, + tokensAfter: 10, + }; +} + +function session(id: string, turns: ConversationTurn[], through?: number): SessionState { + return { + ...createEmptySessionState({ id, workingDir: "/w" }), + turns, + turnCount: turns.length / 2, + status: "completed", + updatedAt: 5_000, + ...(through === undefined ? {} : { compaction: compactedAt(turns, through) }), + }; +} + +describe("SessionStore turn archive", () => { + let tmp: string; + let file: string; + let store: SessionStore; + + beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "atomic-agent-turn-archive-")); + file = join(tmp, "sessions.sqlite"); + store = new SessionStore({ dbFile: file }); + }); + + afterEach(() => { + store.close(); + rmSync(tmp, { recursive: true, force: true }); + }); + + function rawPayload(id: string): Record { + const row = store + .getDatabaseHandleForRetention() + .prepare(`SELECT payload FROM sessions WHERE id = ?`) + .get(id) as { payload: string }; + return JSON.parse(row.payload) as Record; + } + + function archivedRows(id: string): number[] { + return ( + store + .getDatabaseHandleForRetention() + .prepare(`SELECT idx FROM ${TURN_ARCHIVE_TABLE} WHERE session_id = ? ORDER BY idx`) + .all(id) as { idx: number }[] + ).map((row) => row.idx); + } + + it("keeps a session without a checkpoint in the row, exactly as before", () => { + const state = session("plain", conversation(3)); + store.save(state); + const payload = rawPayload("plain"); + expect(payload.turns).toEqual(state.turns); + expect(payload).not.toHaveProperty(ARCHIVED_TURNS_KEY); + expect(archivedRows("plain")).toEqual([]); + }); + + it("moves the covered turns out of the payload and loads the full transcript back", () => { + const turns = conversation(5); + const state = session("s1", turns, 6); + store.save(state); + + const payload = rawPayload("s1"); + expect(payload.turns).toEqual(turns.slice(6)); + expect(payload[ARCHIVED_TURNS_KEY]).toEqual({ count: 6, firstUserText: "question 0" }); + expect(archivedRows("s1")).toEqual([0, 1, 2, 3, 4, 5]); + + const loaded = store.load("s1"); + expect(loaded?.turns).toEqual(turns); + expect(loaded).not.toHaveProperty(ARCHIVED_TURNS_KEY); + // The checkpoint still matches the transcript, so it survives the load. + expect(loaded?.compaction?.coveredThrough).toBe(6); + expect(loaded?.compactionWarning).toBeUndefined(); + expect(store.listRecent(5).find((s) => s.id === "s1")?.turns).toEqual(turns); + expect(store.listByWorkingDir("/w", 5).find((s) => s.id === "s1")?.turns).toEqual(turns); + }); + + it("writes each archived turn once and only adds the newly covered ones", () => { + const turns = conversation(4); + store.save(session("s2", turns, 4)); + const db = store.getDatabaseHandleForRetention(); + db.exec( + `CREATE TEMP TABLE archive_writes (idx INTEGER); + CREATE TEMP TRIGGER count_archive_writes AFTER INSERT ON ${TURN_ARCHIVE_TABLE} + BEGIN INSERT INTO archive_writes (idx) VALUES (NEW.idx); END;`, + ); + + // More turns, same checkpoint: nothing new is archived. + let next = recordTurn(session("s2", turns, 4), userTurn("one more", 100)); + store.save(next); + expect(db.prepare(`SELECT idx FROM archive_writes`).all()).toEqual([]); + + // The checkpoint moves on: only the turns between the two are written. + next = recordTurn(next, assistantReplyTurn("done", 101)); + next = { ...next, compaction: compactedAt(next.turns, 8) }; + store.save(next); + expect(db.prepare(`SELECT idx FROM archive_writes ORDER BY idx`).all()).toEqual([ + { idx: 4 }, + { idx: 5 }, + { idx: 6 }, + { idx: 7 }, + ]); + expect(store.load("s2")?.turns).toEqual(next.turns); + }); + + it("keeps the archive when a later copy drops the checkpoint but keeps the history", () => { + const turns = conversation(4); + store.save(session("s3", turns, 4)); + const { compaction: _dropped, ...withoutCheckpoint } = session("s3", turns, 4); + store.save(withoutCheckpoint); + expect(rawPayload("s3")[ARCHIVED_TURNS_KEY]).toEqual({ count: 4, firstUserText: "question 0" }); + expect(store.load("s3")?.turns).toEqual(turns); + }); + + it("stores an older, shorter copy in full and drops the archive it no longer matches", () => { + const turns = conversation(5); + store.save(session("s4", turns, 8)); + const older = session("s4", turns.slice(0, 4)); + store.save(older); + expect(archivedRows("s4")).toEqual([]); + expect(rawPayload("s4")).not.toHaveProperty(ARCHIVED_TURNS_KEY); + expect(store.load("s4")?.turns).toEqual(older.turns); + }); + + it("stores a rewritten history in full when its archived prefix no longer matches", () => { + const turns = conversation(4); + store.save(session("s5", turns, 4)); + const rewritten = conversation(4, 1_000); + store.save(session("s5", rewritten)); + expect(archivedRows("s5")).toEqual([]); + expect(store.load("s5")?.turns).toEqual(rewritten); + }); + + it("still loads a row written before the archive existed", () => { + const turns = conversation(3); + const old = session("legacy", turns, 2); + const raw = new DatabaseCtor(file); + try { + raw + .prepare( + `INSERT INTO sessions (id, working_dir, status, payload, created_at, updated_at) + VALUES (?, '/w', 'completed', ?, 1, 5000)`, + ) + .run("legacy", JSON.stringify(old)); + } finally { + raw.close(); + } + const loaded = store.load("legacy"); + expect(loaded?.turns).toEqual(turns); + expect(loaded?.compaction?.coveredThrough).toBe(2); + // Migrated on its next write. + store.save(loaded!); + expect(archivedRows("legacy")).toEqual([0, 1]); + expect(store.load("legacy")?.turns).toEqual(turns); + }); + + it("reads chat lines and the list row's opening prompt across the archive", () => { + const turns = conversation(3); + store.save(session("s6", turns, 4)); + expect(store.listChatLines("s6")).toEqual( + turns.map((turn) => ({ kind: turn.kind, text: (turn as { text: string }).text })), + ); + const row = store.listSummaryPage({ limit: 10 }).find((s) => s.id === "s6"); + expect(row?.firstPrompt).toBe("question 0"); + }); + + it("removes the archived turns with the session", () => { + store.save(session("gone", conversation(3), 4)); + expect(archivedRows("gone")).toHaveLength(4); + store.delete("gone"); + expect(archivedRows("gone")).toEqual([]); + }); + + it("removes the archived turns of sessions the retention pass deletes", () => { + store.save({ ...session("old", conversation(3), 4), updatedAt: 1 }); + expect(archivedRows("old")).toHaveLength(4); + pruneSessions({ + db: store.getDatabaseHandleForRetention(), + maxAgeDays: 1, + maxRows: null, + now: 10 * 24 * 60 * 60 * 1000, + }); + expect(store.load("old")).toBeNull(); + expect(archivedRows("old")).toEqual([]); + }); +}); diff --git a/src/session/session-summary-page.ts b/src/session/session-summary-page.ts index 61dd33e5d..de4b47abe 100644 --- a/src/session/session-summary-page.ts +++ b/src/session/session-summary-page.ts @@ -1,4 +1,5 @@ import type { SessionSummary } from "./session-summary.js"; +import { ARCHIVED_TURNS_KEY } from "./turn-archive.js"; /** * One page of list rows, newest first, with the fields a list renders @@ -27,6 +28,11 @@ import type { SessionSummary } from "./session-summary.js"; * index rather than sorting the table, and reads the cursor as a range * seek on it (`SEARCH sessions USING INDEX idx_sessions_updated_id`) — * page 40 costs what page 1 costs. + * + * A session whose compacted turns moved to the archive (`turn-archive.ts`) + * may have its opening prompt there rather than in `$.turns`; the + * payload's marker carries that prompt, and counts as "someone has spoken + * to it" for the `EXISTS` guard. */ function summaryPageSql(cursored: boolean): string { return ` @@ -37,19 +43,22 @@ SELECT id, updated_at AS updatedAt, json_extract(payload, '$.turnCount') AS turnCount, json_extract(payload, '$.stepCount') AS stepCount, - (SELECT json_extract(t.value, '$.text') - FROM json_each(payload, '$.turns') AS t - WHERE t.type = 'object' - AND json_extract(t.value, '$.kind') = 'user' - LIMIT 1) AS firstPrompt, + COALESCE( + json_extract(payload, '$.${ARCHIVED_TURNS_KEY}.firstUserText'), + (SELECT json_extract(t.value, '$.text') + FROM json_each(payload, '$.turns') AS t + WHERE t.type = 'object' + AND json_extract(t.value, '$.kind') = 'user' + LIMIT 1)) AS firstPrompt, json_extract(payload, '$.metadata.title') AS title, json_extract(payload, '$.metadata.importedFrom') AS importedFrom FROM sessions WHERE json_valid(payload) AND json_type(payload, '$.turns') = 'array' - AND EXISTS (SELECT 1 - FROM json_each(payload, '$.turns') AS t - WHERE t.type = 'object' - AND json_extract(t.value, '$.kind') = 'user')${ + AND (json_type(payload, '$.${ARCHIVED_TURNS_KEY}.firstUserText') = 'text' + OR EXISTS (SELECT 1 + FROM json_each(payload, '$.turns') AS t + WHERE t.type = 'object' + AND json_extract(t.value, '$.kind') = 'user'))${ cursored ? ` AND (updated_at < :afterUpdatedAt diff --git a/src/session/turn-archive.ts b/src/session/turn-archive.ts new file mode 100644 index 000000000..c34de7fbc --- /dev/null +++ b/src/session/turn-archive.ts @@ -0,0 +1,242 @@ +import type Database from "better-sqlite3"; +import { createHash } from "node:crypto"; +import type { ConversationTurn } from "./conversation-turn.js"; +import type { SessionState } from "./session-state.js"; + +/** + * Turns a context compaction already covers, kept out of the session row + * (ATO-1271). + * + * Before this the row's payload carried the whole transcript, and every + * write (each step's checkpoint, every turn's end) turned all of it into + * one string with `JSON.stringify`, however much of it a compaction had + * long since summarised. A session that runs for hours grows that string + * without bound, and the peak memory of a write grows with it. + * + * Now the covered prefix lives here, one row per turn, written once when a + * compaction first covers it and never again. The payload carries only + * the turns after it, plus {@link ArchivedTurnsMarker} saying how many + * rows go back in front. Readers of the store see no difference: + * `SessionStore.load` hands back the whole transcript with the same + * indices as before, so `coveredThrough`, `macroTurnStarts` and every + * other absolute index keep their meaning. + * + * Archived turns are append-only. A write whose transcript no longer + * starts with the archived rows (an older copy saved over a newer one, + * history replaced by an import) drops the archive and stores the + * transcript in full, so what a write saves is always what a load + * returns. + */ + +export const TURN_ARCHIVE_TABLE = "session_turn_archive"; + +const TURN_ARCHIVE_SCHEMA = ` +CREATE TABLE IF NOT EXISTS ${TURN_ARCHIVE_TABLE} ( + session_id TEXT NOT NULL, + idx INTEGER NOT NULL, + hash TEXT NOT NULL, + turn TEXT NOT NULL, + PRIMARY KEY (session_id, idx) +); +`; + +/** Payload key that says how many archived turns go in front of `turns`. */ +export const ARCHIVED_TURNS_KEY = "archivedTurns"; + +/** + * Stored in the payload, never in a live `SessionState`. `firstUserText` + * is the first user message of the archived prefix, for list rows that + * read the opening prompt in SQL (`session-summary-page.ts`). + */ +export interface ArchivedTurnsMarker { + count: number; + firstUserText?: string; +} + +export interface TurnArchiveStatements { + last: Database.Statement; + first: Database.Statement; + insert: Database.Statement; + clear: Database.Statement; + read: Database.Statement; +} + +/** + * Create the archive table. Like the column helpers beside the store, a + * failure (a file this process may only read, a write lock held past the + * busy timeout) does not stop the store opening: it runs without an + * archive and stores transcripts in full, as before. Returns why, or + * `null`. + */ +export function ensureTurnArchiveTable(db: Database.Database): string | null { + try { + db.exec(TURN_ARCHIVE_SCHEMA); + return null; + } catch (err) { + return err instanceof Error ? err.message : String(err); + } +} + +export function prepareTurnArchiveStatements( + db: Database.Database, +): TurnArchiveStatements { + return { + last: db.prepare( + `SELECT idx, hash FROM ${TURN_ARCHIVE_TABLE} + WHERE session_id = ? ORDER BY idx DESC LIMIT 1`, + ), + first: db.prepare( + `SELECT hash FROM ${TURN_ARCHIVE_TABLE} WHERE session_id = ? AND idx = 0`, + ), + insert: db.prepare( + `INSERT OR REPLACE INTO ${TURN_ARCHIVE_TABLE} (session_id, idx, hash, turn) + VALUES (@session_id, @idx, @hash, @turn)`, + ), + clear: db.prepare(`DELETE FROM ${TURN_ARCHIVE_TABLE} WHERE session_id = ?`), + read: db.prepare( + `SELECT turn FROM ${TURN_ARCHIVE_TABLE} + WHERE session_id = ? AND idx < ? ORDER BY idx ASC`, + ), + }; +} + +function hashText(text: string): string { + return createHash("sha256").update(text).digest("hex"); +} + +/** The digest the stored `hash` column holds for this turn. */ +export function archivedTurnHash(turn: ConversationTurn): string { + return hashText(JSON.stringify(turn)); +} + +/** + * How many leading turns the compaction checkpoint covers, or 0. Not the + * full `validSessionCompaction` check: the archive is storage and any + * prefix is a correct split, so only the bounds matter here. + */ +export function coveredPrefixLength(state: SessionState): number { + const covered = state.compaction?.coveredThrough; + if (typeof covered !== "number" || !Number.isInteger(covered)) return 0; + if (covered <= 0) return 0; + return Math.min(covered, state.turns.length); +} + +/** + * Bring the archive in step with `state` and return how many leading + * turns it now holds: the ones the payload leaves out. Must run inside + * the same transaction as the payload write. + */ +export function syncTurnArchive( + statements: TurnArchiveStatements, + state: SessionState, +): number { + const turns = state.turns; + let stored = 0; + const last = statements.last.get(state.id) as + | { idx: number; hash: string } + | undefined; + if (last !== undefined) { + stored = last.idx + 1; + const first = statements.first.get(state.id) as + | { hash: string } + | undefined; + const consistent = + stored <= turns.length && + first !== undefined && + archivedTurnHash(turns[stored - 1]!) === last.hash && + archivedTurnHash(turns[0]!) === first.hash; + if (!consistent) { + statements.clear.run(state.id); + stored = 0; + } + } + const keep = Math.max(stored, coveredPrefixLength(state)); + for (let idx = stored; idx < keep; idx++) { + const text = JSON.stringify(turns[idx]); + statements.insert.run({ + session_id: state.id, + idx, + hash: hashText(text), + turn: text, + }); + } + return keep; +} + +/** The marker for an archive of the first `count` turns of `turns`. */ +export function archivedTurnsMarker( + turns: readonly ConversationTurn[], + count: number, +): ArchivedTurnsMarker { + for (let i = 0; i < count; i++) { + const turn = turns[i]; + if (turn?.kind === "user" && typeof turn.text === "string") { + return { count, firstUserText: turn.text }; + } + } + return { count }; +} + +/** The marker a parsed payload carries, or `null` when it has none. */ +export function readArchivedTurnsMarker( + raw: Record, +): ArchivedTurnsMarker | null { + const marker = raw[ARCHIVED_TURNS_KEY]; + if (typeof marker !== "object" || marker === null) return null; + const count = (marker as { count?: unknown }).count; + if (typeof count !== "number" || !Number.isInteger(count) || count <= 0) { + return null; + } + return { count }; +} + +/** + * Put the archived prefix back in front of a parsed payload's `turns` + * and drop the marker, in place. A payload without a marker keeps its + * turns as they are. A row that will not parse is skipped: the + * transcript then no longer matches its checkpoint, which + * `normalizeSessionState` notices and drops, rather than the whole + * session being lost. + */ +export function restoreArchivedTurns( + statements: TurnArchiveStatements | null, + id: string, + raw: Record, +): void { + const marker = readArchivedTurnsMarker(raw); + if (ARCHIVED_TURNS_KEY in raw) delete raw[ARCHIVED_TURNS_KEY]; + if (marker === null || statements === null) return; + const tail = Array.isArray(raw.turns) ? (raw.turns as unknown[]) : []; + const turns: unknown[] = []; + const rows = statements.read.iterate(id, marker.count) as IterableIterator<{ + turn: string; + }>; + for (const row of rows) { + try { + turns.push(JSON.parse(row.turn)); + } catch { + // See above: skipped, the checkpoint check catches the gap. + } + } + for (const turn of tail) turns.push(turn); + raw.turns = turns; +} + +/** + * Remove the archived turns of deleted sessions. Tolerates a database + * without the table (one only an older binary has opened). + */ +export function deleteArchivedTurns( + db: Database.Database, + ids: readonly string[], +): void { + if (ids.length === 0) return; + const exists = db + .prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`) + .get(TURN_ARCHIVE_TABLE); + if (exists === undefined) return; + const placeholders = ids.map(() => "?").join(", "); + db.prepare( + `DELETE FROM ${TURN_ARCHIVE_TABLE} WHERE session_id IN (${placeholders})`, + ).run(...ids); +} From 942aac879e68b694927d350e20c1cba27583c31b Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:17:57 +0300 Subject: [PATCH 038/155] Bound compaction reasoning on Gemini and AI/ML API thinking models The compaction summary call asks for low reasoning effort, but the field was only sent to OpenRouter and OpenAI-compatible providers. Gemini and AI/ML API got nothing, so a thinking model there could reason past the 4096-token headroom and the summary fell back to the crude extractive one. reasoning_effort is now sent on Gemini for its thinking models (2.5 and 3.x, not image, speech, live or embedding variants) and on AI/ML API for OpenAI reasoning models (o1, o3, o4, gpt-5). Other models on those providers keep the body as before, since the field is not documented for them. ATO-1359 --- src/llm/provider/openai/model-params.ts | 36 +++++++++++++++++++ .../provider/openai/openai-build-body.test.ts | 30 ++++++++++++++++ src/llm/provider/openai/openai-build-body.ts | 6 +++- 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/llm/provider/openai/model-params.ts b/src/llm/provider/openai/model-params.ts index 41414478b..e987b5ca0 100644 --- a/src/llm/provider/openai/model-params.ts +++ b/src/llm/provider/openai/model-params.ts @@ -123,15 +123,41 @@ export function forgetLearnedModelParams(): void { learnedTemperatures.clear(); } +/** + * Gemini models that think and take `reasoning_effort` on Gemini's + * OpenAI-compatible endpoint: 2.5 and 3.x (`gemini-2.5-flash`, + * `gemini-2.5-pro`, `gemini-3-pro-preview`, with or without the + * `models/` prefix). Not 2.0 and older, which do not think, and not the + * image, speech, live or embedding variants, which are not chat + * reasoners. + */ +const GEMINI_THINKING_MODEL_RE = /^gemini-(?:2\.5|[3-9])(?![0-9])/i; +const GEMINI_NON_CHAT_VARIANT_RE = /(?:image|tts|audio|live|embedding)/i; + +function bareModelId(modelId: string): string { + return modelId.includes("/") + ? modelId.slice(modelId.lastIndexOf("/") + 1) + : modelId; +} + /** * How `reasoningEffort` is spelled for a provider kind. OpenRouter takes a * `reasoning` object; OpenAI-compatible services take the flat * `reasoning_effort` OpenAI documents; the rest document neither, and a * field a service does not know is at best ignored and at worst a 400. + * + * Gemini and AI/ML API get the flat field only for models known to take + * it (`modelId`, ATO-1359): Gemini's thinking models (2.5 and 3.x) on its + * OpenAI-compatible endpoint, and OpenAI's reasoning models on AI/ML API. + * Without it a compaction summary on those models thought without bound + * (Gemini 2.5 thinks dynamically by default), spent the reasoning room + * and fell back to the crude summary. Other models there keep the body + * as before, since the field is not documented for them. */ export function reasoningEffortField( providerKind: string | undefined, effort: "low" | "medium" | "high", + modelId?: string, ): Record { switch (providerKind) { case "openrouter": @@ -139,6 +165,16 @@ export function reasoningEffortField( case "openai-compatible": case "qwen-openai-compatible": return { reasoning_effort: effort }; + case "gemini": { + const bare = modelId ? bareModelId(modelId) : ""; + return GEMINI_THINKING_MODEL_RE.test(bare) && !GEMINI_NON_CHAT_VARIANT_RE.test(bare) + ? { reasoning_effort: effort } + : {}; + } + case "aimlapi": + return modelId && OPENAI_REASONING_MODEL_RE.test(bareModelId(modelId)) + ? { reasoning_effort: effort } + : {}; default: return {}; } diff --git a/src/llm/provider/openai/openai-build-body.test.ts b/src/llm/provider/openai/openai-build-body.test.ts index 3198bb0f8..0ea37ce8e 100644 --- a/src/llm/provider/openai/openai-build-body.test.ts +++ b/src/llm/provider/openai/openai-build-body.test.ts @@ -678,6 +678,36 @@ describe("reasoning effort and the turn's output ceiling (F20)", () => { }); }); +describe("reasoning effort on Gemini and AI/ML API (ATO-1359)", () => { + const bodyFor = (providerKind: string, model: string) => + buildOpenAiChatBody({ prompt: "hi", reasoningEffort: "low", disableThinking: true }, model, false, undefined, undefined, undefined, undefined, { providerKind }); + + it("bounds a Gemini thinking model's reasoning with reasoning_effort", () => { + for (const model of ["gemini-2.5-flash", "gemini-2.5-pro", "gemini-2.5-flash-lite", "models/gemini-2.5-flash", "gemini-3-pro-preview"]) { + const body = bodyFor("gemini", model); + expect(body.reasoning_effort).toBe("low"); + expect(body).not.toHaveProperty("reasoning"); + // The local-only template switch is still not sent to a cloud API. + expect(body).not.toHaveProperty("chat_template_kwargs"); + } + }); + + it("leaves Gemini models that do not think, or are not chat reasoners, untouched", () => { + for (const model of ["gemini-2.0-flash", "gemini-1.5-pro", "gemini-2.5-flash-image", "gemini-2.5-flash-preview-tts", "gemma-3-27b-it", "m"]) { + expect(bodyFor("gemini", model)).not.toHaveProperty("reasoning_effort"); + } + }); + + it("sends reasoning_effort on AI/ML API only for OpenAI reasoning models", () => { + for (const model of ["o3-mini", "openai/o4-mini", "gpt-5", "openai/gpt-5-mini"]) { + expect(bodyFor("aimlapi", model).reasoning_effort).toBe("low"); + } + for (const model of ["gpt-4o", "deepseek/deepseek-r1", "olmo-2", "m"]) { + expect(bodyFor("aimlapi", model)).not.toHaveProperty("reasoning_effort"); + } + }); +}); + describe("buildOpenAiChatBody: the template's thinking switch", () => { const ask = (request: Parameters[0], options: Parameters[7], extraBody?: Record) => buildOpenAiChatBody(request, "m", false, extraBody, undefined, undefined, undefined, options); diff --git a/src/llm/provider/openai/openai-build-body.ts b/src/llm/provider/openai/openai-build-body.ts index 39b4404fd..4234ce4af 100644 --- a/src/llm/provider/openai/openai-build-body.ts +++ b/src/llm/provider/openai/openai-build-body.ts @@ -156,7 +156,11 @@ export function buildOpenAiChatBody( if (filtered.reasoningEffort !== undefined) { Object.assign( body, - reasoningEffortField(options.providerKind, filtered.reasoningEffort), + reasoningEffortField( + options.providerKind, + filtered.reasoningEffort, + defaultChatModel, + ), ); } // The template's thinking switch, for a request that asked for none From e65550078e9054d8abc16e221065996d6cd2005d Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:17:59 +0300 Subject: [PATCH 039/155] fix(goal): share one money budget across all goal rounds Each /goal round is an ordinary turn, and each got the full task money cap, while the result's spendUsd covered only the last round. A goal under an overnight budget could spend the cap once per round, and the task's spend record undercounted it. The goal controller now gives each round what the earlier ones left of one cap (the run's own, else agent.task.maxCostUsd), stops with budget_limited when it is used up, and reports the rounds' spend summed. A cap of 0 still means none. ATO-1288 --- src/agent/goal/goal-controller.test.ts | 34 ++++++++++++++++++ src/agent/goal/goal-controller.ts | 36 ++++++++++++++++--- src/runtime/composition/runtime-goal.ts | 6 +++- .../composition/runtime-turn-service.ts | 4 ++- 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/agent/goal/goal-controller.test.ts b/src/agent/goal/goal-controller.test.ts index f285eff4a..5a77b8d7b 100644 --- a/src/agent/goal/goal-controller.test.ts +++ b/src/agent/goal/goal-controller.test.ts @@ -188,6 +188,40 @@ describe("runGoal", () => { expect(out.goal?.status).toBe("budget_limited"); }); + it("gives each round what is left of one money budget and sums the spend (ATO-1288)", async () => { + const budgets: (number | undefined)[] = []; + const h = harness({ + runRound: async (s, _m, options) => { + budgets.push(options.taskMaxCostUsd); + return { ...replied(s, "worked"), spendUsd: 0.75 }; + }, + judgeAnswers: Array(5).fill(verdict("not_met")), + }); + const out = await runGoal(h.deps, { + session, message: "start", goal: goal(["a"]), signal, taskMaxCostUsd: 2, + }); + expect(budgets).toEqual([2, 1.25, 0.5]); + expect(out.result.spendUsd).toBeCloseTo(2.25); + expect(out.goal?.status).toBe("budget_limited"); + expect(out.goal?.statusReason ?? "").toContain("money budget"); + }); + + it("leaves rounds uncapped when the money budget is 0", async () => { + const budgets: (number | undefined)[] = []; + const h = harness({ + runRound: async (s, _m, options) => { + budgets.push(options.taskMaxCostUsd); + return replied(s, "worked"); + }, + judgeAnswers: [verdict("met")], + }); + const out = await runGoal(h.deps, { + session, message: "start", goal: goal(["a"]), signal, taskMaxCostUsd: 0, + }); + expect(budgets).toEqual([undefined]); + expect(out.result).not.toHaveProperty("spendUsd"); + }); + it("honours a pause or clear requested between rounds", async () => { const controls: ("pause" | "clear" | null)[] = [null, "pause"]; const h = harness({ diff --git a/src/agent/goal/goal-controller.ts b/src/agent/goal/goal-controller.ts index 580d45553..824f02e3e 100644 --- a/src/agent/goal/goal-controller.ts +++ b/src/agent/goal/goal-controller.ts @@ -25,7 +25,7 @@ export interface GoalControllerDeps { runRound( session: SessionState, message: string, - options: { taskMaxDurationMs?: number }, + options: { taskMaxDurationMs?: number; taskMaxCostUsd?: number }, ): Promise; /** Run the goal's shell checks in the session's folder. */ runChecks( @@ -54,6 +54,11 @@ export interface GoalRunInput { signal: AbortSignal; /** Wall-clock budget for all rounds together; absent, rounds alone limit. */ taskMaxDurationMs?: number; + /** + * Money budget for all rounds together, in USD (ATO-1288). Each round + * gets what the earlier ones left; absent or `0`, no cap. + */ + taskMaxCostUsd?: number; } export interface GoalRunOutcome { @@ -75,6 +80,11 @@ export async function runGoal( let session = input.session; let message = input.message; let last: RunTurnResult | undefined; + const costCap = input.taskMaxCostUsd !== undefined && Number.isFinite(input.taskMaxCostUsd) && input.taskMaxCostUsd > 0 + ? input.taskMaxCostUsd + : undefined; + // What the rounds spent together; `undefined` until one reports spend. + let spentUsd: number | undefined; const commit = (next: GoalState): GoalState => { goal = next; @@ -84,12 +94,12 @@ export async function runGoal( }; const stop = (status: GoalState["status"], reason: string): GoalRunOutcome => { commit(setGoalStatus(goal, status, now(), reason)); - return finish(last, session, goal); + return finish(last, session, goal, spentUsd); }; for (;;) { const control = deps.pendingControl?.(session.id) ?? null; - if (control === "clear") return finish(last, session, null); + if (control === "clear") return finish(last, session, null, spentUsd); if (control === "pause") return stop("paused", "paused by you"); if (input.signal.aborted) return stop("paused", "the turn was stopped"); if (goal.roundsUsed >= goal.maxRounds) { @@ -99,10 +109,18 @@ export async function runGoal( if (remaining !== undefined && remaining <= 0) { return stop("budget_limited", "the time limit for this run ran out"); } + const costLeft = costCap === undefined ? undefined : costCap - (spentUsd ?? 0); + if (costLeft !== undefined && costLeft <= 0) { + return stop("budget_limited", "the money budget for this run ran out"); + } commit({ ...goal, roundsUsed: goal.roundsUsed + 1, updatedAt: now() }); - last = await deps.runRound(session, message, remaining === undefined ? {} : { taskMaxDurationMs: remaining }); + last = await deps.runRound(session, message, { + ...(remaining === undefined ? {} : { taskMaxDurationMs: remaining }), + ...(costLeft === undefined ? {} : { taskMaxCostUsd: costLeft }), + }); session = last.session; + if (last.spendUsd !== undefined) spentUsd = (spentUsd ?? 0) + last.spendUsd; const abnormal = abnormalEnding(last); if (abnormal) return stop(abnormal.status, abnormal.reason); @@ -223,6 +241,7 @@ function finish( last: RunTurnResult | undefined, session: SessionState, goal: GoalState | null, + spentUsd: number | undefined, ): GoalRunOutcome { // Stopped before the first round ran: an empty result on the input session. const result: RunTurnResult = last ?? { session, reason: "cancelled", stepCount: 0 }; @@ -233,7 +252,14 @@ function finish( metadata: withGoalState(result.session.metadata, goal), }; return { - result: { ...result, session: finalSession, ...(goal === null ? {} : { goal }) }, + // The money of every round, not only the last one: a task's budget + // and its report read this figure. + result: { + ...result, + session: finalSession, + ...(goal === null ? {} : { goal }), + ...(spentUsd === undefined ? {} : { spendUsd: spentUsd }), + }, goal, }; } diff --git a/src/runtime/composition/runtime-goal.ts b/src/runtime/composition/runtime-goal.ts index 68785d9cc..8fc7d7af7 100644 --- a/src/runtime/composition/runtime-goal.ts +++ b/src/runtime/composition/runtime-goal.ts @@ -47,6 +47,7 @@ export interface RuntimeGoalRunOptions { goal: GoalSpec; signal?: AbortSignal; taskMaxDurationMs?: number; + taskMaxCostUsd?: number; onGoalProgress?: (goal: GoalState) => void; } @@ -55,6 +56,7 @@ export type GoalRoundRunner = ( session: SessionState, message: string, taskMaxDurationMs: number | undefined, + taskMaxCostUsd?: number, ) => Promise; export function createRuntimeGoalRunner(deps: RuntimeGoalDeps) { @@ -155,7 +157,7 @@ export function createRuntimeGoalRunner(deps: RuntimeGoalDeps) { try { const outcome = await runGoal( { - runRound: (s, m, o) => runRound(s, m, o.taskMaxDurationMs), + runRound: (s, m, o) => runRound(s, m, o.taskMaxDurationMs, o.taskMaxCostUsd), runChecks, judge, saveGoal, @@ -173,6 +175,8 @@ export function createRuntimeGoalRunner(deps: RuntimeGoalDeps) { signal: options.signal ?? new AbortController().signal, // All rounds together stay inside the turn's own time ceiling. taskMaxDurationMs: options.taskMaxDurationMs ?? getConfig().agent.task.maxDurationMs, + // And inside its money budget (ATO-1288): one cap for all rounds. + taskMaxCostUsd: options.taskMaxCostUsd ?? getConfig().agent.task.maxCostUsd, }, ); if (outcome.goal === null) saveGoal(session.id, null); diff --git a/src/runtime/composition/runtime-turn-service.ts b/src/runtime/composition/runtime-turn-service.ts index f0fbfea30..88e1cdc16 100644 --- a/src/runtime/composition/runtime-turn-service.ts +++ b/src/runtime/composition/runtime-turn-service.ts @@ -634,13 +634,15 @@ export function createRuntimeTurnService( goal, ...(runOptions.signal ? { signal: runOptions.signal } : {}), ...(runOptions.taskMaxDurationMs === undefined ? {} : { taskMaxDurationMs: runOptions.taskMaxDurationMs }), + ...(runOptions.taskMaxCostUsd === undefined ? {} : { taskMaxCostUsd: runOptions.taskMaxCostUsd }), ...(onGoalProgress ? { onGoalProgress } : {}), }, // Each round is an ordinary turn: queued, approved and saved alike. - (roundSession, message, remainingMs) => + (roundSession, message, remainingMs, remainingUsd) => runTurn(roundSession, message, { ...turnOptions, ...(remainingMs === undefined ? {} : { taskMaxDurationMs: remainingMs }), + ...(remainingUsd === undefined ? {} : { taskMaxCostUsd: remainingUsd }), }), ); } From e710a83f6ce952270d59d4232a4f68cf580ac4b0 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:19:39 +0300 Subject: [PATCH 040/155] fix(tasks): report a task's spend against its cap and a used-up budget as needs you The Telegram report's meta line showed this run's spend only, while the cap covers every attempt of the task. It now shows the task's total as "spent ~$X of $Y" when the task has a cap, and this run's spend only when no total is recorded. A task the runner blocks with budget_spent: or time_spent: never reached a turn, so its report was the bare error line. It now gets the usual card with the budget or time line under Needs you, the error kept under Failed. ATO-1288 --- .../task-report-message-sections.test.ts | 24 ++++++++++++++++ src/channels/telegram/task-report-message.ts | 28 +++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/channels/telegram/task-report-message-sections.test.ts b/src/channels/telegram/task-report-message-sections.test.ts index b4f6e4b77..d5cbbe057 100644 --- a/src/channels/telegram/task-report-message-sections.test.ts +++ b/src/channels/telegram/task-report-message-sections.test.ts @@ -167,4 +167,28 @@ describe("formatTaskReportDigest", () => { for (const r of group) expect(text).toContain(r.taskId); }); }); + + it("shows the task's spend against its cap, not only this run's (ATO-1288)", () => { + const text = formatTaskReportMessage( + makeReport({ spendUsd: 0.5, spentUsd: 4.8, maxCostUsd: 5, stopCause: "cost_ceiling" }), + ); + expect(text).toContain("spent ~$4.80 of $5.00"); + expect(text).not.toContain("spend ~$0.50"); + }); + + it("puts a task blocked on a used-up budget under Needs you (ATO-1288)", () => { + const text = formatTaskReportMessage( + makeReport({ + status: "blocked", + replyText: null, + errorCategory: "tool", + errorMessage: "budget_spent: $5.00 of $5.00 already spent by earlier attempts", + spentUsd: 5, + maxCostUsd: 5, + }), + ); + expect(text).toContain("Needs you:"); + expect(text).toContain("Stopped at the budget limit."); + expect(text).toContain("budget_spent: $5.00 of $5.00"); + }); }); diff --git a/src/channels/telegram/task-report-message.ts b/src/channels/telegram/task-report-message.ts index 5ff48462a..6659fa5bc 100644 --- a/src/channels/telegram/task-report-message.ts +++ b/src/channels/telegram/task-report-message.ts @@ -92,12 +92,16 @@ export function formatTaskReportMessage( minText: RESULT_MIN_CHARS, } : null; - const hasRun = report.summary != null || report.stopCause != null; + // A task blocked before its turn because earlier attempts used up its + // budget is a "needs you", not only an error line (ATO-1288). + const usedUp = report.status === "blocked" ? usedUpBudgetCause(report.errorMessage) : null; + const stopCause = report.stopCause ?? usedUp; + const hasRun = report.summary != null || stopCause != null; const card = hasRun ? buildTaskReportCard({ status: report.status, summary: report.summary ?? null, - stopCause: report.stopCause ?? null, + stopCause: stopCause ?? null, }) : null; if (card === null && errorLine !== null) { @@ -203,6 +207,17 @@ export function formatTaskReportDigest( return out; } +/** + * The ceiling a `budget_spent:` / `time_spent:` block names + * (`TaskRunner.runOne`), or `null` for any other error. + */ +function usedUpBudgetCause(message: string | null): string | null { + if (message === null) return null; + if (message.startsWith("budget_spent:")) return "cost_ceiling"; + if (message.startsWith("time_spent:")) return "time_ceiling"; + return null; +} + function metaLine(report: TaskReport): string { const parts = [`Attempt ${report.attempts}/${report.maxAttempts}`]; if (report.durationMs !== null) { @@ -210,7 +225,14 @@ function metaLine(report: TaskReport): string { } if (report.stepCount != null) parts.push(`${report.stepCount} steps`); if (report.stopCause != null) parts.push(`stop: ${report.stopCause}`); - if (report.spendUsd != null) parts.push(`spend ~${formatSpend(report.spendUsd)}`); + // The task's spend over all its attempts against its cap, which is + // what the cap governs (ATO-1288); this run's own spend otherwise. + if (report.spentUsd != null) { + const spent = `spent ~${formatSpend(report.spentUsd)}`; + parts.push(report.maxCostUsd != null ? `${spent} of ${formatSpend(report.maxCostUsd)}` : spent); + } else if (report.spendUsd != null) { + parts.push(`spend ~${formatSpend(report.spendUsd)}`); + } return parts.join(" · "); } From 062cce1e77d46ec12308edd7046be5c66f5b0081 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:21:08 +0300 Subject: [PATCH 041/155] fix(tasks): end an approval wait when the task's time limit passes The loop checks the task's wall-clock ceiling only when a step starts, and a tool waiting for approval waits until someone answers. A desktop "Run overnight" task that hit an approval prompt with nobody there sat on it past its limit until morning. For a turn whose caller set the time limit (a task run), the loop now arms a timer for each step at the time the task has left. When it fires it denies the session's pending approvals with a plain reason, and keeps denying new ones every 5 s until the step ends; the next step sees the ceiling and writes the closing summary. Ordinary chat turns are not affected. Prompts of Fusion workers (their own sessions) are not covered. ATO-1367 --- src/agent/agent-contract.ts | 6 ++ src/agent/agent-loop.ts | 17 +++++- src/agent/turn/approval-deadline.test.ts | 58 +++++++++++++++++++ src/agent/turn/approval-deadline.ts | 73 ++++++++++++++++++++++++ src/runtime/bootstrap.ts | 3 + 5 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 src/agent/turn/approval-deadline.test.ts create mode 100644 src/agent/turn/approval-deadline.ts diff --git a/src/agent/agent-contract.ts b/src/agent/agent-contract.ts index a889bd376..7a8b06ccf 100644 --- a/src/agent/agent-contract.ts +++ b/src/agent/agent-contract.ts @@ -118,6 +118,12 @@ export interface AgentLoopDependencies { * for the rest of the turn, and a new turn or a steer is a new answer. */ forgetDeclinedApprovals?: (sessionId: string) => void; + /** + * Deny every approval `sessionId` is waiting on, as nobody's decision + * (`ApprovalGate.denyPendingForSession`). The loop calls it once a + * task's time limit passes during a step (ATO-1367). + */ + denyPendingApprovals?: (sessionId: string, reason: string) => void; slotManager: SlotManager; grammar: string; llmComplete: (params: LlmStreamParams) => Promise; diff --git a/src/agent/agent-loop.ts b/src/agent/agent-loop.ts index 14b5b7fb6..998c672c3 100644 --- a/src/agent/agent-loop.ts +++ b/src/agent/agent-loop.ts @@ -14,6 +14,7 @@ import { prepareAgentTurn, prepareTurnPolicies, prepareTurnBudgets, createTurnLo import { refreshMemoryContext, createSurfacedMemoryTracker } from "./turn/turn-memory-context.js"; export { MAX_SURFACED_NOTE_ALLOWLIST } from "./turn/turn-memory-context.js"; import { finalizeAgentTurn, finalizeFailedAgentTurn } from "./turn/turn-finalization.js"; +import { armApprovalDeadline } from "./turn/approval-deadline.js"; import { costCeilingNotice, costCeilingReached, recordStepSpend } from "./turn/turn-spend.js"; export { formatTaskStoppedReply } from "./turn/turn-finalization.js"; import { delegationProducedWork, recordDelegation, wouldRefuse as fusionGateWouldRefuse } from "./policies/fusion-orchestrator-mode.js"; @@ -642,7 +643,21 @@ export class AgentLoop { }); } } - const outcome = await executeStep(stepContext, stepDeps); + // A task's time limit also ends a wait on an approval prompt + // nobody is there to answer (ATO-1367). + const disposeApprovalDeadline = + options.taskMaxDurationMs !== undefined && this.deps.denyPendingApprovals + ? armApprovalDeadline({ + msLeft: durationCeilingMs - (Date.now() - taskStartedAt), + deny: (reason) => this.deps.denyPendingApprovals?.(state.id, reason), + }) + : () => undefined; + let outcome: Awaited>; + try { + outcome = await executeStep(stepContext, stepDeps); + } finally { + disposeApprovalDeadline(); + } requestDeadline.dispose(); const durationMs = Date.now() - started; if (turn.awaitingRecovery) { diff --git a/src/agent/turn/approval-deadline.test.ts b/src/agent/turn/approval-deadline.test.ts new file mode 100644 index 000000000..e7f8898b9 --- /dev/null +++ b/src/agent/turn/approval-deadline.test.ts @@ -0,0 +1,58 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + APPROVAL_DEADLINE_REPEAT_MS, + APPROVAL_TIME_LIMIT_REASON, + armApprovalDeadline, +} from "./approval-deadline.js"; + +describe("armApprovalDeadline (ATO-1367)", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("denies pending prompts once the time is up, and keeps denying until the step ends", () => { + vi.useFakeTimers(); + const deny = vi.fn(); + const dispose = armApprovalDeadline({ msLeft: 30_000, deny }); + vi.advanceTimersByTime(29_999); + expect(deny).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1); + expect(deny).toHaveBeenCalledTimes(1); + expect(deny).toHaveBeenCalledWith(APPROVAL_TIME_LIMIT_REASON); + vi.advanceTimersByTime(APPROVAL_DEADLINE_REPEAT_MS * 2); + expect(deny).toHaveBeenCalledTimes(3); + dispose(); + vi.advanceTimersByTime(APPROVAL_DEADLINE_REPEAT_MS * 4); + expect(deny).toHaveBeenCalledTimes(3); + }); + + it("never fires for a step that ends before the limit", () => { + vi.useFakeTimers(); + const deny = vi.fn(); + const dispose = armApprovalDeadline({ msLeft: 10_000, deny }); + vi.advanceTimersByTime(5_000); + dispose(); + vi.advanceTimersByTime(60_000); + expect(deny).not.toHaveBeenCalled(); + }); + + it("fires at once when the limit already passed, and survives a throwing gate", () => { + vi.useFakeTimers(); + const deny = vi.fn(() => { + throw new Error("gate down"); + }); + const dispose = armApprovalDeadline({ msLeft: -5, deny }); + vi.advanceTimersByTime(0); + expect(deny).toHaveBeenCalledTimes(1); + dispose(); + }); + + it("does nothing without a finite limit", () => { + vi.useFakeTimers(); + const deny = vi.fn(); + armApprovalDeadline({ msLeft: Number.POSITIVE_INFINITY, deny })(); + vi.advanceTimersByTime(1e9); + expect(deny).not.toHaveBeenCalled(); + }); +}); diff --git a/src/agent/turn/approval-deadline.ts b/src/agent/turn/approval-deadline.ts new file mode 100644 index 000000000..393e1ca8d --- /dev/null +++ b/src/agent/turn/approval-deadline.ts @@ -0,0 +1,73 @@ +/** + * The task's time limit, applied to a step parked on an approval prompt + * (ATO-1367). + * + * The loop checks its wall-clock ceiling only when a step starts, and a + * tool waiting for approval waits until someone answers. An unattended + * run (the desktop's "Run overnight") with nobody to answer would then + * sit on the prompt past its limit until morning. Once the limit + * passes, every approval this session is waiting on is denied with a + * plain reason, and again every few seconds while the step runs, so a + * second prompt in the same step cannot park it either. The next step + * sees the ceiling and writes the closing summary. + * + * Only for a turn whose caller set the time limit (a task run), so an + * ordinary chat turn is never denied behind a person's back. + */ +export const APPROVAL_TIME_LIMIT_REASON = + "The time limit for this task was reached while this action waited for approval; it was not run."; + +/** How often, once past the limit, a new prompt is denied. */ +export const APPROVAL_DEADLINE_REPEAT_MS = 5_000; + +export interface ApprovalDeadlineTimers { + setTimeout(fn: () => void, ms: number): unknown; + clearTimeout(handle: unknown): void; + setInterval(fn: () => void, ms: number): unknown; + clearInterval(handle: unknown): void; +} + +const realTimers: ApprovalDeadlineTimers = { + setTimeout: (fn, ms) => { + const handle = setTimeout(fn, ms); + handle.unref?.(); + return handle; + }, + clearTimeout: (handle) => clearTimeout(handle as ReturnType), + setInterval: (fn, ms) => { + const handle = setInterval(fn, ms); + handle.unref?.(); + return handle; + }, + clearInterval: (handle) => clearInterval(handle as ReturnType), +}; + +/** + * Arm the limit for one step. `msLeft` is what the task has left when + * the step starts; `deny` denies the session's pending prompts. Returns + * the disposer to call when the step ends. + */ +export function armApprovalDeadline(input: { + msLeft: number; + deny: (reason: string) => void; + timers?: ApprovalDeadlineTimers; +}): () => void { + const timers = input.timers ?? realTimers; + if (!Number.isFinite(input.msLeft)) return () => undefined; + let repeat: unknown = null; + const fire = (): void => { + try { + input.deny(APPROVAL_TIME_LIMIT_REASON); + } catch { + // A failing gate must not take the step down with it. + } + }; + const first = timers.setTimeout(() => { + fire(); + repeat = timers.setInterval(fire, APPROVAL_DEADLINE_REPEAT_MS); + }, Math.max(0, input.msLeft)); + return () => { + timers.clearTimeout(first); + if (repeat !== null) timers.clearInterval(repeat); + }; +} diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index e497b92a3..785cc7919 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -558,6 +558,9 @@ export async function createAgentRuntime( ); }, forgetDeclinedApprovals: (sessionId: string) => approvals.forgetDeclined(sessionId), + denyPendingApprovals: (sessionId: string, reason: string) => { + approvals.denyPendingForSession(sessionId, reason); + }, slotManager, grammar: getGrammar(), persistContext: (state: SessionState) => { From d40f7ced89bf9c6372c8b3e699a74d6c4d0a7c5b Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:21:53 +0300 Subject: [PATCH 042/155] fix(llm): size DeepSeek's reasoning against the body the capacity check measures The reasoning budget was worked out from the request's own cap and raw tools, but the capacity check measures the finished body: `max_tokens` from extraBody or model params (it is not a reserved key) and the strict tools that actually go on the wire. A DeepSeek model with `max_tokens: 65536` on a 128k window could still fail the check. The body is now assembled once without any reasoning, measured with the same input, reply reserve and margin as `checkedBody`, and assembled again with what the window has left (less ten per cent for escaping) as the reasoning budget. Tests pin the size in a tight window and cover a model-params `max_tokens` and strict tools, each with no capacity throw and reasoning still sent. Linear: ATO-1397 --- .../provider/openai/openai-build-body.test.ts | 71 +++++++++---- src/llm/provider/openai/openai-build-body.ts | 100 +++++++++++------- 2 files changed, 112 insertions(+), 59 deletions(-) diff --git a/src/llm/provider/openai/openai-build-body.test.ts b/src/llm/provider/openai/openai-build-body.test.ts index 2a066de65..247acd721 100644 --- a/src/llm/provider/openai/openai-build-body.test.ts +++ b/src/llm/provider/openai/openai-build-body.test.ts @@ -763,28 +763,63 @@ describe("buildOpenAiChatBody: reasoning DeepSeek requires back (ATO-1397)", () expect(assistant(ask({ providerKind: "openai-compatible" }))).not.toHaveProperty("reasoning_content"); }); - it("sends less reasoning instead of failing the capacity check when the window is tight", () => { - const long = Array.from({ length: 4000 }, (_, i) => `step${i}`).join(" "); - const tight = { - ...messages, - turns: messages.turns.map((turn) => - turn.kind === "assistant_tool_call" ? { ...turn, reasoning: long } : turn, - ), - }; - const request = { + const words = (count: number) => Array.from({ length: count }, (_, i) => `step${i}`).join(" "); + const cloudRequest = (reasoning: string, window: number, extra: Record = {}) => + ({ prompt: "flat", - messages: tight, + messages: { + ...messages, + turns: messages.turns.map((turn) => + turn.kind === "assistant_tool_call" ? { ...turn, reasoning } : turn, + ), + }, tools, modelMode: { mode: "cloud" as const, source: "provider" as const, providerId: "deepseek", modelId: "deepseek-v4-flash" }, - contextBudget: { window: 4000, replyReserve: 1000 }, - } as Parameters[0]; - const build = (options: Parameters[7]) => - buildOpenAiChatBody(request, "deepseek-v4-flash", false, undefined, undefined, undefined, undefined, options); - const body = build({ providerKind: "openai-compatible", reasoningRequired: true }); + contextBudget: { window, replyReserve: 1000 }, + ...extra, + }) as Parameters[0]; + + it("sends less reasoning instead of failing the capacity check when the window is tight", () => { + const long = words(4000); + const body = buildOpenAiChatBody(cloudRequest(long, 4000), "deepseek-v4-flash", false, + undefined, undefined, undefined, undefined, { providerKind: "openai-compatible", reasoningRequired: true }); const sent = assistant(body)?.reasoning_content as string; - expect(sent.length).toBeLessThan(long.length); - expect(long.endsWith(sent)).toBe(true); - // The whole reasoning would not have fitted: proof the cap did the work. + // The whole reasoning would not have fitted: the cap did the work. expect(estimateTokens(long)).toBeGreaterThan(4000); + expect(sent.length).toBeGreaterThan(0); + expect(long.endsWith(sent)).toBe(true); + // Roughly the room left: 4000 - 1000 reply - 1024 margin - the rest of the body. + expect(estimateTokens(sent)).toBeGreaterThan(1000); + expect(estimateTokens(sent)).toBeLessThanOrEqual(4000 - 1000 - 1024); + }); + + it("sizes the reasoning against a max_tokens from model params, as the capacity check does", () => { + const long = words(40000); + const build = () => + buildOpenAiChatBody(cloudRequest(long, 128000), "deepseek-v4-flash", false, + undefined, undefined, undefined, undefined, + { providerKind: "openai-compatible", reasoningRequired: true, modelParams: { max_tokens: 65536 } }); + expect(build).not.toThrow(); + const body = build(); + expect(body.max_tokens).toBe(65536); + const sent = assistant(body)?.reasoning_content as string; + expect(sent.length).toBeGreaterThan(0); + expect(estimateTokens(sent)).toBeLessThanOrEqual(128000 - 65536 - 1024); + expect(estimateTokens(sent)).toBeGreaterThan(40000); + }); + + it("measures the strict tools that go on the wire", () => { + const long = words(4000); + const strictTools = [{ type: "function" as const, function: { name: "os__fs__read", + parameters: { type: "object", properties: Object.fromEntries( + Array.from({ length: 40 }, (_, i) => [`field_${i}`, { type: "string", description: `field number ${i}` }]), + ) } } }]; + const build = () => + buildOpenAiChatBody(cloudRequest(long, 6000, { tools: strictTools }), "deepseek-v4-flash", false, + undefined, undefined, true, undefined, { providerKind: "openai-compatible", reasoningRequired: true }); + expect(build).not.toThrow(); + const sent = assistant(build())?.reasoning_content as string; + expect(sent.length).toBeGreaterThan(0); + expect(long.endsWith(sent)).toBe(true); }); }); diff --git a/src/llm/provider/openai/openai-build-body.ts b/src/llm/provider/openai/openai-build-body.ts index 42d0777fd..c888e13c5 100644 --- a/src/llm/provider/openai/openai-build-body.ts +++ b/src/llm/provider/openai/openai-build-body.ts @@ -93,6 +93,38 @@ export function buildOpenAiChatBody( strictTools?: boolean, providerPreferences?: Record, options: OpenAiBodyOptions = {}, +): Record { + const assemble = (reasoningBudget: number | undefined) => + assembleOpenAiChatBody(request, defaultChatModel, stream, extraBody, + maxOutputTokens, strictTools, providerPreferences, options, reasoningBudget); + const budget = + request.modelMode?.mode === "cloud" ? request.contextBudget : undefined; + if (!options.reasoningRequired || !budget || budget.window === null) { + return checkedBody(assemble(undefined), request); + } + // The reasoning DeepSeek requires back is not counted by the cloud + // prompt's budget, so it is sized here against the capacity check + // itself (ATO-1397): the finished body without any reasoning — after + // the extraBody / model params merge, with the tools as they go on the + // wire — is measured the way `checkedBody` measures, and what the + // window has left, less ten per cent for JSON escaping, is the budget. + // Without it a long thinking turn fails the check instead of sending + // less of it. + const bare = assemble(0); + const room = budget.window - capacityUsed(bare, budget.replyReserve); + return checkedBody(assemble(Math.max(0, Math.floor(room * 0.9))), request); +} + +function assembleOpenAiChatBody( + request: CompletionRequest, + defaultChatModel: string, + stream: boolean, + extraBody: Record | undefined, + maxOutputTokens: number | undefined, + strictTools: boolean | undefined, + providerPreferences: Record | undefined, + options: OpenAiBodyOptions, + reasoningBudget: number | undefined, ): Record { const filtered = filterCloudCompletionRequest(request); const profile = modelParamProfile(defaultChatModel); @@ -109,10 +141,6 @@ export function buildOpenAiChatBody( // layout; a sub-call, or a service that refused the layout, sends the // flat text — which `messages` was built beside, from the same packed // conversation, so both say the same thing. - // Settled before the messages too: the reasoning DeepSeek requires back - // is held to the room the window has left after it (ATO-1397). - const cap = - filtered.maxTokens ?? filtered.maxOutputTokens ?? maxOutputTokens; const nativeOptions: NativeMessageOptions = { nameEscape: options.nameEscape ?? nameEscape, // A local server renders the template, which reads the @@ -128,9 +156,7 @@ export function buildOpenAiChatBody( resolveMessageShape(filtered, options) === "native" && filtered.messages ? buildNativeMessages(filtered.messages, { ...nativeOptions, - ...(options.reasoningRequired - ? reasoningRoom(filtered, filtered.messages, nativeOptions, cap) - : {}), + ...(reasoningBudget !== undefined ? { reasoningBudget } : {}), }) : [ { @@ -175,6 +201,8 @@ export function buildOpenAiChatBody( // family's own: OpenAI's reasoning models answer `max_tokens` with // "'max_tokens' is not supported with this model. Use // 'max_completion_tokens' instead." + const cap = + filtered.maxTokens ?? filtered.maxOutputTokens ?? maxOutputTokens; if (typeof cap === "number") body[profile.capField] = cap; // Reasoning effort, in the field this kind reads (`model-params.ts`); // omitted for a kind without a known one rather than guessed. @@ -266,7 +294,7 @@ export function buildOpenAiChatBody( ); } const modelParams = options.modelParams; - if (!extraBody && !modelParams) return checkedBody(body, request); + if (!extraBody && !modelParams) return body; // Vendor passthrough, then the model's own parameters. Merged last so // they can reach fields this builder does not model, then reserved // keys are restored on top. @@ -292,45 +320,35 @@ export function buildOpenAiChatBody( ) { delete merged.temperature; } - return checkedBody(merged, request); -} - -/** - * The reasoning budget for a body that must carry its tool calls' - * reasoning (`reasoningRequired`): what the cloud window has left once the - * same messages without any reasoning, the tools, the reply reserve and - * the safety margin `checkedBody` checks against are counted. The cloud - * prompt's own budget counts calls and results, not reasoning, so without - * this a long thinking turn would fail the capacity check instead of - * sending less of it. Ten per cent is kept back for JSON escaping. No - * budget (unbounded) when the request has no cloud window to check. - */ -function reasoningRoom( - request: CompletionRequest, - prompt: NonNullable, - nativeOptions: NativeMessageOptions, - cap: number | undefined, -): { reasoningBudget?: number } { - const budget = - request.modelMode?.mode === "cloud" ? request.contextBudget : undefined; - if (!budget || budget.window === null) return {}; - const bare = buildNativeMessages(prompt, { ...nativeOptions, reasoningBudget: 0 }); - const used = estimateTokens( - JSON.stringify({ messages: bare, tools: request.tools, response_format: request.responseFormat }), - ); - const reserve = - typeof cap === "number" && Number.isFinite(cap) ? cap : budget.replyReserve; - const room = budget.window - used - reserve - 1024; - return { reasoningBudget: Math.max(0, Math.floor(room * 0.9)) }; + return merged; } function checkedBody(body: Record, request: CompletionRequest): Record { if (request.modelMode?.mode === "cloud" && request.contextBudget) { - const cap = body.max_completion_tokens ?? body.max_tokens; - assertContextCapacity({ messages: body.messages, tools: body.tools, response_format: body.response_format }, { + assertContextCapacity(capacityInput(body), { window: request.contextBudget.window, - replyReserve: typeof cap === "number" && Number.isFinite(cap) ? cap : request.contextBudget.replyReserve, + replyReserve: capacityReserve(body, request.contextBudget.replyReserve), }); } return body; } + +/** What the capacity check measures of a finished body. */ +function capacityInput(body: Record): Record { + return { messages: body.messages, tools: body.tools, response_format: body.response_format }; +} + +/** The reply reserve the capacity check uses: the body's own cap, else the turn's. */ +function capacityReserve(body: Record, replyReserve: number): number { + const cap = body.max_completion_tokens ?? body.max_tokens; + return typeof cap === "number" && Number.isFinite(cap) ? cap : replyReserve; +} + +/** + * Tokens the capacity check counts against the window for this body: + * the same estimate and margin `assertContextCapacity` applies. + */ +function capacityUsed(body: Record, replyReserve: number): number { + return estimateTokens(JSON.stringify(capacityInput(body))) + + capacityReserve(body, replyReserve) + 1024; +} From b24b2d3d5984f5d3f52746fc72481e7b11b02f19 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:12:22 +0300 Subject: [PATCH 043/155] feat(http): take the last plain question and answer back Adds truncateSessionAt, which cuts a chat back to an earlier row and fixes every field that remembers a position in the transcript (macro-turn boundaries, the held pack start, the rolling summary, the cloud context, the context gauge), and planLastTurnRollback, which allows it only for a turn that changed nothing outside the transcript. POST /api/sessions/{id}/rollback-last-turn uses them so a client can ask the same question again without the model reading the old answer. Retry used to resend the question on top of the old pair, so the chat showed it twice and the model saw both tries. Refs ATO-1297, ATO-1298 --- src/http/README.md | 2 + src/http/route-rollback-last-turn.test.ts | 102 ++++++++ src/http/route-rollback-last-turn.ts | 72 ++++++ src/http/route-table.ts | 3 + src/session/index.ts | 7 + src/session/session-truncate.test.ts | 298 ++++++++++++++++++++++ src/session/session-truncate.ts | 195 ++++++++++++++ 7 files changed, 679 insertions(+) create mode 100644 src/http/route-rollback-last-turn.test.ts create mode 100644 src/http/route-rollback-last-turn.ts create mode 100644 src/session/session-truncate.test.ts create mode 100644 src/session/session-truncate.ts diff --git a/src/http/README.md b/src/http/README.md index afd6d3147..6a2a0fd97 100644 --- a/src/http/README.md +++ b/src/http/README.md @@ -55,4 +55,6 @@ Auth on the network (ATO-1126): the bearer token is compared in constant time ([ Per-chat folders (ATO-731, [session-working-dir.ts](session-working-dir.ts)): every tool step reads its session's `workingDir`, so `POST /v1/chat/completions` takes `working_dir` (absolute, existing) for a NEW session, `POST /api/sessions/{id}/working-dir` `{ workingDir }` moves one chat (`409 turn_in_progress` only while that chat runs a turn), and `GET /api/sessions?scope=all` lists chats from every folder. None of it restarts `serve`. +Taking a turn back (ATO-1298, [route-rollback-last-turn.ts](route-rollback-last-turn.ts)): `POST /api/sessions/{id}/rollback-last-turn` removes the chat's last question and its answer and returns `200 { id, text }`, the question's text. The body may carry `{ expectText }`, the question the client means; if the stored last question differs (a failed turn the agent never recorded), it answers `409 turn_mismatch` and removes nothing. It only removes: the client sends `text` again through `POST /v1/chat/completions`, so a retry runs the same path as a first try and the old answer is no longer in the chat the model reads. It does not reach outside the transcript: what the turn's reflection already wrote to long-term memory stays, and a chat cut back to nothing keeps its title and leaves `GET /api/sessions` (no question in it) until the question is sent again. Only a turn that changed nothing outside the transcript qualifies: a question answered by one reply, a stop marker, or nothing (a provider error). Otherwise `409` with `code` `turn_has_tools`, `turn_steered`, `turn_compacted`, `turn_in_progress`, `nothing_to_undo`, `turn_unrecognized` or `turn_mismatch`, and the chat is left as it was. `404 session_not_found` is an unknown chat; an agent without the route answers 404 without that code. The cut itself is [truncateSessionAt](../session/session-truncate.ts). + [Model tiers route](route-model-tiers.ts): `GET /api/model-tiers` returns the Cheap / Balanced / Best picks per provider key (`openrouter`, `aimlapi`, `gemini`, `claude-cli`) from the one data file, [model-tiers.ts](../llm/provider/presets/model-tiers.ts) (ATO-32). The TUI pickers list those models first under the same labels. diff --git a/src/http/route-rollback-last-turn.test.ts b/src/http/route-rollback-last-turn.test.ts new file mode 100644 index 000000000..1ba1ede1a --- /dev/null +++ b/src/http/route-rollback-last-turn.test.ts @@ -0,0 +1,102 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + createEmptySessionState, + type ConversationTurn, + type SessionState, +} from "../session/index.js"; + +import { startTestHarness, type Harness } from "./test-harness.js"; + +describe("POST /api/sessions/{id}/rollback-last-turn (ATO-1298)", () => { + let harness: Harness; + beforeEach(async () => { harness = await startTestHarness(); }); + afterEach(async () => { vi.restoreAllMocks(); await harness.cleanup(); }); + + const post = (id: string, body: unknown = {}) => fetch(`${harness.baseUrl}/api/sessions/${id}/rollback-last-turn`, { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), + }); + + function seed(id: string, turns: ConversationTurn[], starts: number[]): SessionState { + const state: SessionState = { + ...createEmptySessionState({ id, workingDir: harness.workingDir }), + status: "completed", + turns, + turnCount: starts.length, + macroTurnStarts: starts, + }; + harness.runtime.sessionStore.save(state); + return state; + } + + it("removes the last plain question and answer and returns the question", async () => { + seed("rb-1", [ + { kind: "user", text: "first", at: 1 }, + { kind: "assistant_reply", text: "one", at: 2 }, + { kind: "user", text: "Hello", at: 3 }, + { kind: "assistant_reply", text: "Hi!", at: 4 }, + ], [2, 4]); + const res = await post("rb-1"); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ id: "rb-1", text: "Hello" }); + const after = harness.runtime.sessionStore.load("rb-1"); + expect(after?.turns.map((t) => t.kind)).toEqual(["user", "assistant_reply"]); + expect(after?.turnCount).toBe(1); + expect(after?.macroTurnStarts).toEqual([2]); + }); + + it("refuses a turn with a tool and leaves the chat as it was", async () => { + seed("rb-2", [ + { kind: "user", text: "read it", at: 1 }, + { kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "a" }, at: 2 }, + { kind: "tool_result", tool: "os.fs.read", status: "ok", summary: "x", at: 3 }, + { kind: "assistant_reply", text: "done", at: 4 }, + ], [4]); + const res = await post("rb-2"); + expect(res.status).toBe(409); + const body = (await res.json()) as { error: { code: string } }; + expect(body.error.code).toBe("turn_has_tools"); + expect(harness.runtime.sessionStore.load("rb-2")?.turns).toHaveLength(4); + }); + + it("answers 409 turn_in_progress while the chat runs a turn", async () => { + seed("rb-3", [ + { kind: "user", text: "a", at: 1 }, + { kind: "assistant_reply", text: "b", at: 2 }, + ], [2]); + vi.spyOn(harness.runtime.turnController, "isBusy").mockImplementation((id: string) => id === "rb-3"); + const res = await post("rb-3"); + expect(res.status).toBe(409); + expect(((await res.json()) as { error: { code: string } }).error.code).toBe("turn_in_progress"); + expect(harness.runtime.sessionStore.load("rb-3")?.turns).toHaveLength(2); + }); + + it("refuses with turn_mismatch when expectText is not the last question", async () => { + seed("rb-5", [ + { kind: "user", text: "first", at: 1 }, + { kind: "assistant_reply", text: "one", at: 2 }, + ], [2]); + const res = await post("rb-5", { expectText: "a question the agent never stored" }); + expect(res.status).toBe(409); + expect(((await res.json()) as { error: { code: string } }).error.code).toBe("turn_mismatch"); + expect(harness.runtime.sessionStore.load("rb-5")?.turns).toHaveLength(2); + expect((await post("rb-5", { expectText: 42 })).status).toBe(400); + expect((await post("rb-5", { expectText: "first" })).status).toBe(200); + }); + + it("answers 404 session_not_found for an unknown chat", async () => { + const res = await post("nope"); + expect(res.status).toBe(404); + expect(((await res.json()) as { error: { code: string } }).error.code).toBe("session_not_found"); + }); + + it("stops a background compaction before writing", async () => { + seed("rb-4", [ + { kind: "user", text: "a", at: 1 }, + { kind: "assistant_reply", text: "b", at: 2 }, + ], [2]); + const cancel = vi.spyOn(harness.runtime, "cancelSessionCompaction"); + expect((await post("rb-4")).status).toBe(200); + expect(cancel).toHaveBeenCalledWith("rb-4"); + }); +}); diff --git a/src/http/route-rollback-last-turn.ts b/src/http/route-rollback-last-turn.ts new file mode 100644 index 000000000..cfa0147d1 --- /dev/null +++ b/src/http/route-rollback-last-turn.ts @@ -0,0 +1,72 @@ +import { planLastTurnRollback } from "../session/session-truncate.js"; +import { openaiError } from "./openai-errors.js"; +import { readJsonBody, sendError, sendJson, type HttpHandler } from "./request-context.js"; + +/** Plain words for each refusal; `code` is what a client branches on. */ +const REFUSAL_MESSAGES: Record = { + nothing_to_undo: "this chat has no question to take back", + turn_in_progress: "this chat has a turn in progress; try again when it finishes", + turn_has_tools: "the last turn ran tools, so it cannot be taken back; send the question again instead", + turn_steered: "a message was added to the last turn while it ran, so it cannot be taken back", + turn_compacted: "the last turn is already part of a compacted summary, so it cannot be taken back", + turn_unrecognized: "the last turn is not a single question and answer, so it cannot be taken back", + turn_mismatch: "the chat's last question is not the one given in expectText, so nothing was taken back", +}; + +/** + * `POST /api/sessions/{id}/rollback-last-turn` (ATO-1298): remove the + * chat's last question and its answer so a client can ask it again + * without the model reading the old answer. Body `{}` or + * `{ expectText }`: the question the client means; when the stored last + * question differs, `409 turn_mismatch` and nothing is removed. + * + * Only removes. The answer is `200 { id, text }` with the question's + * text; the client sends it again through `/v1/chat/completions` like + * any message, so a retry runs exactly the path a first try does. + * + * `404 session_not_found` for an unknown chat (an agent without this + * route answers 404 with no code, which is how a client tells the two + * apart). `409` with a `code` when the turn cannot be taken back: + * `turn_in_progress`, `turn_has_tools`, `turn_steered`, + * `turn_compacted`, `nothing_to_undo`, `turn_unrecognized`, `turn_mismatch` (see + * `planLastTurnRollback`). A refusal leaves the chat as it was. + */ +export function createRollbackLastTurnHandler(): HttpHandler { + return async (req, res, ctx) => { + const id = ctx.params.id ?? ""; + let expectText: string | undefined; + try { + const body = await readJsonBody(req); + const raw = (body as { expectText?: unknown } | null)?.expectText; + if (raw !== undefined && typeof raw !== "string") { + sendError(res, 400, openaiError("expectText must be a string", "invalid_request_error", "expectText")); + return; + } + expectText = raw; + } catch (err) { + sendError(res, 400, openaiError(`Invalid JSON: ${err instanceof Error ? err.message : String(err)}`)); + return; + } + const session = ctx.runtime.sessionStore.load(id); + if (!session) { + sendError(res, 404, openaiError(`session not found: ${id}`, "invalid_request_error", null, "session_not_found")); + return; + } + // This process's turn, or one another process on the same state dir runs (the store's live statuses). + const plan = ctx.runtime.turnController.isBusy(id) + ? ({ ok: false, code: "turn_in_progress" } as const) + : planLastTurnRollback(session, expectText === undefined ? {} : { expectText }); + if (!plan.ok) { + sendError(res, 409, openaiError(REFUSAL_MESSAGES[plan.code] ?? plan.code, "invalid_request_error", null, plan.code)); + return; + } + // Stop a compaction the person started on this chat: it would summarise + // the turn being removed. A background rolling summary needs no stop: it + // reloads the session and checks its boundary before it saves. + ctx.runtime.cancelSessionCompaction(id); + ctx.runtime.sessionStore.save(plan.state); + // A stopped turn keeps its resume marker; the turn it points at is gone. + ctx.runtime.sessionStore.clearTurnProgress(id); + sendJson(res, 200, { id, text: plan.text }); + }; +} diff --git a/src/http/route-table.ts b/src/http/route-table.ts index 00f07679b..ce36beee9 100644 --- a/src/http/route-table.ts +++ b/src/http/route-table.ts @@ -1,4 +1,5 @@ import { createSessionCompactionHandler } from "./route-compaction.js"; +import { createRollbackLastTurnHandler } from "./route-rollback-last-turn.js"; import type { RouteDefinition } from "./http-server.js"; import { createChatCompletionsHandler } from "./openai-chat-completions.js"; @@ -181,6 +182,8 @@ export function buildRouteTable(): RouteDefinition[] { }, // ATO-731: one chat's folder, changed without restarting serve. { method: "POST", path: "/api/sessions/{id}/working-dir", handler: createSetSessionWorkingDirHandler() }, + // ATO-1298: take the last plain question and answer back, for a real retry. + { method: "POST", path: "/api/sessions/{id}/rollback-last-turn", handler: createRollbackLastTurnHandler() }, { method: "POST", path: "/api/sessions/{id}/steer", diff --git a/src/session/index.ts b/src/session/index.ts index 38708a585..3fcdfd25e 100644 --- a/src/session/index.ts +++ b/src/session/index.ts @@ -35,6 +35,13 @@ export type { SessionSummaryPageOptions, } from "./session-summary-page.js"; export { normalizeSessionState } from "./normalize-session-state.js"; +export { planLastTurnRollback, truncateSessionAt } from "./session-truncate.js"; +export type { + LastTurnRollbackOptions, + LastTurnRollbackPlan, + LastTurnRollbackRefusal, + TruncateSessionResult, +} from "./session-truncate.js"; export { createEmptySessionState, appendFact, diff --git a/src/session/session-truncate.test.ts b/src/session/session-truncate.test.ts new file mode 100644 index 000000000..366a5526b --- /dev/null +++ b/src/session/session-truncate.test.ts @@ -0,0 +1,298 @@ +import { describe, expect, it } from "vitest"; + +import type { ConversationTurn } from "./conversation-turn.js"; +import { stoppedTurnMarker } from "./conversation-turn.js"; +import { compactionBoundaryHash } from "./session-compaction.js"; +import { + createEmptySessionState, + incrementTurnCount, + recordTurn, + type SessionState, +} from "./session-state.js"; +import { planLastTurnRollback, truncateSessionAt } from "./session-truncate.js"; + +const user = (text: string, at: number, steered = false): ConversationTurn => + steered ? { kind: "user", text, steered: true, at } : { kind: "user", text, at }; +const reply = (text: string, at: number): ConversationTurn => ({ + kind: "assistant_reply", + text, + at, +}); +const call = (at: number): ConversationTurn => ({ + kind: "assistant_tool_call", + tool: "os.fs.read", + args: { path: "a.txt" }, + at, +}); +const result = (at: number): ConversationTurn => ({ + kind: "tool_result", + tool: "os.fs.read", + status: "ok", + summary: "ok", + at, +}); + +/** Run `rows` as one turn the way the loop records it. */ +function turn(state: SessionState, ...rows: ConversationTurn[]): SessionState { + let next = state; + for (const row of rows) next = recordTurn(next, row); + return { ...incrementTurnCount(next), status: "completed" }; +} + +function fresh(): SessionState { + return createEmptySessionState({ id: "s1", workingDir: "/tmp" }); +} + +describe("truncateSessionAt", () => { + it("cuts two turns to one and the next turn gets the right boundary", () => { + let state = turn(fresh(), user("hi", 1), reply("hello", 2)); + state = turn(state, user("again", 3), reply("sure", 4)); + expect(state.macroTurnStarts).toEqual([2, 4]); + expect(state.turnCount).toBe(2); + + const { state: cut, compactionCovered } = truncateSessionAt(state, 2); + expect(compactionCovered).toBe(false); + expect(cut.turns).toHaveLength(2); + expect(cut.turnCount).toBe(1); + expect(cut.macroTurnStarts).toEqual([2]); + + const after = turn(cut, user("third", 5), reply("ok", 6)); + expect(after.macroTurnStarts).toEqual([2, 4]); + expect(after.turnCount).toBe(2); + }); + + it("cuts a single turn down to an empty chat", () => { + const state = turn(fresh(), user("hi", 1), reply("hello", 2)); + const { state: cut } = truncateSessionAt(state, 0); + expect(cut.turns).toEqual([]); + expect(cut.turnCount).toBe(0); + expect(cut.macroTurnStarts).toEqual([]); + expect(cut.status).toBe("pending"); + }); + + it("returns the state unchanged when the cut is at the end", () => { + const state = turn(fresh(), user("hi", 1), reply("hello", 2)); + expect(truncateSessionAt(state, 2).state).toBe(state); + }); + + it("counts cut prompts when no boundaries were recorded", () => { + const state: SessionState = { + ...fresh(), + turns: [user("a", 1), reply("b", 2), user("c", 3), reply("d", 4)], + turnCount: 2, + }; + delete state.macroTurnStarts; + const { state: cut } = truncateSessionAt(state, 2); + expect(cut.turnCount).toBe(1); + expect(cut.macroTurnStarts).toBeUndefined(); + }); + + it("does not uncount a turn whose end was never recorded", () => { + // The process died after the reply was checkpointed but before the + // turn closed: two rows, no boundary, and the turn was never counted. + let state = turn(fresh(), user("a", 1), reply("b", 2)); + state = recordTurn(recordTurn(state, user("c", 3)), reply("d", 4)); + expect(state.turnCount).toBe(1); + expect(state.macroTurnStarts).toEqual([2]); + const { state: cut } = truncateSessionAt(state, 2); + expect(cut.turnCount).toBe(1); + expect(cut.macroTurnStarts).toEqual([2]); + }); + + it("leaves the chat waiting for its next message", () => { + let state = turn(fresh(), user("a", 1), reply("b", 2)); + state = { ...turn(state, user("c", 3)), status: "failed" }; + expect(truncateSessionAt(state, 2).state.status).toBe("pending"); + }); + + it("drops a held pack start inside the cut part and keeps one before it", () => { + let state = turn(fresh(), user("a", 1), reply("b", 2)); + state = turn(state, user("c", 3), reply("d", 4)); + const inside = truncateSessionAt( + { ...state, conversationPackStart: { index: 2, at: 3, boundBy: "pairs" } }, + 2, + ).state; + expect(inside.conversationPackStart).toBeUndefined(); + const before = truncateSessionAt( + { ...state, conversationPackStart: { index: 1, at: 2, boundBy: "tokens" } }, + 2, + ).state; + expect(before.conversationPackStart).toEqual({ index: 1, at: 2, boundBy: "tokens" }); + }); + + it("drops a rolling summary that reaches past the cut", () => { + let state = turn(fresh(), user("a", 1), reply("b", 2)); + state = turn(state, user("c", 3), reply("d", 4)); + const rolling = { + version: 1 as const, + summary: "s", + from: 0, + baseHash: "x", + coveredThrough: 4, + boundaryHash: "y", + createdAt: 1, + model: null, + }; + expect(truncateSessionAt({ ...state, rollingSummary: rolling }, 2).state.rollingSummary) + .toBeUndefined(); + expect( + truncateSessionAt({ ...state, rollingSummary: { ...rolling, coveredThrough: 2 } }, 2) + .state.rollingSummary, + ).toBeDefined(); + }); + + it("drops the context gauge and the last error", () => { + const state = turn(fresh(), user("a", 1), reply("b", 2)); + const { state: cut } = truncateSessionAt( + { + ...state, + lastError: "boom", + contextUsage: { promptTokens: 5 } as unknown as SessionState["contextUsage"], + }, + 0, + ); + expect(cut.contextUsage).toBeUndefined(); + expect(cut.lastError).toBeNull(); + }); + + it("reports a compaction checkpoint over the cut part without touching it", () => { + let state = turn(fresh(), user("a", 1), reply("b", 2)); + state = turn(state, user("c", 3), reply("d", 4)); + const compaction = { + version: 1 as const, + summary: "s", + coveredThrough: 4, + boundaryHash: "h", + createdAt: 1, + reason: "manual", + model: null, + tokensBefore: 1, + tokensAfter: 1, + } as unknown as NonNullable; + const covered = truncateSessionAt({ ...state, compaction }, 2); + expect(covered.compactionCovered).toBe(true); + expect(covered.state.compaction).toBe(compaction); + const clear = truncateSessionAt( + { ...state, compaction: { ...compaction, coveredThrough: 2 } }, + 2, + ); + expect(clear.compactionCovered).toBe(false); + }); + + it("cuts the cloud context back to the kept rows", () => { + let state = turn(fresh(), user("a", 1), reply("b", 2)); + state = turn(state, user("c", 3), reply("d", 4)); + const entry = (i: number) => ({ + id: `turn:${i}`, + through: i + 1, + turn: { kind: "user" as const, text: String(i) }, + }); + const { state: cut } = truncateSessionAt( + { + ...state, + cloudContext: { + version: 1, + turnsThrough: 4, + boundaryHash: compactionBoundaryHash(state.turns[3]!), + entries: [entry(0), entry(1), entry(2), entry(3)], + }, + }, + 2, + ); + expect(cut.cloudContext?.turnsThrough).toBe(2); + expect(cut.cloudContext?.boundaryHash).toBe(compactionBoundaryHash(state.turns[1]!)); + expect(cut.cloudContext?.entries.map((e) => e.id)).toEqual(["turn:0", "turn:1"]); + }); +}); + +describe("planLastTurnRollback", () => { + it("takes back a plain question and answer and returns the question", () => { + let state = turn(fresh(), user("first", 1), reply("one", 2)); + state = turn(state, user("Hello", 3), reply("Hi!", 4)); + const plan = planLastTurnRollback(state); + expect(plan.ok).toBe(true); + if (!plan.ok) return; + expect(plan.text).toBe("Hello"); + expect(plan.start).toBe(2); + expect(plan.state.turns).toHaveLength(2); + expect(plan.state.turnCount).toBe(1); + }); + + it("takes back a turn that ended on a stop marker", () => { + const state = turn(fresh(), user("long story", 1), stoppedTurnMarker(2)); + const plan = planLastTurnRollback(state); + expect(plan.ok && plan.state.turns.length).toBe(0); + }); + + it("takes back a turn that has no answer at all (a provider error)", () => { + const state = { + ...turn(fresh(), user("hello", 1)), + status: "failed" as const, + lastError: "402", + }; + const plan = planLastTurnRollback(state); + expect(plan.ok).toBe(true); + if (!plan.ok) return; + expect(plan.state.turns).toEqual([]); + expect(plan.state.lastError).toBeNull(); + expect(plan.state.status).toBe("pending"); + }); + + it("refuses a turn that ran a tool", () => { + const state = turn(fresh(), user("read it", 1), call(2), result(3), reply("done", 4)); + expect(planLastTurnRollback(state)).toEqual({ ok: false, code: "turn_has_tools" }); + }); + + it("refuses a turn with a progress note", () => { + const state = turn(fresh(), user("go", 1), { + kind: "assistant_reply", + text: "on it", + progressNote: true, + at: 2, + }); + expect(planLastTurnRollback(state)).toEqual({ ok: false, code: "turn_has_tools" }); + }); + + it("refuses a reply that delivered files", () => { + const state = turn(fresh(), user("make it", 1), { + kind: "assistant_reply", + text: "here", + attachments: ["/tmp/a.png"], + at: 2, + }); + expect(planLastTurnRollback(state)).toEqual({ ok: false, code: "turn_has_tools" }); + }); + + it("refuses a steered turn", () => { + const state = turn(fresh(), user("a", 1), user("also b", 2, true), reply("c", 3)); + expect(planLastTurnRollback(state)).toEqual({ ok: false, code: "turn_steered" }); + }); + + it("refuses a compacted turn", () => { + const state = { + ...turn(fresh(), user("a", 1), reply("b", 2)), + compaction: { coveredThrough: 2 } as unknown as NonNullable, + }; + expect(planLastTurnRollback(state)).toEqual({ ok: false, code: "turn_compacted" }); + }); + + it("refuses an empty chat and a running one", () => { + expect(planLastTurnRollback(fresh())).toEqual({ ok: false, code: "nothing_to_undo" }); + const running = { ...turn(fresh(), user("a", 1), reply("b", 2)), status: "running" as const }; + expect(planLastTurnRollback(running)).toEqual({ ok: false, code: "turn_in_progress" }); + }); + + it("refuses when the last question is not the one the caller meant", () => { + let state = turn(fresh(), user("first", 1), reply("one", 2)); + // A failed second turn the agent never recorded: its last question is still "first". + expect(planLastTurnRollback(state, { expectText: "second" })) + .toEqual({ ok: false, code: "turn_mismatch" }); + state = turn(state, user("second", 3), reply("two", 4)); + expect(planLastTurnRollback(state, { expectText: " second\n" }).ok).toBe(true); + }); + + it("refuses two answers after one prompt", () => { + const state = turn(fresh(), user("a", 1), reply("b", 2), reply("c", 3)); + expect(planLastTurnRollback(state)).toEqual({ ok: false, code: "turn_unrecognized" }); + }); +}); diff --git a/src/session/session-truncate.ts b/src/session/session-truncate.ts new file mode 100644 index 000000000..f68fdf357 --- /dev/null +++ b/src/session/session-truncate.ts @@ -0,0 +1,195 @@ +import { compactionBoundaryHash } from "./session-compaction.js"; +import type { SessionState } from "./session-state.js"; +import { LIVE_SESSION_STATUSES } from "./session-retention.js"; + +/** + * Cutting a chat back to an earlier point (ATO-1297). + * + * The transcript is otherwise append-only, and several fields remember a + * position in it: the recorded macro-turn boundaries, the packer's held + * cut, the background summary's watermark, the cloud context's mirror of + * the transcript, the context gauge of the last turn. Dropping rows + * without fixing those leaves the next turn reading positions that no + * longer exist (the cloud context throws on its next extension, the + * packer holds a start past the end). This module is the one place that + * knows the whole list. + * + * No policy here: {@link truncateSessionAt} cuts anywhere, tools or not, + * so `/rewind` (ATO-1057) and editing a sent message (ATO-1058) can use + * it too. What may be undone is decided by {@link planLastTurnRollback}. + */ + +export interface TruncateSessionResult { + state: SessionState; + /** + * The compaction checkpoint covers rows that were cut. It is left on + * the state untouched: its summary speaks for rows that are gone, so + * the caller decides (refuse, or drop the checkpoint and pay a full + * re-read). + */ + compactionCovered: boolean; +} + +/** + * The session as it was with only `turns[0, index)`. `index` is clamped + * to `[0, turns.length]`; cutting at the end returns the state as is. + */ +export function truncateSessionAt( + state: SessionState, + index: number, +): TruncateSessionResult { + const length = state.turns.length; + const cut = Math.max(0, Math.min(length, Math.floor(index))); + if (cut >= length) return { state, compactionCovered: false }; + + const removed = state.turns.slice(cut); + const recorded = state.macroTurnStarts; + // Each recorded value is where a closed turn ended (= the next opens), + // so the closed turns removed are the values past the cut. A turn whose + // end was never recorded (the process died before it closed) was never + // counted either. Sessions written before the field count the prompts + // that were cut instead: each one opened a turn. + const closedRemoved = + recorded !== undefined + ? recorded.filter((value) => value > cut).length + : removed.filter((turn) => turn.kind === "user" && turn.steered !== true) + .length; + + const next: SessionState = { + ...state, + turns: state.turns.slice(0, cut), + turnCount: Math.max(0, state.turnCount - closedRemoved), + // What a chat waiting for its next message reads (a turn that ended + // on a reply leaves `pending`; `completed` means `finish` closed the + // task). The status the cut turn ended on goes with it. + status: "pending", + lastError: null, + updatedAt: Date.now(), + }; + if (recorded !== undefined) { + next.macroTurnStarts = recorded.filter((value) => value <= cut); + } + if ( + next.conversationPackStart !== undefined && + next.conversationPackStart.index >= cut + ) { + delete next.conversationPackStart; + } + if ( + next.rollingSummary !== undefined && + next.rollingSummary.coveredThrough > cut + ) { + delete next.rollingSummary; + } + // The gauge is the last turn's prompt, and the last turn is gone. + delete next.contextUsage; + if (next.cloudContext !== undefined && next.cloudContext.turnsThrough > cut) { + next.cloudContext = { + ...next.cloudContext, + turnsThrough: cut, + boundaryHash: cut > 0 ? compactionBoundaryHash(next.turns[cut - 1]!) : null, + entries: next.cloudContext.entries.filter((entry) => entry.through <= cut), + }; + } + return { + state: next, + compactionCovered: + state.compaction !== undefined && state.compaction.coveredThrough > cut, + }; +} + +/** Why the last turn cannot be taken back. */ +export type LastTurnRollbackRefusal = + /** No prompt to take back. */ + | "nothing_to_undo" + /** A turn is running on the chat. */ + | "turn_in_progress" + /** The turn ran tools: their effects stay, so a redo would not be one. */ + | "turn_has_tools" + /** A message was folded into the turn while it ran. */ + | "turn_steered" + /** A compaction checkpoint already summarises the turn. */ + | "turn_compacted" + /** The rows after the prompt are not one plain answer. */ + | "turn_unrecognized" + /** The last question is not the one the caller meant (`expectText`). */ + | "turn_mismatch"; + +export type LastTurnRollbackPlan = + | { + ok: true; + /** Index of the prompt row; the state keeps `turns[0, start)`. */ + start: number; + /** What the person typed, to send again. */ + text: string; + state: SessionState; + } + | { ok: false; code: LastTurnRollbackRefusal }; + +/** + * Take back the last turn when doing so changes nothing but the + * transcript: a prompt answered by one reply, by a stop marker, or by + * nothing at all (a provider error). Anything the turn did outside the + * transcript (tool calls) or that the person added to it mid-way (a + * steer) makes it refuse, because sending the prompt again would then + * not be a fresh try of the same question. + */ +export interface LastTurnRollbackOptions { + /** + * The question the caller means to take back. When the stored last + * question differs (a failed turn the agent never recorded, so its + * last question is the one before), it refuses with `turn_mismatch` + * rather than removing a turn the person did not point at. Compared + * with surrounding whitespace trimmed. + */ + expectText?: string; +} + +export function planLastTurnRollback( + state: SessionState, + options: LastTurnRollbackOptions = {}, +): LastTurnRollbackPlan { + if (LIVE_SESSION_STATUSES.includes(state.status)) { + return { ok: false, code: "turn_in_progress" }; + } + const turns = state.turns; + let start = -1; + for (let i = turns.length - 1; i >= 0; i -= 1) { + if (turns[i]?.kind === "user") { + start = i; + break; + } + } + if (start < 0) return { ok: false, code: "nothing_to_undo" }; + const prompt = turns[start]!; + if (prompt.kind !== "user") return { ok: false, code: "nothing_to_undo" }; + if (prompt.steered === true) return { ok: false, code: "turn_steered" }; + if ( + options.expectText !== undefined && + options.expectText.trim() !== prompt.text.trim() + ) { + return { ok: false, code: "turn_mismatch" }; + } + + const after = turns.slice(start + 1); + if ( + after.some( + (turn) => + turn.kind === "assistant_tool_call" || + turn.kind === "tool_result" || + (turn.kind === "assistant_reply" && turn.progressNote === true), + ) + ) { + return { ok: false, code: "turn_has_tools" }; + } + if (after.length > 1) return { ok: false, code: "turn_unrecognized" }; + const reply = after[0]; + if (reply?.kind === "assistant_reply" && (reply.attachments?.length ?? 0) > 0) { + // Files delivered with a reply were made by a tool. + return { ok: false, code: "turn_has_tools" }; + } + + const { state: next, compactionCovered } = truncateSessionAt(state, start); + if (compactionCovered) return { ok: false, code: "turn_compacted" }; + return { ok: true, start, text: prompt.text, state: next }; +} From a0e2670ae95f96dd83e86c56935346353f7df5fe Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:26:54 +0300 Subject: [PATCH 044/155] fix(overnight): block a task at another approval level instead of leaving it pending Review found that a row skipped for its approval level stayed pending forever at the head of the due queue (ordered by scheduled_for, a batch of 10), so a few such rows held up every other task, and nothing told the person why it never ran. The runner now claims it and blocks it with approval_level_mismatch: and the two levels, which also reaches the task report. The approval_level column is now added on its own: a failure to add it no longer turns the budget columns off. Without it the level is dropped and the task runs as before. ATO-1324 --- src/tasks/task-budget.ts | 26 +++++++++++++-- src/tasks/task-runner.test.ts | 35 ++++++++++++++++----- src/tasks/task-runner.ts | 50 ++++++++++++----------------- src/tasks/task-store.ts | 59 +++++++++++++++++++++-------------- 4 files changed, 105 insertions(+), 65 deletions(-) diff --git a/src/tasks/task-budget.ts b/src/tasks/task-budget.ts index ab601a5fa..8b42e840a 100644 --- a/src/tasks/task-budget.ts +++ b/src/tasks/task-budget.ts @@ -20,11 +20,31 @@ export const TASK_BUDGET_COLUMNS = [ { name: "spent_usd", type: "REAL" }, { name: "elapsed_ms", type: "INTEGER" }, { name: "stop_cause", type: "TEXT" }, - // The approval level the task was started under (ATO-1324). Not a - // budget, but the same kind of nullable column old binaries ignore. - { name: "approval_level", type: "INTEGER" }, ] as const; +/** + * The approval level a task was started under (ATO-1324): the same kind + * of nullable column older binaries ignore, added on its own so a + * failure to add it never turns the budgets off. Returns why it is + * missing, or `null` when it is there. + */ +export function ensureApprovalLevelColumn(db: Database.Database): string | null { + try { + const present = (db.prepare(`PRAGMA table_info(tasks)`).all() as Array<{ name: string }>) + .some((column) => column.name === "approval_level"); + if (present) return null; + try { + db.exec(`ALTER TABLE tasks ADD COLUMN approval_level INTEGER`); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (!/duplicate column/i.test(message)) throw err; + } + return null; + } catch (err) { + return err instanceof Error ? err.message : String(err); + } +} + /** * Give `tasks` its budget columns. Returns why they are missing (the * write lock was not granted, the file is read-only), or `null` when all diff --git a/src/tasks/task-runner.test.ts b/src/tasks/task-runner.test.ts index 3889ad91f..9deca877a 100644 --- a/src/tasks/task-runner.test.ts +++ b/src/tasks/task-runner.test.ts @@ -106,9 +106,9 @@ describe("TaskRunner", () => { ]); }); - it("leaves a task started under another approval level for that level (ATO-1324)", async () => { + it("blocks a task started under another approval level instead of running it (ATO-1324)", async () => { const { runtime, calls } = fakeRuntime(); - let level = 5; + const level = 5; const runner = new TaskRunner({ store, runtime, @@ -128,13 +128,32 @@ describe("TaskRunner", () => { approvalLevel: 2, }); expect(store.get(t.id)?.approvalLevel).toBe(2); - const skipped = await runner.runOne(t.id); - expect(skipped?.status).toBe("pending"); - expect(store.get(t.id)?.attempts).toBe(0); - const drained = await runner.drainPending(); - expect(drained.drained).toBe(0); + const blocked = await runner.runOne(t.id); + expect(blocked?.status).toBe("blocked"); + expect(blocked?.lastError).toMatch(/^approval_level_mismatch: started at approval level 2, this agent runs at 5/); expect(calls).toEqual([]); - level = 2; + }); + + it("runs a task at the approval level it was started under (ATO-1324)", async () => { + const { runtime, calls } = fakeRuntime(); + const runner = new TaskRunner({ + store, + runtime, + sessionLoader: fakeSessionLoader(session), + defaultMaxSteps: 5, + backoff: { initialMs: 1, maxMs: 10 }, + enabled: true, + runOnCreate: false, + sleep: async () => undefined, + approvalLevel: () => 2, + }); + const t = store.create({ + sessionId: session.id, + userMessage: "overnight work", + origin: "cli", + maxAttempts: 1, + approvalLevel: 2, + }); const final = await runner.runOne(t.id); expect(final?.status).toBe("completed"); expect(calls).toHaveLength(1); diff --git a/src/tasks/task-runner.ts b/src/tasks/task-runner.ts index a84cfaa7e..4b6a98964 100644 --- a/src/tasks/task-runner.ts +++ b/src/tasks/task-runner.ts @@ -151,8 +151,8 @@ export interface TaskRunnerOptions { logger?: StructuredLogger; /** * This process's live approval level (ATO-1324). A task that recorded - * the level it was started under is left for a process at that level. - * Omitted (tests, hosts without a gate): every task is runnable. + * another level than this one is blocked with `approval_level_mismatch:` + * instead of run. Omitted (tests, hosts without a gate): no check. */ approvalLevel?: () => number; metrics?: AgentMetrics; @@ -237,8 +237,6 @@ export class TaskRunner { private readonly runs = new Map(); /** Set by `stop`: nothing is claimed after it. */ private stopping = false; - /** Tasks already logged as left for a process at another approval level. */ - private readonly levelSkipsLogged = new Set(); /** Summary collectors of the runs in progress, for `liveStatus`. */ private readonly live = new Map< string, @@ -357,11 +355,6 @@ export class TaskRunner { if (!task) return null; if (task.status !== "pending") return task; - // Started under another approval level (an interrupted overnight - // run): leave the row for a process at that level instead of - // resuming it with more, or fewer, actions allowed. - if (this.leftForOtherLevel(task)) return task; - const resolvedTask = this.ensureSessionBeforeClaim(task); if (!resolvedTask) { return this.options.store.get(task.id); @@ -407,6 +400,24 @@ export class TaskRunner { return blocked; } + // Started under another approval level (an interrupted overnight + // run, ATO-1324): resuming it here would allow what it denied, or + // wait for a person who is not there. Blocked with a plain reason + // rather than left pending, where it would sit at the head of the + // due queue forever and hold up every other task. + const processLevel = this.options.approvalLevel?.(); + if (!approvalLevelAllows(claimed, processLevel)) { + const blocked = this.options.store.markBlocked(claimed.id, { + category: "tool", + message: + `approval_level_mismatch: started at approval level ${claimed.approvalLevel}, ` + + `this agent runs at ${processLevel}; start it again with \`atomic-agent overnight\``, + }); + this.recordTerminal(blocked); + this.maybeReport(blocked, null); + return blocked; + } + // The same for the time limit: a resume after a restart, clean or // not, gets what is left of it, not a new one. const timeLeft = remainingDurationMs(claimed); @@ -908,9 +919,6 @@ export class TaskRunner { let current: TaskRecord | null = task; while (current && current.status === "pending") { if (signal?.aborted || this.stopping) return; - // `runOne` hands such a row back untouched; counting it as a - // retry would spin this loop on it (ATO-1324). - if (this.leftForOtherLevel(current)) break; const before = current; current = await this.runOne(current.id, signal); outcome.drained += 1; @@ -949,24 +957,6 @@ export class TaskRunner { } } - /** - * True when `task` was started under another approval level than this - * process runs at (ATO-1324). Logged once per task, not every tick. - */ - private leftForOtherLevel(task: TaskRecord): boolean { - const processLevel = this.options.approvalLevel?.(); - if (approvalLevelAllows(task, processLevel)) return false; - if (!this.levelSkipsLogged.has(task.id)) { - this.levelSkipsLogged.add(task.id); - this.options.logger?.info("task left for a process at its own approval level", { - taskId: task.id, - taskApprovalLevel: task.approvalLevel ?? null, - processApprovalLevel: processLevel ?? null, - }); - } - return true; - } - private handleFailure( task: TaskRecord, category: LlmFailureCategory, diff --git a/src/tasks/task-store.ts b/src/tasks/task-store.ts index 7fcdaf266..c7898e27e 100644 --- a/src/tasks/task-store.ts +++ b/src/tasks/task-store.ts @@ -17,6 +17,7 @@ import { import { applyMigrations } from "./task-schema.js"; import { + ensureApprovalLevelColumn, ensureBudgetColumns, validateMaxCostUsd, validateMaxDurationMs, @@ -198,6 +199,8 @@ export class TaskStore { * or `null`. Without them a task asking for a budget is refused. */ readonly budgetsUnavailable: string | null; + /** Why this database has no `approval_level` column, or `null` (ATO-1324). */ + readonly approvalLevelUnavailable: string | null; /** * Why this database has no `run_owner` column — the open that should * have added it could not (see `ensureRunOwnerColumn`) — or `null`. @@ -218,11 +221,15 @@ export class TaskStore { this.budgetsUnavailable = ensureBudgetColumns(this.db); const withBudgets = this.budgetsUnavailable === null; const budgetColumns = withBudgets - ? ",\n max_cost_usd, max_duration_ms, spent_usd, elapsed_ms, stop_cause, approval_level" + ? ",\n max_cost_usd, max_duration_ms, spent_usd, elapsed_ms, stop_cause" : ""; const budgetValues = withBudgets - ? ",\n @max_cost_usd, @max_duration_ms, @spent_usd, @elapsed_ms, @stop_cause, @approval_level" + ? ",\n @max_cost_usd, @max_duration_ms, @spent_usd, @elapsed_ms, @stop_cause" : ""; + this.approvalLevelUnavailable = ensureApprovalLevelColumn(this.db); + const withLevel = this.approvalLevelUnavailable === null; + const levelColumn = withLevel ? ", approval_level" : ""; + const levelValue = withLevel ? ", @approval_level" : ""; this.ownerProbe = options.ownerProbe ?? currentTurnOwnerProbe(); this.dbIdentity = databaseIdentity(options.dbFile); @@ -232,13 +239,13 @@ export class TaskStore { attempts, max_attempts, last_error, last_error_cat, created_at, updated_at, started_at, completed_at, schedule_kind, schedule_value, scheduled_for, recurring, - last_scheduled_at, trigger_source, notify${budgetColumns} + last_scheduled_at, trigger_source, notify${budgetColumns}${levelColumn} ) VALUES ( @id, @session_id, @user_message, @max_steps, @status, @origin, @attempts, @max_attempts, @last_error, @last_error_cat, @created_at, @updated_at, @started_at, @completed_at, @schedule_kind, @schedule_value, @scheduled_for, @recurring, - @last_scheduled_at, @trigger_source, @notify${budgetValues} + @last_scheduled_at, @trigger_source, @notify${budgetValues}${levelValue} )`, ); // Names no `run_owner`: a copied row never carries a claim. @@ -248,13 +255,13 @@ export class TaskStore { attempts, max_attempts, last_error, last_error_cat, created_at, updated_at, started_at, completed_at, schedule_kind, schedule_value, scheduled_for, recurring, - last_scheduled_at, trigger_source, notify${budgetColumns} + last_scheduled_at, trigger_source, notify${budgetColumns}${levelColumn} ) VALUES ( @id, @session_id, @user_message, @max_steps, @status, @origin, @attempts, @max_attempts, @last_error, @last_error_cat, @created_at, @updated_at, @started_at, @completed_at, @schedule_kind, @schedule_value, @scheduled_for, @recurring, - @last_scheduled_at, @trigger_source, @notify${budgetValues} + @last_scheduled_at, @trigger_source, @notify${budgetValues}${levelValue} )`, ); this.selectStmt = this.db.prepare(`SELECT * FROM tasks WHERE id = ?`); @@ -423,7 +430,7 @@ export class TaskStore { const maxDurationMs = validateMaxDurationMs(input.maxDurationMs); const approvalLevel = validateApprovalLevel(input.approvalLevel); if ( - (maxCostUsd !== null || maxDurationMs !== null || approvalLevel !== null) && + (maxCostUsd !== null || maxDurationMs !== null) && this.budgetsUnavailable !== null ) { // Never drop a budget the caller asked for: an overnight run @@ -867,25 +874,29 @@ export class TaskStore { /** The row as bound to an INSERT: without budget keys when the columns are missing. */ private bindRow(row: TaskRow): TaskRow { + const bound: TaskRow = { ...row }; if (this.budgetsUnavailable === null) { - return { - ...row, - max_cost_usd: row.max_cost_usd ?? null, - max_duration_ms: row.max_duration_ms ?? null, - spent_usd: row.spent_usd ?? null, - elapsed_ms: row.elapsed_ms ?? null, - stop_cause: row.stop_cause ?? null, - approval_level: row.approval_level ?? null, - }; + bound.max_cost_usd = row.max_cost_usd ?? null; + bound.max_duration_ms = row.max_duration_ms ?? null; + bound.spent_usd = row.spent_usd ?? null; + bound.elapsed_ms = row.elapsed_ms ?? null; + bound.stop_cause = row.stop_cause ?? null; + } else { + delete bound.max_cost_usd; + delete bound.max_duration_ms; + delete bound.spent_usd; + delete bound.elapsed_ms; + delete bound.stop_cause; } - const bare = { ...row }; - delete bare.max_cost_usd; - delete bare.max_duration_ms; - delete bare.spent_usd; - delete bare.elapsed_ms; - delete bare.stop_cause; - delete bare.approval_level; - return bare; + // Without the column (it could not be added) the level is dropped and + // the task runs as one without, as before ATO-1324: refusing to start + // the run would be worse than the old behaviour. + if (this.approvalLevelUnavailable === null) { + bound.approval_level = row.approval_level ?? null; + } else { + delete bound.approval_level; + } + return bound; } private requireRecord(id: string): TaskRecord { From 3799e04ca163d8ad840f3fdacbe4302e08f20c3e Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:27:12 +0300 Subject: [PATCH 045/155] fix(overnight): count a used-up budget on resume and lost credit as stopped Review: a resume the runner blocked with budget_spent: or time_spent: still said "overnight blocked" and exited 1, while the same limit hit mid-run exits 2. It now says stopped and exits 2 too. The help text names credit_exhausted, which also exits 2. ATO-1323 --- src/cli/overnight-command.ts | 3 ++- src/cli/overnight-progress.test.ts | 11 ++++++++- src/cli/overnight-progress.ts | 39 +++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src/cli/overnight-command.ts b/src/cli/overnight-command.ts index 35ad49628..33cf1e516 100644 --- a/src/cli/overnight-command.ts +++ b/src/cli/overnight-command.ts @@ -59,7 +59,8 @@ const HELP = [ " is denied, since nobody is there to answer.", "", "Check a run with `atomic-agent task show `, stop it with `atomic-agent task cancel `.", - "Exit code: 0 finished, 2 stopped at the budget, time or step limit, 1 failed or cancelled.", + "Exit code: 0 finished; 2 stopped at a limit (budget, time, steps, no progress)", + " or out of provider credit; 1 failed or cancelled.", "", "Examples:", ' atomic-agent overnight "migrate the tests to vitest" --budget 5 --dir ~/code/app', diff --git a/src/cli/overnight-progress.test.ts b/src/cli/overnight-progress.test.ts index b0e6a3e81..6a655526f 100644 --- a/src/cli/overnight-progress.test.ts +++ b/src/cli/overnight-progress.test.ts @@ -135,10 +135,19 @@ describe("overnight progress", () => { }); it("exits 2 and says stopped when a limit ended the run (ATO-1323)", () => { - for (const cause of ["cost_ceiling", "time_ceiling", "step_ceiling", "no_progress"]) { + for (const cause of ["cost_ceiling", "time_ceiling", "step_ceiling", "no_progress", "credit_exhausted"]) { expect(overnightExitCode({ status: "completed", stopCause: cause })).toBe(2); expect(overnightOutcome({ status: "completed", stopCause: cause })).toBe(`stopped (${cause})`); } expect(overnightOutcome({ status: "completed", stopCause: null })).toBe("completed"); }); + + it("treats a resume blocked on a used-up budget or time as stopped (exit 2)", () => { + const spent = { status: "blocked" as const, stopCause: null, lastError: "budget_spent: $5.00 of $5.00 already spent" }; + expect(overnightExitCode(spent)).toBe(2); + expect(overnightOutcome(spent)).toBe("stopped (cost_ceiling)"); + const timed = { status: "blocked" as const, stopCause: null, lastError: "time_spent: 480 min of 480 min" }; + expect(overnightOutcome(timed)).toBe("stopped (time_ceiling)"); + expect(overnightExitCode({ status: "blocked", stopCause: null, lastError: "session_not_found: s" })).toBe(1); + }); }); diff --git a/src/cli/overnight-progress.ts b/src/cli/overnight-progress.ts index aefa18701..35e64a65c 100644 --- a/src/cli/overnight-progress.ts +++ b/src/cli/overnight-progress.ts @@ -85,9 +85,8 @@ export function formatOvernightSummary(input: OvernightSummaryInput): string { ` time: ${formatElapsed(input.elapsedMs)}`, ` spent: ${input.capUsd > 0 ? `${spent} of ${formatUsd(input.capUsd)}` : spent} (estimate; local models count as $0)`, ]; - if (stoppedByLimit(task)) { - lines.push(` ${stopCauseLine(task.stopCause!)}`); - } + const cause = limitCause(task); + if (cause !== null) lines.push(` ${stopCauseLine(cause)}`); if (input.deniedApprovals > 0) { lines.push( ` denied ${input.deniedApprovals} action(s) that needed approval; rerun with --no-approval to allow them`, @@ -123,23 +122,45 @@ export function formatDetachedNotice(taskId: string, logPath: string, dir: strin * beside it (ATO-1323). For the person or script reading the result * that is not "done": the work may be unfinished. */ -function stoppedByLimit(task: Pick): boolean { - return task.status === "completed" && typeof task.stopCause === "string" && task.stopCause.length > 0; +type OutcomeFields = Pick & Partial>; + +/** + * The limit that ended the run, if one did: a ceiling the turn hit (or + * the provider credit running out), or a resume the runner blocked + * because earlier attempts had already used the budget or the time. + */ +function limitCause(task: OutcomeFields): string | null { + if (task.status === "completed" && typeof task.stopCause === "string" && task.stopCause.length > 0) { + return task.stopCause; + } + if (task.status === "blocked" && typeof task.lastError === "string") { + if (task.lastError.startsWith("budget_spent:")) return "cost_ceiling"; + if (task.lastError.startsWith("time_spent:")) return "time_ceiling"; + } + return null; +} + +function stoppedByLimit(task: OutcomeFields): boolean { + return limitCause(task) !== null; } /** `completed`, `stopped (cost_ceiling)`, `failed`, ... */ -export function overnightOutcome(task: Pick): string { - return stoppedByLimit(task) ? `stopped (${task.stopCause})` : task.status; +export function overnightOutcome(task: OutcomeFields): string { + const cause = limitCause(task); + return cause !== null ? `stopped (${cause})` : task.status; } -/** Exit code when a step, time or money limit (or no progress) ended the run. */ +/** + * Exit code when a limit ended the run: the budget, the time, the steps, + * no progress, or the provider credit running out. + */ export const OVERNIGHT_STOPPED_EXIT_CODE = 2; /** * 0 only when the agent finished on its own; 2 when a limit stopped it * (the run itself worked, the task may not be done); 1 otherwise. */ -export function overnightExitCode(task: Pick): number { +export function overnightExitCode(task: OutcomeFields): number { if (stoppedByLimit(task)) return OVERNIGHT_STOPPED_EXIT_CODE; return task.status === "completed" ? 0 : 1; } From 97477f781f2fab7205b001305483136b5d5e9d44 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:27:28 +0300 Subject: [PATCH 046/155] fix(overnight): also say why when no Telegram bot is set up or Telegram is off Review: without a bot token the channel drops the report at once (nothing is queued), and with Telegram turned off the report waits in a channel that never starts. Both now print the plain reason before exit. ATO-1358 --- src/cli/overnight-command.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/cli/overnight-command.ts b/src/cli/overnight-command.ts index e2ea03a60..0a47bbd06 100644 --- a/src/cli/overnight-command.ts +++ b/src/cli/overnight-command.ts @@ -355,6 +355,9 @@ async function runForeground(args: OvernightArgs, taskId: string): Promise>; +const TELEGRAM_NO_BOT = "no Telegram bot is set up for this agent"; +const TELEGRAM_TURNED_OFF = "Telegram is turned off in this agent's settings"; + /** How long a channel still connecting may take to send the report. */ const TELEGRAM_FLUSH_WAIT_MS = 15_000; @@ -368,8 +371,13 @@ async function warnIfReportStuck(runtime: OvernightRuntime, taskId: string): Pro // The report is handed to the channel from the runner's report hook. await new Promise((resolve) => setImmediate(resolve)); const channel = runtime.telegramChannel; - if (!channel) { - process.stderr.write(formatTelegramUndelivered("Telegram is not set up for this agent", taskId)); + // Neither of these queues the report: it was dropped at once. + if (!channel || !channel.hasToken()) { + process.stderr.write(formatTelegramUndelivered(TELEGRAM_NO_BOT, taskId)); + return; + } + if (channel.state() === "disabled" && channel.hasPendingTaskReport(taskId)) { + process.stderr.write(formatTelegramUndelivered(TELEGRAM_TURNED_OFF, taskId)); return; } const deadline = Date.now() + TELEGRAM_FLUSH_WAIT_MS; From bf6edb397a85f738d9a95c749a639c329d7bcdec Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:27:39 +0300 Subject: [PATCH 047/155] docs(turn): keep resolveTurnLocalEngine's comment on its own function ATO-1350 --- src/runtime/composition/runtime-turn-service.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/runtime/composition/runtime-turn-service.ts b/src/runtime/composition/runtime-turn-service.ts index 48f6345bc..f34964c2d 100644 --- a/src/runtime/composition/runtime-turn-service.ts +++ b/src/runtime/composition/runtime-turn-service.ts @@ -746,13 +746,6 @@ export function createRuntimeTurnService( return { executeTurn, steer, runTurn, controlGoal: goals.control }; } -/** - * Which engine serves a turn on `providerId`, for `message_sent`'s - * `local_engine`; undefined for a cloud provider. The managed engine only - * answers for the synthesized `local-llama` entry in managed mode; any - * other llama-server entry, or external mode, is a server the operator - * runs. A fixed word, never a URL. - */ /** * Whether a turn ended because money ran out: its own budget, or the * provider account's credit (ATO-1350). No further paid side call @@ -762,6 +755,13 @@ export function stoppedForMoney(stopCause: RunTurnResult["stopCause"]): boolean return stopCause === "cost_ceiling" || stopCause === "credit_exhausted"; } +/** + * Which engine serves a turn on `providerId`, for `message_sent`'s + * `local_engine`; undefined for a cloud provider. The managed engine only + * answers for the synthesized `local-llama` entry in managed mode; any + * other llama-server entry, or external mode, is a server the operator + * runs. A fixed word, never a URL. + */ export function resolveTurnLocalEngine(config: AtomicAgentConfig, providerId: string): LocalEngine | undefined { if (!providerIdIsLlamaServer(resolveLlmConfig(config), providerId)) return undefined; if (providerId !== "local-llama" || config.localModels.mode !== "managed") return "external"; From 87c5f01d060a31b4d83a76f3834268a60c60e1c5 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:48:23 +0300 Subject: [PATCH 048/155] fix(fusion): name the spend cap, not a stop, when the cap ends the pool The cap aborts the same signal the pool now checks, so the next task read "not started: the turn was stopped". Ask the cap first. --- src/tools/fusion/worker-runner.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/tools/fusion/worker-runner.ts b/src/tools/fusion/worker-runner.ts index 31d77365f..d10721953 100644 --- a/src/tools/fusion/worker-runner.ts +++ b/src/tools/fusion/worker-runner.ts @@ -664,10 +664,12 @@ export async function runWorkerTasks( const runners = Array.from({ length: width }, async () => { for (let i = claim(); i < options.tasks.length; i = claim()) { // A stopped turn starts nothing more, capped or not (ATO-1334): - // `shouldStart` is only wired when a budget cap is set. - const stop = options.signal.aborted - ? "the turn was stopped" - : options.shouldStart?.(); + // `shouldStart` is only wired when a budget cap is set. The cap is + // asked first: a cap reached aborts the same signal, and its row + // must say the cap, not the person, stopped it. + const stop = + options.shouldStart?.() ?? + (options.signal.aborted ? "the turn was stopped" : undefined); results[i] = stop === undefined ? await runOneTask(deps, options, options.tasks[i]!) From 14ba9a7aeec177e4bbacba05695a0843153b9576 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:49:01 +0300 Subject: [PATCH 049/155] Gemini Flash: keep thinking off for the compaction summary; skip gpt-5-chat on AI/ML API A call that asks for no thinking (the compaction summary) sent reasoning_effort low to Gemini 2.5 Flash-Lite, which does not think by default, so it turned thinking on. On 2.5 Flash and Flash-Lite such a call now sends reasoning_effort none. gpt-5-chat ids on AI/ML API are not reasoners and no longer get the field. --- src/llm/provider/openai/model-params.ts | 23 +++++++++++++++---- .../provider/openai/openai-build-body.test.ts | 18 +++++++++++++-- src/llm/provider/openai/openai-build-body.ts | 1 + 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/llm/provider/openai/model-params.ts b/src/llm/provider/openai/model-params.ts index e987b5ca0..71ab31a80 100644 --- a/src/llm/provider/openai/model-params.ts +++ b/src/llm/provider/openai/model-params.ts @@ -133,6 +133,14 @@ export function forgetLearnedModelParams(): void { */ const GEMINI_THINKING_MODEL_RE = /^gemini-(?:2\.5|[3-9])(?![0-9])/i; const GEMINI_NON_CHAT_VARIANT_RE = /(?:image|tts|audio|live|embedding)/i; +/** + * 2.5 Flash and Flash-Lite can switch thinking off with + * `reasoning_effort: "none"`. Flash-Lite does not think by default, so a + * `low` sent there turns thinking on; Pro and 3.x cannot switch it off. + */ +const GEMINI_FLASH_25_RE = /^gemini-2\.5-flash/i; +/** `gpt-5-chat-latest` and the like: OpenAI ids that are not reasoners. */ +const OPENAI_NON_REASONING_VARIANT_RE = /-chat(?![a-z])/i; function bareModelId(modelId: string): string { return modelId.includes("/") @@ -158,6 +166,7 @@ export function reasoningEffortField( providerKind: string | undefined, effort: "low" | "medium" | "high", modelId?: string, + options: { disableThinking?: boolean } = {}, ): Record { switch (providerKind) { case "openrouter": @@ -167,12 +176,18 @@ export function reasoningEffortField( return { reasoning_effort: effort }; case "gemini": { const bare = modelId ? bareModelId(modelId) : ""; - return GEMINI_THINKING_MODEL_RE.test(bare) && !GEMINI_NON_CHAT_VARIANT_RE.test(bare) - ? { reasoning_effort: effort } - : {}; + if (!GEMINI_THINKING_MODEL_RE.test(bare) || GEMINI_NON_CHAT_VARIANT_RE.test(bare)) return {}; + // A call that wants no thinking (the compaction summary) must not + // turn it on: Flash takes "none", and elsewhere `low` is the floor. + if (options.disableThinking === true && GEMINI_FLASH_25_RE.test(bare)) { + return { reasoning_effort: "none" }; + } + return { reasoning_effort: effort }; } case "aimlapi": - return modelId && OPENAI_REASONING_MODEL_RE.test(bareModelId(modelId)) + return modelId && + OPENAI_REASONING_MODEL_RE.test(bareModelId(modelId)) && + !OPENAI_NON_REASONING_VARIANT_RE.test(bareModelId(modelId)) ? { reasoning_effort: effort } : {}; default: diff --git a/src/llm/provider/openai/openai-build-body.test.ts b/src/llm/provider/openai/openai-build-body.test.ts index 0ea37ce8e..88b9dfb65 100644 --- a/src/llm/provider/openai/openai-build-body.test.ts +++ b/src/llm/provider/openai/openai-build-body.test.ts @@ -682,8 +682,22 @@ describe("reasoning effort on Gemini and AI/ML API (ATO-1359)", () => { const bodyFor = (providerKind: string, model: string) => buildOpenAiChatBody({ prompt: "hi", reasoningEffort: "low", disableThinking: true }, model, false, undefined, undefined, undefined, undefined, { providerKind }); + const thinkingBodyFor = (providerKind: string, model: string) => + buildOpenAiChatBody({ prompt: "hi", reasoningEffort: "low" }, model, false, undefined, undefined, undefined, undefined, { providerKind }); + + it("switches thinking off on 2.5 Flash and Flash-Lite when the call wants none", () => { + for (const model of ["gemini-2.5-flash", "gemini-2.5-flash-lite", "models/gemini-2.5-flash", "gemini-2.5-flash-lite-preview-06-17"]) { + expect(bodyFor("gemini", model).reasoning_effort).toBe("none"); + } + }); + + it("sends the asked effort on Flash-Lite when thinking is wanted", () => { + expect(thinkingBodyFor("gemini", "gemini-2.5-flash-lite").reasoning_effort).toBe("low"); + expect(thinkingBodyFor("gemini", "gemini-2.5-flash").reasoning_effort).toBe("low"); + }); + it("bounds a Gemini thinking model's reasoning with reasoning_effort", () => { - for (const model of ["gemini-2.5-flash", "gemini-2.5-pro", "gemini-2.5-flash-lite", "models/gemini-2.5-flash", "gemini-3-pro-preview"]) { + for (const model of ["gemini-2.5-pro", "gemini-3-pro-preview", "models/gemini-3-flash-preview"]) { const body = bodyFor("gemini", model); expect(body.reasoning_effort).toBe("low"); expect(body).not.toHaveProperty("reasoning"); @@ -702,7 +716,7 @@ describe("reasoning effort on Gemini and AI/ML API (ATO-1359)", () => { for (const model of ["o3-mini", "openai/o4-mini", "gpt-5", "openai/gpt-5-mini"]) { expect(bodyFor("aimlapi", model).reasoning_effort).toBe("low"); } - for (const model of ["gpt-4o", "deepseek/deepseek-r1", "olmo-2", "m"]) { + for (const model of ["gpt-4o", "deepseek/deepseek-r1", "olmo-2", "m", "gpt-5-chat-latest", "openai/gpt-5-chat"]) { expect(bodyFor("aimlapi", model)).not.toHaveProperty("reasoning_effort"); } }); diff --git a/src/llm/provider/openai/openai-build-body.ts b/src/llm/provider/openai/openai-build-body.ts index 4234ce4af..36925b321 100644 --- a/src/llm/provider/openai/openai-build-body.ts +++ b/src/llm/provider/openai/openai-build-body.ts @@ -160,6 +160,7 @@ export function buildOpenAiChatBody( options.providerKind, filtered.reasoningEffort, defaultChatModel, + { disableThinking: filtered.disableThinking === true }, ), ); } From ca469ffa7b63bbb623794b0e5dd65c97bdacb05d Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:49:16 +0300 Subject: [PATCH 050/155] fix(tasks): split a very long approval deadline into timer hops Node runs a setTimeout delay over 2^31-1 ms after 1 ms, so a task time limit of more than ~24.8 days denied every approval at once. The wait is now split into hops of at most that, like request-deadline.ts. Adds a real-timer test for 2^31 ms and a loop-level test of the wiring with and without a task time limit. ATO-1367 --- .../agent-loop-approval-deadline.test.ts | 146 ++++++++++++++++++ src/agent/turn/approval-deadline.test.ts | 22 +++ src/agent/turn/approval-deadline.ts | 31 +++- 3 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 src/agent/agent-loop-approval-deadline.test.ts diff --git a/src/agent/agent-loop-approval-deadline.test.ts b/src/agent/agent-loop-approval-deadline.test.ts new file mode 100644 index 000000000..12df56463 --- /dev/null +++ b/src/agent/agent-loop-approval-deadline.test.ts @@ -0,0 +1,146 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { AgentLoop } from "./agent-loop.js"; +import type { RunTurnOptions } from "./agent-loop.js"; +import { APPROVAL_TIME_LIMIT_REASON } from "./turn/approval-deadline.js"; +import { buildDefaultToolRegistry } from "../tools/index.js"; +import { SlotManager } from "../llm/slot-manager.js"; +import { createEmptySessionState } from "../session/session-state.js"; +import type { CompletionResult } from "../llm/llama-server-client.js"; +import type { + CapabilitiesSummary, + ToolDescriptor, +} from "../prompt/stable-prefix.js"; + +/** + * ATO-1367: a task's time limit ends a step parked on an approval + * prompt. The tool below stands in for one waiting on the gate: it + * resolves only when the loop's `denyPendingApprovals` dep is called. + */ + +function completion(content: string): CompletionResult { + return { + content, + reasoningContent: "", + stop: true, + truncated: false, + timing: { promptMs: 1, predictedMs: 1, promptTokens: 10, predictedTokens: 5 }, + cacheHitTokens: 0, + slotId: 0, + modelId: "mock", + }; +} + +const TOOLS: ToolDescriptor[] = [ + { name: "reply", summary: "Reply to the user.", argsSchema: '{"text": string}' }, + { name: "finish", summary: "Finish the session.", argsSchema: '{"summary": string}' }, + { name: "os.fs.write", summary: "Write.", argsSchema: '{"path": string}' }, +]; + +const CAPS: CapabilitiesSummary = { + platform: "darwin", + arch: "arm64", + browserChannel: "chrome", + workingDir: "/work", + hasClipboard: true, + hasWmctrl: false, + hasNotifications: true, +}; + +describe("AgentLoop approval wait and the task time limit (ATO-1367)", () => { + let workingDir: string; + + beforeEach(() => { + vi.useFakeTimers(); + workingDir = mkdtempSync(join(tmpdir(), "atomic-approval-deadline-")); + }); + + afterEach(() => { + vi.useRealTimers(); + rmSync(workingDir, { recursive: true, force: true }); + }); + + function setup() { + const denied: Array<{ sessionId: string; reason: string }> = []; + let release: ((reason: string) => void) | null = null; + const registry = buildDefaultToolRegistry(); + registry.register({ + name: "os.fs.write", + description: "write, waiting on an approval nobody answers", + readonly: false, + run: (_args, ctx) => + new Promise((resolve, reject) => { + release = (reason) => + resolve({ tool: "os.fs.write", status: "error", summary: `denied: ${reason}` }); + ctx.signal?.addEventListener("abort", () => reject(new Error("aborted")), { once: true }); + }), + }); + let call = 0; + const agent = new AgentLoop({ + registry, + slotManager: new SlotManager(1), + grammar: 'root ::= "ok"', + toolTransport: "grammar", + toolCallAdapter: null, + supportsSlotAffinity: false, + llmComplete: async () => { + call += 1; + return completion( + call === 1 + ? JSON.stringify([{ tool: "os.fs.write", args: { path: "a.txt" } }]) + : JSON.stringify([{ tool: "reply", args: { text: "out of time" } }]), + ); + }, + denyPendingApprovals: (sessionId, reason) => { + denied.push({ sessionId, reason }); + release?.(reason); + release = null; + }, + toolDescriptors: TOOLS, + capabilities: CAPS, + skillCatalog: [], + }); + return { agent, denied }; + } + + function options(extra: Partial): RunTurnOptions { + return { + userMessage: "write the file", + maxSteps: 10, + taskMaxSteps: 10, + signal: new AbortController().signal, + ...extra, + }; + } + + it("denies the waiting approval at the limit and ends on a summary step", async () => { + const { agent, denied } = setup(); + const turn = agent.runTurn( + createEmptySessionState({ id: "s-approval", workingDir }), + options({ taskMaxDurationMs: 30_000 }), + ); + await vi.advanceTimersByTimeAsync(29_000); + expect(denied).toEqual([]); + await vi.advanceTimersByTimeAsync(1_000); + expect(denied).toEqual([{ sessionId: "s-approval", reason: APPROVAL_TIME_LIMIT_REASON }]); + const result = await turn; + expect(result.stopCause).toBe("time_ceiling"); + }); + + it("never denies anything on a turn without a task time limit", async () => { + const user = new AbortController(); + const { agent, denied } = setup(); + const turn = agent.runTurn( + createEmptySessionState({ id: "s-chat", workingDir }), + options({ signal: user.signal }), + ); + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(denied).toEqual([]); + user.abort(); + await turn.catch(() => undefined); + expect(denied).toEqual([]); + }); +}); diff --git a/src/agent/turn/approval-deadline.test.ts b/src/agent/turn/approval-deadline.test.ts index e7f8898b9..5a511dc97 100644 --- a/src/agent/turn/approval-deadline.test.ts +++ b/src/agent/turn/approval-deadline.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { + MAX_TIMER_MS, APPROVAL_DEADLINE_REPEAT_MS, APPROVAL_TIME_LIMIT_REASON, armApprovalDeadline, @@ -55,4 +56,25 @@ describe("armApprovalDeadline (ATO-1367)", () => { vi.advanceTimersByTime(1e9); expect(deny).not.toHaveBeenCalled(); }); + + it("does not fire early on a limit longer than setTimeout allows (real timers)", async () => { + const deny = vi.fn(); + const dispose = armApprovalDeadline({ msLeft: 2 ** 31, deny }); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(deny).not.toHaveBeenCalled(); + dispose(); + }); + + it("splits a very long limit into hops and fires at the real deadline", () => { + vi.useFakeTimers(); + const deny = vi.fn(); + const dispose = armApprovalDeadline({ msLeft: MAX_TIMER_MS + 1_000, deny }); + vi.advanceTimersByTime(MAX_TIMER_MS); + expect(deny).not.toHaveBeenCalled(); + vi.advanceTimersByTime(999); + expect(deny).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1); + expect(deny).toHaveBeenCalledTimes(1); + dispose(); + }); }); diff --git a/src/agent/turn/approval-deadline.ts b/src/agent/turn/approval-deadline.ts index 393e1ca8d..b9c845665 100644 --- a/src/agent/turn/approval-deadline.ts +++ b/src/agent/turn/approval-deadline.ts @@ -20,6 +20,13 @@ export const APPROVAL_TIME_LIMIT_REASON = /** How often, once past the limit, a new prompt is denied. */ export const APPROVAL_DEADLINE_REPEAT_MS = 5_000; +/** + * The longest delay `setTimeout` honours. Node runs a larger one after + * 1 ms, which would deny every prompt at once under a limit of more + * than ~24.8 days; such a wait is split into hops of at most this. + */ +export const MAX_TIMER_MS = 2_147_483_647; + export interface ApprovalDeadlineTimers { setTimeout(fn: () => void, ms: number): unknown; clearTimeout(handle: unknown): void; @@ -51,10 +58,14 @@ export function armApprovalDeadline(input: { msLeft: number; deny: (reason: string) => void; timers?: ApprovalDeadlineTimers; + now?: () => number; }): () => void { const timers = input.timers ?? realTimers; + const now = input.now ?? Date.now; if (!Number.isFinite(input.msLeft)) return () => undefined; + const deadline = now() + Math.max(0, input.msLeft); let repeat: unknown = null; + let pending: unknown = null; const fire = (): void => { try { input.deny(APPROVAL_TIME_LIMIT_REASON); @@ -62,12 +73,22 @@ export function armApprovalDeadline(input: { // A failing gate must not take the step down with it. } }; - const first = timers.setTimeout(() => { - fire(); - repeat = timers.setInterval(fire, APPROVAL_DEADLINE_REPEAT_MS); - }, Math.max(0, input.msLeft)); + const arm = (): void => { + const left = deadline - now(); + if (left > MAX_TIMER_MS) { + // Not there yet: wait one full hop and look again. + pending = timers.setTimeout(arm, MAX_TIMER_MS); + return; + } + pending = timers.setTimeout(() => { + pending = null; + fire(); + repeat = timers.setInterval(fire, APPROVAL_DEADLINE_REPEAT_MS); + }, Math.max(0, left)); + }; + arm(); return () => { - timers.clearTimeout(first); + if (pending !== null) timers.clearTimeout(pending); if (repeat !== null) timers.clearInterval(repeat); }; } From 02d61a839c6b380f44f53af632b4125c181d1eba Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:49:17 +0300 Subject: [PATCH 051/155] fix(openrouter): no default effort when the request turns thinking off The risk reviewer and context compaction send disableThinking with a small output cap. Adding the Fable medium default there would turn thinking on inside that cap. --- src/llm/provider/openai/openai-build-body.test.ts | 7 ++++++- src/llm/provider/openai/openai-build-body.ts | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/llm/provider/openai/openai-build-body.test.ts b/src/llm/provider/openai/openai-build-body.test.ts index c26c4f018..9b9bfe32a 100644 --- a/src/llm/provider/openai/openai-build-body.test.ts +++ b/src/llm/provider/openai/openai-build-body.test.ts @@ -585,7 +585,12 @@ describe("buildOpenAiChatBody — per-model parameters", () => { const direct = buildOpenAiChatBody({ prompt: "hi" }, "claude-fable-5-2", false, undefined, undefined, undefined, undefined, { providerKind: "openai-compatible", }); - expect(direct).not.toHaveProperty("reasoning_effort"); + expect(direct).not.toHaveProperty("reasoning_effort"); // A request that turned thinking off (the risk reviewer) gets no default. + const reviewer = buildOpenAiChatBody( + { prompt: "x", disableThinking: true, maxTokens: 384 }, + fable, false, undefined, undefined, undefined, undefined, or, + ); + expect(reviewer).not.toHaveProperty("reasoning"); }); }); diff --git a/src/llm/provider/openai/openai-build-body.ts b/src/llm/provider/openai/openai-build-body.ts index 07068bd03..0ade62314 100644 --- a/src/llm/provider/openai/openai-build-body.ts +++ b/src/llm/provider/openai/openai-build-body.ts @@ -157,9 +157,12 @@ export function buildOpenAiChatBody( // that asked for none gets the model's own default where the service's // overspends (Fable on OpenRouter, ATO-1368); `extraBody` and model // params are merged later and still win. + // A request that asked for no thinking at all (the risk reviewer, context + // compaction) never gets the default: it would turn thinking on inside + // a small output cap. const effort = filtered.reasoningEffort ?? - (options.providerKind === "openrouter" + (options.providerKind === "openrouter" && !filtered.disableThinking ? defaultReasoningEffort(defaultChatModel) : undefined); if (effort !== undefined) { From 7514a99105026896fabe0b4cfae876b385305fcf Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:49:20 +0300 Subject: [PATCH 052/155] test(fusion): cover the spend readout wiring Move the worker-session forward into forwardWorkerUsage so it is tested, and test that escalation and judge calls name the delegating session and are recorded on it. --- src/runtime/bootstrap.ts | 8 ++--- .../composition/runtime-tool-catalog.test.ts | 30 +++++++++++++++++++ src/session/usage-meter.test.ts | 13 ++++++++ src/session/usage-meter.ts | 19 ++++++++++++ src/tools/fusion/judge.test.ts | 16 ++++++++++ 5 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index a7a1323a3..06fc88c5b 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -61,7 +61,7 @@ import { import { resolveLlmConfig } from "../llm/provider/index.js"; import { CostAccumulator } from "../llm/provider/cost-accumulator.js"; -import { SessionUsageLedger } from "../session/usage-meter.js"; +import { SessionUsageLedger, forwardWorkerUsage } from "../session/usage-meter.js"; import { ProviderFallbackChain } from "../llm/fallback/index.js"; import { createFallbackChainResolver } from "./fallback-chain-resolver.js"; @@ -753,11 +753,7 @@ export async function createAgentRuntime( const { createSession, createEphemeralSession: createWorkerSession } = createRuntimeSessionFactories(workingDir, sessionStore, ensureRecorder); // A fusion worker's calls count on the session that spawned it, so the // composer's `$` readout includes what the workers spent (ATO-1388). - const createEphemeralSession: typeof createWorkerSession = (meta, inheritedDir) => { - const worker = createWorkerSession(meta, inheritedDir); - sessionUsage.forward(worker.id, meta.parentSessionId); - return worker; - }; + const createEphemeralSession: typeof createWorkerSession = forwardWorkerUsage(createWorkerSession, sessionUsage); const previewPrompt = createRuntimePromptPreview(config, promptPreviewDeps); const { executeTurn, steer, runTurn, controlGoal } = createRuntimeTurnService(config, { diff --git a/src/runtime/composition/runtime-tool-catalog.test.ts b/src/runtime/composition/runtime-tool-catalog.test.ts index 11594fe09..ee6f3ae85 100644 --- a/src/runtime/composition/runtime-tool-catalog.test.ts +++ b/src/runtime/composition/runtime-tool-catalog.test.ts @@ -310,6 +310,36 @@ describe("runtime tool assembly phases and live MCP catalog", () => { }); }); +describe("fusion escalation calls in the session readout (ATO-1388)", () => { + it("records an escalation or judge call on the session that delegated, and nothing without one", async () => { + const directory = await mkdtemp(join(tmpdir(), "catalog-usage-")); + try { + const config = getConfig(); const toolRegistry = new ToolRegistry(); + const approvals = new ApprovalGate({ emit: () => {}, level: 5 }); + const builder = vi.spyOn(fusionTools, "buildFusionDelegateTool"); + const record = vi.fn(); + const usage = { promptTokens: 1_000, completionTokens: 200, totalTokens: 1_200 }; + const provider = { complete: vi.fn(async () => ({ content: "advice", usage, modelId: "big" })) }; + registerRuntimeFusionAndReadScope({ config, toolRegistry, approvals, dangerous: { approvals, approvalRequired: false }, declaredInputs: new osTools.DeclaredInputsRegistry(), + resolveOriginalRequest: () => undefined, runTurn: async () => externalOperation(), createEphemeralSession: () => createEmptySessionState({ id: "worker", workingDir: directory }), + prepareLocalLink: async () => {}, emitAgentLoopEventFor: () => {}, resolveCurrentRunMode: () => resolveRunMode(resolveLlmConfig(config)), + providerRegistry: { getProvider: () => provider as never }, llama: { measuredTokensPerSecond: () => null, fetchSlots: async () => ({ slots: [] }), probeHealth: async () => "answered" as const }, + slotManager: { poolSize: () => 2 }, workingDir: directory, logger, sessionUsage: { record } }); + const wired = builder.mock.calls.at(-1)?.[0]; + if (!wired?.escalationComplete) throw new Error("escalation seam missing"); + const signal = new AbortController().signal; + await wired.escalationComplete({ providerId: "cloud", prompt: "p", maxOutputTokens: 100, signal, sessionId: "s-1" }); + expect(record).toHaveBeenCalledTimes(1); + expect(record).toHaveBeenCalledWith("s-1", expect.objectContaining({ providerId: "cloud", modelId: "big", usage, local: false })); + await wired.escalationComplete({ providerId: "cloud", prompt: "p", maxOutputTokens: 100, signal }); + expect(record).toHaveBeenCalledTimes(1); + } finally { + vi.restoreAllMocks(); + await rm(directory, { recursive: true, force: true }); + } + }); +}); + describe("runtimeKnownSecrets (ATO-1077)", () => { it("lists the env's secret variables and the credentials stored in config", () => { const config = { diff --git a/src/session/usage-meter.test.ts b/src/session/usage-meter.test.ts index dd6c54c64..7d138d6be 100644 --- a/src/session/usage-meter.test.ts +++ b/src/session/usage-meter.test.ts @@ -4,6 +4,7 @@ import { SessionUsageLedger, STALE_READING_MS, activeLimits, + forwardWorkerUsage, formatLimit, formatResetTime, formatSessionCost, @@ -95,6 +96,18 @@ describe("SessionUsageLedger", () => { expect(seen.every((id) => id === "s-parent")).toBe(true); }); + it("links every worker the wrapped factory mints to its parent (ATO-1388)", () => { + const ledger = new SessionUsageLedger({ now: () => NOW }); + const create = forwardWorkerUsage( + (meta: { parentSessionId: string; taskId: string }, dir?: string) => ({ id: `s-w-${meta.taskId}`, dir }), + ledger, + ); + expect(create({ parentSessionId: "s-parent", taskId: "t1" }, "/repo")).toEqual({ id: "s-w-t1", dir: "/repo" }); + ledger.record("s-w-t1", { providerId: "openrouter", modelId: "m", usage, model: priced, local: false }); + expect(ledger.get("s-parent")?.costUsd).toBeCloseTo(4.5); + expect(ledger.get("s-w-t1")).toBeUndefined(); + }); + it("continues from what a reopened session persisted, once", () => { const ledger = new SessionUsageLedger({ now: () => NOW }); const stored: SessionUsageState = { ...EMPTY_SESSION_USAGE, costUsd: 1, pricedCalls: 1, costSources: ["catalog"] }; diff --git a/src/session/usage-meter.ts b/src/session/usage-meter.ts index 3b96d83b0..699327b6e 100644 --- a/src/session/usage-meter.ts +++ b/src/session/usage-meter.ts @@ -259,6 +259,25 @@ export class SessionUsageLedger { } } +/** + * Wrap the runtime's worker-session factory so every worker it mints + * counts its calls on the session that spawned it (ATO-1388). + */ +export function forwardWorkerUsage< + M extends { parentSessionId: string }, + S extends { id: string }, + D = string | undefined, +>( + create: (meta: M, dir?: D) => S, + ledger: Pick, +): (meta: M, dir?: D) => S { + return (meta, dir) => { + const worker = create(meta, dir); + ledger.forward(worker.id, meta.parentSessionId); + return worker; + }; +} + /* ---------------- formatting ---------------- */ /** diff --git a/src/tools/fusion/judge.test.ts b/src/tools/fusion/judge.test.ts index a4f67f758..755703be1 100644 --- a/src/tools/fusion/judge.test.ts +++ b/src/tools/fusion/judge.test.ts @@ -215,6 +215,22 @@ describe("a node closes only on a green list (ATO-944)", () => { rmSync(dir, { recursive: true, force: true }); }); + it("names the delegating session on every judge call, for its spend readout (ATO-1388)", async () => { + const dir = mkdtempSync(join(tmpdir(), "judge-sid-")); + const sessions: Array = []; + const d = deps(dir, async (call) => { + sessions.push(call.sessionId); + return { content: JSON.stringify({ items: [{ id: "a2", pass: true, why: "ok" }] }) }; + }); + await buildFusionDelegateTool(d).run( + { tasks: TASK }, + { workingDir: dir, sessionId: "s-p", stepIndex: 0, signal: new AbortController().signal }, + ); + expect(sessions.length).toBeGreaterThan(0); + expect(sessions.every((id) => id === "s-p")).toBe(true); + rmSync(dir, { recursive: true, force: true }); + }); + it("closes a task the retry fixed", async () => { const dir = mkdtempSync(join(tmpdir(), "judge-g-")); let judged = 0; From 4ce2f34a9e93daac17ee665e116a016c0f240daf Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:49:31 +0300 Subject: [PATCH 053/155] docs(overnight): describe exit 2 for a run a limit stopped ATO-1323 --- docs/user/overnight.md | 2 +- src/cli/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user/overnight.md b/docs/user/overnight.md index 3d3c2bc7b..dfbb45ac2 100644 --- a/docs/user/overnight.md +++ b/docs/user/overnight.md @@ -22,7 +22,7 @@ atomic-agent overnight "migrate the tests to vitest" --budget 5 --hours 8 --dir `atag overnight` is the same command. -In the foreground you get one line per step, for example `[step 12 | 14m | $0.42 of $5.00] read src/app.ts`, and a summary at the end. Ctrl+C cancels the run; a second Ctrl+C quits. The exit code is 0 only when the task completed. +In the foreground you get one line per step, for example `[step 12 | 14m | $0.42 of $5.00] read src/app.ts`, and a summary at the end. Ctrl+C cancels the run; a second Ctrl+C quits. The summary says `overnight completed` when the agent finished on its own and `overnight stopped ()` when a limit ended the run. The exit code is 0 when the agent finished, 2 when it stopped at the budget, time or step limit, made no progress, or the provider ran out of credit (including a resume that found the budget or time already used), and 1 when the run failed or was cancelled. With `--detach`, check on the run with `atomic-agent task show ` and stop it with `atomic-agent task cancel `. diff --git a/src/cli/README.md b/src/cli/README.md index 72ce2d63a..901cbad93 100644 --- a/src/cli/README.md +++ b/src/cli/README.md @@ -19,7 +19,7 @@ On a Windows console, startup switches the code page to UTF-8 (65001) so non-ASC `data backup|restore|move|reset|path` (ATO-579, [data-command.ts](data-command.ts), [data-archive.ts](data-archive.ts)) acts on the state dir the terminal and the desktop share. `backup ` writes an owner-only archive with a manifest; each `*.sqlite` goes in as a snapshot through SQLite's backup API, and `models/` (unless `--with-models`), `browser-profile/` (cookies), WAL/SHM, locks and runtime records stay out. `restore --yes` unpacks into a sibling folder, refuses an archive without the manifest, carries this machine's `models/` over when the backup has none, then swaps folders and keeps the old one as `.before-restore-