From 052fc084517c1a6fd8cfbecdf08c05b635358a32 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 21:16:02 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=96=20fix:=20activate=20durable=20comp?= =?UTF-8?q?action=20cancellation=20across=20turn=20admission?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ --- .../agentSession.admissionGates.test.ts | 4 + .../agentSession.agentSkillSnapshot.test.ts | 101 +- .../agentSession.autoCompaction.test.ts | 56 +- ...gentSession.compactionCancellation.test.ts | 1620 +++++++++++++++++ .../agentSession.continuousCompaction.test.ts | 18 +- .../services/agentSession.disposeRace.test.ts | 47 +- .../agentSession.editMessageId.test.ts | 36 +- .../agentSession.mcpPromptSnapshot.test.ts | 15 +- .../agentSession.pinnedBudget.test.ts | 22 +- .../agentSession.preTurnMessages.test.ts | 14 +- .../agentSession.preparationAdmission.test.ts | 20 +- .../agentSession.preparedHistory.test.ts | 92 +- .../agentSession.queueDispatch.test.ts | 497 ++++- .../agentSession.scopedLifetimes.test.ts | 82 +- .../agentSession.startupAutoRetry.test.ts | 23 +- .../services/agentSession.tokenBudget.test.ts | 205 ++- src/node/services/agentSession.ts | 848 ++++++++- .../agentSession.turnCompletion.test.ts | 159 +- src/node/services/aiService.test.ts | 112 ++ src/node/services/aiService.ts | 6 + .../compactionCancellation.storage.test.ts | 596 +++++- .../services/compactionCancellation.test.ts | 106 +- src/node/services/compactionCancellation.ts | 154 +- .../compactionHandler.preparation.test.ts | 32 +- src/node/services/compactionHandler.ts | 39 +- ...yService.compactionFollowUpCleanup.test.ts | 75 + .../historyService.replacement.test.ts | 57 + .../historyService.truncation.test.ts | 189 +- src/node/services/historyService.ts | 309 +++- src/node/services/messageQueue.test.ts | 130 ++ src/node/services/messageQueue.ts | 138 +- src/node/services/streamManager.test.ts | 281 ++- src/node/services/streamManager.ts | 53 +- src/node/services/taskService.test.ts | 55 +- src/node/services/turnRequestBuilder.ts | 3 + src/node/services/workspaceService.test.ts | 533 +++++- src/node/services/workspaceService.ts | 133 +- tests/ipc/streamCollector.ts | 47 + .../ipc/streaming/stopAdmission.mock.test.ts | 228 +++ .../ipc/streaming/streamErrorRecovery.test.ts | 47 +- 40 files changed, 6594 insertions(+), 588 deletions(-) create mode 100644 src/node/services/agentSession.compactionCancellation.test.ts create mode 100644 tests/ipc/streaming/stopAdmission.mock.test.ts diff --git a/src/node/services/agentSession.admissionGates.test.ts b/src/node/services/agentSession.admissionGates.test.ts index 523f113e27d..1ca18a6e411 100644 --- a/src/node/services/agentSession.admissionGates.test.ts +++ b/src/node/services/agentSession.admissionGates.test.ts @@ -74,6 +74,7 @@ describe("AgentSession.sendMessage (admission gates)", () => { "family trigger", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, preTurnMessages: [ createMuxMessage("family-payload-stale", "assistant", "untrusted payload", { @@ -110,6 +111,7 @@ describe("AgentSession.sendMessage (admission gates)", () => { "peer trigger", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, preTurnMessages: [ createMuxMessage("peer-payload-stale", "assistant", "untrusted payload", { @@ -152,6 +154,7 @@ describe("AgentSession.sendMessage (admission gates)", () => { "peer trigger", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, preTurnMessages: [ createMuxMessage("peer-payload-stuck", "assistant", "untrusted payload", { @@ -196,6 +199,7 @@ describe("AgentSession.sendMessage (admission gates)", () => { "hello", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, onAccepted: () => { acceptedCalls += 1; diff --git a/src/node/services/agentSession.agentSkillSnapshot.test.ts b/src/node/services/agentSession.agentSkillSnapshot.test.ts index 061a1c73094..3986f88ea9e 100644 --- a/src/node/services/agentSession.agentSkillSnapshot.test.ts +++ b/src/node/services/agentSession.agentSkillSnapshot.test.ts @@ -71,15 +71,16 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { historyCleanup = cleanup; const messages: MuxMessage[] = []; - const realAppend = historyService.appendToHistory.bind(historyService); - const appendToHistory = spyOn(historyService, "appendToHistory").mockImplementation( - async (wId: string, message: MuxMessage) => { - messages.push(message); - return realAppend(wId, message); - } - ); + const send = session.sendMessage.bind(session); + spyOn(session, "sendMessage").mockImplementation(async (...args) => { + const result = await send(...args); + const persisted = await historyService.getHistoryFromLatestBoundary(workspaceId); + if (!persisted.success) throw new Error(persisted.error); + messages.splice(0, messages.length, ...persisted.data); + return result; + }); - return { session, appendToHistory, messages, historyService }; + return { session, messages, historyService }; } it("persists a synthetic agent skill snapshot before the user message", async () => { @@ -90,7 +91,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillBody: "Follow this skill.", }); - const { session, appendToHistory, messages } = await createSessionHarness({ + const { session, messages } = await createSessionHarness({ workspaceId, workspacePath, }); @@ -108,7 +109,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const [snapshotMessage, userMessage] = messages; expect(snapshotMessage.role).toBe("user"); @@ -141,7 +142,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { const srcBaseDir = await fs.mkdtemp(path.join(os.tmpdir(), "mux-agent-skill-src-")); - const { session, appendToHistory, messages } = await createSessionHarness({ + const { session, messages } = await createSessionHarness({ workspaceId, workspacePath: projectPath, runtimeConfig: { type: "worktree", srcBaseDir }, @@ -161,7 +162,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const [snapshotMessage] = messages; const snapshotText = snapshotMessage.parts.find((p) => p.type === "text")?.text; @@ -197,7 +198,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { "utf-8" ); - const { session, appendToHistory, messages } = await createSessionHarness({ + const { session, messages } = await createSessionHarness({ workspaceId, workspacePath: subprojectPath, aiServiceOverrides: { @@ -227,7 +228,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const [snapshotMessage] = messages; expect(snapshotMessage.metadata?.agentSkillSnapshot?.skillName).toBe("plugin-skill"); @@ -243,7 +244,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillBody: "Follow this skill.", }); - const { session, appendToHistory } = await createSessionHarness({ + const { session, messages } = await createSessionHarness({ workspaceId, workspacePath, }); @@ -261,7 +262,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { const first = await session.sendMessage("do X", baseOptions); expect(first.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const second = await session.sendMessage("do Y", { ...baseOptions, @@ -273,9 +274,9 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { expect(second.success).toBe(true); // First send: snapshot + user. Second send: user only. - expect(appendToHistory.mock.calls).toHaveLength(3); + expect(messages).toHaveLength(3); - const appendedIds = appendToHistory.mock.calls.map((call) => call[1].id); + const appendedIds = messages.map((message) => message.id); const secondSendAppendedIds = appendedIds.slice(2); expect(secondSendAppendedIds).toHaveLength(1); expect(secondSendAppendedIds[0]).toStartWith("user-"); @@ -292,7 +293,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillBody, }); - const { session, appendToHistory, messages } = await createSessionHarness({ + const { session, messages } = await createSessionHarness({ workspaceId, workspacePath, }); @@ -310,7 +311,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { const first = await session.sendMessage("do X", baseOptions); expect(first.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const firstSnapshot = messages[0]; expect(firstSnapshot.id).toStartWith("agent-skill-snapshot-"); @@ -337,7 +338,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { expect(second.success).toBe(true); // Second send should persist a new snapshot (frontmatter differs) + user message. - expect(appendToHistory.mock.calls).toHaveLength(4); + expect(messages).toHaveLength(4); const secondSnapshot = messages[2]; expect(secondSnapshot.id).toStartWith("agent-skill-snapshot-"); @@ -360,7 +361,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { { skillName: "beta-skill", skillBody: "Follow beta." }, ], }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("do X", { model: "anthropic:claude-3-5-sonnet-latest", @@ -375,7 +376,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(3); + expect(messages).toHaveLength(3); const [alphaSnapshot, betaSnapshot, userMessage] = messages; expect(alphaSnapshot.metadata?.synthetic).toBe(true); @@ -395,7 +396,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillName: "test-skill", skillBody: "Follow slash skill.", }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("do X", { model: "anthropic:claude-3-5-sonnet-latest", @@ -410,7 +411,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const [snapshotMessage, userMessage] = messages; expect(snapshotMessage.metadata?.synthetic).toBe(true); @@ -425,7 +426,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillName: "valid-skill", skillBody: "Follow valid skill.", }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("do X", { model: "anthropic:claude-3-5-sonnet-latest", @@ -440,7 +441,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); expect(messages[0].metadata?.agentSkillSnapshot?.skillName).toBe("valid-skill"); expect(getMessageText(messages[0])).toContain("Follow valid skill."); expect(getMessageText(messages[1])).toBe("do X"); @@ -451,7 +452,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillName: "alpha-skill", skillBody: "Follow alpha.", }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("do X", { model: "anthropic:claude-3-5-sonnet-latest", @@ -466,7 +467,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); expect(messages[0].metadata?.agentSkillSnapshot?.skillName).toBe("alpha-skill"); expect(getMessageText(messages[0])).toContain("Follow alpha."); expect(getMessageText(messages[1])).toBe("do X"); @@ -474,7 +475,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { it("still throws when a slash skill name is invalid", async () => { const { workspacePath } = await createTestWorkspaceWithSkills({ skills: [] }); - const { session, appendToHistory } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("do X", { model: "anthropic:claude-3-5-sonnet-latest", @@ -496,12 +497,12 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { throw new Error("Expected invalid slash skill failure to use unknown error shape"); } expect(result.error.raw).toContain("Invalid agent skill name"); - expect(appendToHistory.mock.calls).toHaveLength(0); + expect(messages).toHaveLength(0); }); it("still throws when a slash skill is missing", async () => { const { workspacePath } = await createTestWorkspaceWithSkills({ skills: [] }); - const { session, appendToHistory } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("do X", { model: "anthropic:claude-3-5-sonnet-latest", @@ -515,7 +516,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(result.success).toBe(false); - expect(appendToHistory.mock.calls).toHaveLength(0); + expect(messages).toHaveLength(0); }); it("dedupes against recent history per-skill", async () => { @@ -525,7 +526,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { { skillName: "beta-skill", skillBody: "Follow beta." }, ], }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const first = await session.sendMessage("first", { model: "anthropic:claude-3-5-sonnet-latest", @@ -536,7 +537,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }, }); expect(first.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const second = await session.sendMessage("second", { model: "anthropic:claude-3-5-sonnet-latest", @@ -551,7 +552,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(second.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(4); + expect(messages).toHaveLength(4); expect(messages[2].metadata?.agentSkillSnapshot?.skillName).toBe("beta-skill"); expect(getMessageText(messages[2])).toContain("Follow beta."); expect(getMessageText(messages[3])).toBe("second"); @@ -562,7 +563,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillName: "fix-issue", skillBody: "Fix issue $1 with priority $2.\nSummary: $ARGUMENTS", }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("Using skill fix-issue: 123 high", { model: "anthropic:claude-3-5-sonnet-latest", @@ -577,7 +578,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const [snapshotMessage, userMessage] = messages; expect(getMessageText(snapshotMessage)).toContain( @@ -592,7 +593,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillName: "fix-issue", skillBody: "Fix issue $ARGUMENTS.", }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const baseOptions = { model: "anthropic:claude-3-5-sonnet-latest", @@ -610,7 +611,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }, }); expect(first.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const second = await session.sendMessage("Using skill fix-issue: 456", { ...baseOptions, @@ -626,7 +627,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { // Different arguments produce different substituted bodies, so the sha256 dedupe // must NOT collapse the second snapshot: snapshot + user, snapshot + user. - expect(appendToHistory.mock.calls).toHaveLength(4); + expect(messages).toHaveLength(4); const firstSnapshot = messages[0]; const secondSnapshot = messages[2]; @@ -642,7 +643,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillName: "fix-issue", skillBody: "Fix issue $ARGUMENTS.", }); - const { session, appendToHistory } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const options = { model: "anthropic:claude-3-5-sonnet-latest", @@ -658,12 +659,12 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { const first = await session.sendMessage("Using skill fix-issue: 123", options); expect(first.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); const second = await session.sendMessage("Using skill fix-issue: 123", options); expect(second.success).toBe(true); // Identical substituted body → snapshot deduped; only the user message is appended. - expect(appendToHistory.mock.calls).toHaveLength(3); + expect(messages).toHaveLength(3); }); it("leaves placeholders in inline-referenced skill bodies untouched", async () => { @@ -672,7 +673,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillName: "fix-issue", skillBody, }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("Please follow $fix-issue for 123", { model: "anthropic:claude-3-5-sonnet-latest", @@ -684,7 +685,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); // Inline refs carry no argument concept: the body must stay byte-identical. expect(getMessageText(messages[0])).toContain(skillBody); }); @@ -696,7 +697,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillName: "no-placeholders", skillBody, }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("Using skill no-placeholders: 123 high", { model: "anthropic:claude-3-5-sonnet-latest", @@ -711,7 +712,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); // Fallback: no placeholders → body untouched; the arguments stay visible in the // user message only. expect(getMessageText(messages[0])).toContain(skillBody); @@ -722,7 +723,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { skillName: "fix-issue", skillBody: "Fix issue [$1] with [$ARGUMENTS].", }); - const { session, appendToHistory, messages } = await createSessionHarness({ workspacePath }); + const { session, messages } = await createSessionHarness({ workspacePath }); const result = await session.sendMessage("Use skill fix-issue", { model: "anthropic:claude-3-5-sonnet-latest", @@ -737,7 +738,7 @@ describe("AgentSession.sendMessage (agent skill snapshots)", () => { }); expect(result.success).toBe(true); - expect(appendToHistory.mock.calls).toHaveLength(2); + expect(messages).toHaveLength(2); expect(getMessageText(messages[0])).toContain("Fix issue [] with []."); }); diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 7f94d1a8611..16827ebe860 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -1443,15 +1443,66 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () return predicate(); } - test("applies compaction when synthetic agent-initiated send crosses the threshold", async () => { + test("unresolved Stop refuses automatic guidance before creating a compaction continuation", async () => { + const workspaceId = "ws-unresolved-stop-guidance"; + const fixture = await createGuidanceHarness({ workspaceId }); + try { + expect(await fixture.session.interruptStream()).toEqual(Ok(undefined)); + const accepted = mock(() => undefined); + const result = await fixture.session.sendMessage( + "Guidance that would otherwise force compaction", + { model: "openai:gpt-4o", agentId: "exec" }, + { + acceptanceOrigin: "automatic", + synthetic: true, + agentInitiated: true, + startStreamInBackground: true, + onAccepted: accepted, + } + ); + expect(result.success).toBe(false); + expect(accepted).not.toHaveBeenCalled(); + expect(fixture.streamHistories).toEqual([]); + expect(await fixture.historyService.getHistoryFromLatestBoundary(workspaceId)).toEqual( + Ok([]) + ); + expect(fixture.session.isBusy()).toBe(false); + expect(fixture.session.queuedMessageEntryCount()).toBe(0); + } finally { + await fixture.session.dispose(); + } + }); + + test("applies compaction for fresh automatic guidance without Stop", async () => { const fixture = await createGuidanceHarness({ workspaceId: "ws-auto-compaction-synthetic-guidance", }); + const monitor = (fixture.session as unknown as { compactionMonitor: CompactionMonitor }) + .compactionMonitor; + let firstCheck = true; + spyOn(monitor, "checkBeforeSend").mockImplementation(() => { + const high = firstCheck; + firstCheck = false; + return { + shouldShowWarning: high, + shouldForceCompact: high, + usagePercentage: high ? 95 : 1, + thresholdPercentage: 70, + contextTokens: high ? 95_000 : 1_000, + maxTokens: 100_000, + }; + }); + const result = await fixture.session.sendMessage( "Updated guidance from parent: focus on the failing tests.", { model: "openai:gpt-4o", agentId: "exec" }, - { synthetic: true, agentInitiated: true, startStreamInBackground: true } + { + acceptanceOrigin: "automatic", + synthetic: true, + agentInitiated: true, + startStreamInBackground: true, + } ); expect(result.success).toBe(true); @@ -1481,6 +1532,7 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () history.some( (message) => message.role === "user" && + message.metadata?.muxMetadata?.type !== "compaction-request" && message.parts.some( (part) => part.type === "text" && part.text.includes("focus on the failing tests") ) diff --git a/src/node/services/agentSession.compactionCancellation.test.ts b/src/node/services/agentSession.compactionCancellation.test.ts new file mode 100644 index 00000000000..fd6586bb69d --- /dev/null +++ b/src/node/services/agentSession.compactionCancellation.test.ts @@ -0,0 +1,1620 @@ +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; +import * as fs from "node:fs"; +import * as fileIO from "node:fs/promises"; +import * as path from "node:path"; +import * as atomicWrite from "write-file-atomic"; +import { historyWriteLockPath } from "./workspaceRemoval"; +import assert from "@/common/utils/assert"; +import nodeAssert from "node:assert/strict"; +import { createMuxMessage, type MuxMessage } from "@/common/types/message"; +import { Err, Ok } from "@/common/types/result"; +import { SESSION_HISTORY_MAX_LINE_BYTES } from "@/common/constants/contextBudget"; +import type { FileChangeTracker } from "./utils/fileChangeTracker"; +import type { WorkspaceGoalService } from "./workspaceGoalService"; +import type { TurnCompletion } from "./streamManager"; +import type { CompactionMonitor } from "./compactionMonitor"; +import type { ContinuousCompactor } from "./continuousCompactor"; +import type { TurnCoordinator } from "./turnCoordinator"; +import { HistoryService } from "./historyService"; +import { + FileCompactionCancellationStorage, + CompactionCancellation, + CompactionCancellationReadRefusedError, +} from "./compactionCancellation"; +import { + createAgentSessionHarness, + createStartedTurnHandle, + type AgentSessionHarness, +} from "./agentSession.testHarness"; + +const workspaceId = "cancellation-runtime"; +const options = { model: "openai:gpt-4o", agentId: "exec" }; +const fixtures: AgentSessionHarness[] = []; + +interface Internals { + compactionMonitor: CompactionMonitor; + coordinator: TurnCoordinator; + compactionCancellation: CompactionCancellation; + fileChangeTracker: FileChangeTracker; + continuousCompactor: ContinuousCompactor; + recoverCompaction(): Promise; + interruptForCompaction(): Promise; + compactionRecoveryBlocked(): Promise; + observeCompaction( + ...args: Parameters + ): ReturnType; + dispatchPendingFollowUp(): Promise; + scheduleStartupAutoRetryIfNeeded(): Promise; +} + +async function fixture(workspaceGoalService?: WorkspaceGoalService) { + const h = await createAgentSessionHarness({ workspaceId, workspaceGoalService }); + fixtures.push(h); + const rows = async () => { + const result = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + assert(result.success); + return result.data; + }; + return { + ...h, + stream: spyOn(h.aiService, "streamMessage"), + rows, + state: h.session as unknown as Internals, + storage: h.historyService.getCompactionCancellationStorage(workspaceId), + }; +} + +afterEach(async () => { + mock.restore(); + for (const h of fixtures.splice(0).reverse()) { + await h.session.dispose(); + await h.cleanup(); + } +}); + +describe("compaction cancellation runtime", () => { + test.each([95, 10])( + "automatic input preserves scoped Stop cleanup debt at %s percent usage", + async (usagePercentage) => { + const h = await fixture(); + await h.session.cancelCompaction(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("canceled-summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "old continuation", model: options.model, agentId: "exec" }, + }, + }) + ); + expect(await h.session.isAutomaticSendBlocked()).toBe(false); + const stopped = await h.storage.read(); + expect(stopped?.scope.kind).toBe("summary"); + const before = await h.rows(); + const cleanup = spyOn(h.historyService, "cleanupCompactionFollowUp").mockResolvedValue( + Err("injected cleanup failure") + ); + const failedCleanup = await h.session + .dispatchPendingCompactionFollowUpIfNeeded() + .catch((error: unknown) => String(error)); + expect(failedCleanup).toContain("injected cleanup failure"); + spyOn(h.state.compactionMonitor, "getThreshold").mockReturnValue(0.7); + spyOn(h.state.compactionMonitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: usagePercentage > 70, + shouldForceCompact: usagePercentage > 70, + usagePercentage, + contextTokens: usagePercentage * 1_000, + maxTokens: 100_000, + thresholdPercentage: 70, + }); + expect( + await h.session.sendMessage("fresh automatic input", options, { + acceptanceOrigin: "automatic", + }) + ).toEqual(Ok(undefined)); + const after = await h.rows(); + expect(after[0]).toEqual(before[0]); + expect(after.filter((row) => row.metadata?.compactionBoundary)).toHaveLength(1); + expect(after.some((row) => row.metadata?.muxMetadata?.type === "compaction-request")).toBe( + false + ); + expect( + after.at(-1)?.parts.map((part) => (part.type === "text" ? part.text : part.type)) + ).toEqual(["fresh automatic input"]); + expect(h.stream).toHaveBeenCalledTimes(1); + expect(await h.storage.read()).toEqual(stopped); + + cleanup.mockRestore(); + await h.session.dispose(); + const restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + fixtures.push(restarted); + expect(await restarted.session.dispatchPendingCompactionFollowUpIfNeeded()).toBe(false); + expect(await h.storage.read()).toBeNull(); + expect((await h.rows())[0].metadata?.muxMetadata).not.toHaveProperty("pendingFollowUp"); + } + ); + + test("automatic input still starts legacy compaction without cancellation debt", async () => { + const h = await fixture(); + spyOn(h.state.compactionMonitor, "getThreshold").mockReturnValue(0.7); + spyOn(h.state.compactionMonitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: true, + shouldForceCompact: true, + usagePercentage: 95, + contextTokens: 95_000, + maxTokens: 100_000, + thresholdPercentage: 70, + }); + expect( + await h.session.sendMessage("fresh automatic input", options, { + acceptanceOrigin: "automatic", + }) + ).toEqual(Ok(undefined)); + expect( + (await h.rows()).some((row) => row.metadata?.muxMetadata?.type === "compaction-request") + ).toBe(true); + expect(h.stream).toHaveBeenCalledTimes(1); + }); + + test.each(["send", "resume"] as const)( + "manual %s retains published input but refuses foreign Stop during attachment preparation", + async (intent) => { + const h = await fixture(); + if (intent === "resume") { + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("retry-input", "user", "already published input") + ); + } + const foreign = new CompactionCancellation( + new HistoryService(h.config).getCompactionCancellationStorage(workspaceId) + ); + const detect = h.state.fileChangeTracker.getChangedAttachments.bind( + h.state.fileChangeTracker + ); + spyOn(h.state.fileChangeTracker, "getChangedAttachments").mockImplementationOnce(async () => { + const detected = await detect(); + // This await is after acceptance, inside streamWithHistory. The local session + // receives no interrupt notification from the second backend. + await foreign.cancel(); + return detected; + }); + const result = + intent === "send" + ? await h.session.sendMessage("already published input", options) + : await h.session.resumeStream(options); + expect(result.success).toBe(false); + expect( + (await h.rows()) + .filter((row) => row.role === "user") + .map((row) => row.parts.map((part) => (part.type === "text" ? part.text : part.type))) + ).toEqual([["already published input"]]); + expect(h.stream).not.toHaveBeenCalled(); + expect(await foreign.read()).not.toBeNull(); + } + ); + + test.each([false, true])( + "late compaction completion keeps its original frontier after foreign Stop (retired=%s)", + async (retired) => { + const h = await fixture(); + const completed = Promise.withResolvers(); + h.stream.mockResolvedValueOnce( + Ok({ messageId: "late-summary", completion: completed.promise }) + ); + const observed = spyOn(h.state.coordinator, "consumeCompletion"); + const followUpContent = { + text: "authored continuation", + model: options.model, + agentId: "exec", + fileParts: [ + { + type: "file" as const, + url: "data:text/plain;base64,YXV0aG9yZWQ=", + mediaType: "text/plain", + }, + ], + }; + expect( + await h.session.sendMessage("summarize", { + ...options, + agentId: "compact", + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { followUpContent }, + }, + }) + ).toEqual(Ok(undefined)); + const request = (await h.rows()).find( + (row) => row.metadata?.muxMetadata?.type === "compaction-request" + ); + assert(request); + const original = structuredClone(request); + const policy = observed.mock.results.at(-1); + assert(policy?.type === "return"); + const foreign = new CompactionCancellation( + new HistoryService(h.config).getCompactionCancellationStorage(workspaceId) + ); + await foreign.cancel(); + const stopped = await foreign.read(); + assert(stopped); + if (retired) await foreign.retire(stopped.nonce); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("late-summary", "assistant", "late summary") + ); + completed.resolve({ + status: "completed", + streamEnd: { + type: "stream-end", + workspaceId, + metadata: { model: options.model }, + parts: [{ type: "text", text: "late summary" }], + }, + }); + await policy.value; + const persisted = await h.historyService.getLastMessages(workspaceId, 10); + assert(persisted.success); + expect(persisted.data.find((row) => row.id === request.id)).toEqual(original); + expect( + persisted.data.some( + (row) => + row.metadata?.muxMetadata?.type === "compaction-summary" && + row.metadata.muxMetadata.pendingFollowUp !== undefined + ) + ).toBe(false); + expect(h.stream).toHaveBeenCalledTimes(1); + } + ); + + test.each(["unresolved", "retained", "late foreign"] as const)( + "reset heartbeat preserves history behind %s Stop", + async (kind) => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("before-reset", "user", "keep context") + ); + if (kind === "late foreign") { + const gate = h.session.isAutomaticSendBlocked.bind(h.session); + spyOn(h.session, "isAutomaticSendBlocked").mockImplementationOnce(async () => { + const blocked = await gate(); + await new CompactionCancellation(h.storage).cancel(); + return blocked; + }); + } else expect(await h.session.cancelCompaction(kind === "retained")).toEqual(Ok(undefined)); + const result = await h.session.appendHeartbeatContextResetBoundary({ + boundaryText: "reset context", + pendingFollowUp: { text: "heartbeat", model: options.model, agentId: "exec" }, + }); + expect(result.success).toBe(false); + expect((await h.rows()).map((row) => row.id)).toEqual(["before-reset"]); + expect(h.stream).not.toHaveBeenCalled(); + expect(await h.storage.read()).not.toBeNull(); + } + ); + + test.each([false, true])( + "reset preserves scoped cancellation debt until recovery (restart=%s)", + async (restart) => { + const h = await fixture(); + await h.session.cancelCompaction(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("canceled-summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "old continuation", model: options.model, agentId: "exec" }, + }, + }) + ); + // Ordinary automatic input can be admitted; a reset must keep this debt in the active epoch. + expect(await h.session.isAutomaticSendBlocked()).toBe(false); + const stopped = await h.storage.read(); + expect(stopped?.scope.kind).toBe("summary"); + let session = h.session; + if (restart) { + await session.dispose(); + const fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + fixtures.push(fresh); + session = fresh.session; + } + const reset = { + boundaryText: "reset context", + pendingFollowUp: { text: "fresh heartbeat", model: options.model, agentId: "exec" }, + }; + expect((await session.appendHeartbeatContextResetBoundary(reset)).success).toBe(false); + expect((await h.rows()).map((row) => row.id)).toEqual(["canceled-summary"]); + expect(await h.storage.read()).toEqual(stopped); + expect(await session.dispatchPendingCompactionFollowUpIfNeeded()).toBe(false); + expect(await h.storage.read()).toBeNull(); + expect((await session.appendHeartbeatContextResetBoundary(reset)).success).toBe(true); + } + ); + + test("unresolved Stop refuses fresh automatic input across restart until manual replacement", async () => { + const h = await fixture(); + expect(await h.session.interruptStream()).toEqual(Ok(undefined)); + const stopped = await h.storage.read(); + expect(stopped).toMatchObject({ version: 1, scope: { kind: "unresolved" } }); + await h.session.dispose(); + const fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + fixtures.push(fresh); + const stream = spyOn(fresh.aiService, "streamMessage"); + const accepted = mock(() => undefined); + expect( + ( + await fresh.session.sendMessage("fresh automatic input", options, { + acceptanceOrigin: "automatic", + onAccepted: accepted, + }) + ).success + ).toBe(false); + expect(accepted).not.toHaveBeenCalled(); + expect(stream).not.toHaveBeenCalled(); + expect(await h.rows()).toEqual([]); + expect(await h.storage.read()).toEqual(stopped); + expect(fresh.session.isBusy()).toBe(false); + expect(await fresh.session.sendMessage("manual replacement", options)).toEqual(Ok(undefined)); + expect(stream).toHaveBeenCalledTimes(1); + expect(await h.storage.read()).toBeNull(); + expect( + (await h.rows()).map((row) => + row.parts.map((part) => (part.type === "text" ? part.text : part.type)) + ) + ).toEqual([["manual replacement"]]); + }); + + test("refused queued automatic ownership settles before idle and the manual successor", async () => { + const h = await fixture(); + expect(await h.session.interruptStream()).toEqual(Ok(undefined)); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const started = Promise.withResolvers(); + const accepted = mock(() => undefined); + let released = false; + const owner: Disposable = { + [Symbol.dispose]: () => { + released = true; + }, + }; + const failed = mock(async () => { + entered.resolve(); + await release.promise; + owner[Symbol.dispose](); + }); + h.stream.mockImplementation(() => { + expect(released).toBe(true); + started.resolve(); + return Promise.resolve(Ok(createStartedTurnHandle(h.session.closingSignal))); + }); + h.session.queueMessage("owned automatic wake", options, { + acceptanceOrigin: "automatic", + synthetic: true, + onAccepted: accepted, + onAcceptedPreStreamFailure: failed, + }); + h.session.queueMessage("manual successor", options); + h.session.sendQueuedMessages(); + try { + await entered.promise; + expect(h.session.isBusy()).toBe(true); + expect(released).toBe(false); + expect(h.stream).not.toHaveBeenCalled(); + expect(accepted).not.toHaveBeenCalled(); + expect(await h.rows()).toEqual([]); + release.resolve(); + await started.promise; + await h.session.waitForIdle(); + expect(failed).toHaveBeenCalledTimes(1); + expect(released).toBe(true); + expect(h.session.queuedMessageEntryCount()).toBe(0); + expect(h.stream).toHaveBeenCalledTimes(1); + expect(await h.storage.read()).toBeNull(); + expect( + (await h.rows()).map((row) => + row.parts.map((part) => (part.type === "text" ? part.text : part.type)) + ) + ).toEqual([["manual successor"]]); + } finally { + release.resolve(); + await h.session.waitForIdle(); + } + }); + + test.each([ + ["JSON", "before Stop"], + ["schema", "before Stop"], + ["UTF-8", "before Stop"], + ["JSON", "during settlement"], + ["JSON", "failed cleanup"], + ] as const)( + "Stop preserving a malformed %s partial permits a later manual replacement (%s)", + async (damage, timing) => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("prior", "user", "prior") + ); + const partialPath = path.join(path.dirname(h.storage.path), "partial.json"); + const partial = createMuxMessage("broken-partial", "assistant", "unfinished", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "stopped continuation", model: options.model, agentId: "exec" }, + }, + }); + const contents = + damage === "JSON" + ? Buffer.from("{unfinished partial") + : damage === "schema" + ? Buffer.from(JSON.stringify({ ...partial, parts: null })) + : Buffer.from(JSON.stringify(partial).replace("unfinished", "\ufffd")); + if (damage === "UTF-8") contents[contents.indexOf(Buffer.from("\ufffd"))] = 0xff; + if (timing === "during settlement") { + const neutralize = h.historyService.neutralizeCompactionRecoveryUnderHistoryLock.bind( + h.historyService + ); + const firstCleanup = Promise.withResolvers(); + spyOn( + h.historyService, + "neutralizeCompactionRecoveryUnderHistoryLock" + ).mockImplementationOnce(async (...args) => { + const result = await neutralize(...args); + firstCleanup.resolve(); + return result; + }); + spyOn(h.aiService, "stopStream").mockImplementationOnce(async () => { + await firstCleanup.promise; + await fileIO.writeFile(partialPath, contents); + return Ok(undefined); + }); + } else await fileIO.writeFile(partialPath, contents); + if (timing === "failed cleanup") { + const remove = fs.rmSync; + let failed = false; + spyOn(fs, "rmSync").mockImplementation((file, removeOptions) => { + if (file === partialPath && !failed) { + failed = true; + throw new Error("partial cleanup unavailable"); + } + return remove(file, removeOptions); + }); + } + + // ACP/budget Stop preserves partials; corrupt bytes must not become permanent cleanup debt. + expect(await h.session.interruptStream()).toEqual( + timing === "failed cleanup" + ? { success: false, error: "partial cleanup unavailable", streamStopped: true } + : Ok(undefined) + ); + expect(h.state.compactionCancellation.blocksRecovery).toBe(timing === "failed cleanup"); + const cancellation = await h.storage.read(); + if (timing === "failed cleanup") expect(cancellation).toBeNull(); + else assert(cancellation); + expect(h.stream).not.toHaveBeenCalled(); + expect( + (await h.session.sendMessage("invalid replacement", { ...options, model: "invalid" })) + .success + ).toBe(false); + expect(await new HistoryService(h.config).readPartial(workspaceId)).toBeNull(); + await nodeAssert.rejects(fileIO.access(partialPath), { code: "ENOENT" }); + expect(h.state.compactionCancellation.blocksRecovery).toBe(false); + const repaired = await h.storage.read(); + assert(repaired); + if (cancellation) expect(repaired).toEqual(cancellation); + expect(await h.session.sendMessage("accepted replacement", options)).toEqual(Ok(undefined)); + expect(h.stream).toHaveBeenCalledTimes(1); + expect((await h.rows()).map((row) => row.id)).toContain("prior"); + expect((await h.rows()).at(-1)?.metadata?.compactionReplacementNonce).toBe(repaired.nonce); + expect(await h.storage.read()).toBeNull(); + } + ); + + test("Stop clears durable continuation before an older version reads history", async () => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "stopped continuation", model: options.model, agentId: "exec" }, + }, + }) + ); + expect(await h.session.cancelCompaction()).toEqual(Ok(undefined)); + const durable = await new HistoryService(h.config).getLastMessages(workspaceId, 1); + assert(durable.success); + const metadata = durable.data[0].metadata?.muxMetadata; + assert(metadata?.type === "compaction-summary"); + // Older recovery reads this field directly and does not know the cancellation sidecar. + expect(metadata.pendingFollowUp).toBeUndefined(); + expect(h.stream).not.toHaveBeenCalled(); + }); + + test.each(["engine", "policy"] as const)( + "Stop neutralizes a partial published while its %s settles", + async (producer) => { + const h = await fixture(); + const cleared = Promise.withResolvers(); + const producing = Promise.withResolvers(); + const release = Promise.withResolvers(); + const neutralize = h.historyService.neutralizeCompactionRecoveryUnderHistoryLock.bind( + h.historyService + ); + spyOn(h.historyService, "neutralizeCompactionRecoveryUnderHistoryLock").mockImplementation( + async (...args) => { + const result = await neutralize(...args); + cleared.resolve(); + return result; + } + ); + const produce = async () => { + await cleared.promise; + producing.resolve(); + await release.promise; + // A final writer can land after initial Stop publication and neutralization. + return h.historyService.writePartial( + workspaceId, + createMuxMessage("late-partial", "assistant", "summary", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "late continuation", model: options.model, agentId: "exec" }, + }, + }) + ); + }; + if (producer === "engine") spyOn(h.aiService, "stopStream").mockImplementation(produce); + else + spyOn(h.state.coordinator, "captureInterruptSettlement").mockReturnValue( + produce().then(() => undefined) + ); + let finished = false; + const stopping = h.session.interruptStream().then((result) => { + finished = true; + return result; + }); + try { + await producing.promise; + expect(finished).toBe(false); + expect(h.state.compactionCancellation.blocksRecovery).toBe(true); + // Terminal policy must not join the Stop cleanup that is waiting for that policy. + expect(await h.state.dispatchPendingFollowUp()).toBe(false); + } finally { + release.resolve(); + } + expect(await stopping).toEqual(Ok(undefined)); + const partial = await new HistoryService(h.config).readPartial(workspaceId); + const metadata = partial?.metadata?.muxMetadata; + assert(metadata?.type === "compaction-summary"); + expect(metadata.pendingFollowUp).toBeUndefined(); + } + ); + + test.each([1, 2])( + "Stop reports cleanup %s failure after successfully stopping its engine", + async (failAt) => { + const h = await fixture(); + const neutralize = h.historyService.neutralizeCompactionRecoveryUnderHistoryLock.bind( + h.historyService + ); + let calls = 0; + const cleanup = spyOn( + h.historyService, + "neutralizeCompactionRecoveryUnderHistoryLock" + ).mockImplementation(async (...args) => { + if (++calls === failAt) throw new Error("legacy cleanup failed"); + return neutralize(...args); + }); + const stop = spyOn(h.aiService, "stopStream"); + expect(await h.session.interruptStream()).toEqual({ + success: false, + error: "legacy cleanup failed", + streamStopped: true, + }); + expect(stop).toHaveBeenCalledTimes(1); + expect(h.state.compactionCancellation.blocksRecovery).toBe(true); + const record = await h.storage.read(); + if (failAt === 1) expect(record).toBeNull(); + else assert(record); + cleanup.mockRestore(); + expect(await h.state.compactionCancellation.retry()).toBe("applied"); + const retried = await h.storage.read(); + assert(retried); + if (record) expect(retried).toEqual(record); + } + ); + + test("an engine Stop failure keeps its original result even when legacy cleanup also fails", async () => { + const h = await fixture(); + spyOn(h.aiService, "stopStream").mockResolvedValueOnce(Err("engine did not stop")); + spyOn(h.historyService, "neutralizeCompactionRecoveryUnderHistoryLock").mockRejectedValueOnce( + new Error("legacy cleanup failed") + ); + expect(await h.session.interruptStream()).toEqual(Err("engine did not stop")); + expect(h.state.compactionCancellation.needsPersistence).toBe(true); + }); + + test.each(["follow-up", "automatic send"] as const)( + "%s cannot join a newer Stop waiting for its captured policy", + async (ingress) => { + const h = await fixture(); + await h.session.cancelCompaction(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "summary", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "old continuation", + model: options.model, + agentId: "exec", + }, + }, + }) + ); + const reading = Promise.withResolvers(); + const releaseHistory = Promise.withResolvers(); + const policySettled = Promise.withResolvers(); + const firstCleanup = Promise.withResolvers(); + const narrowed = Promise.withResolvers(); + const pause = async (result: T) => { + reading.resolve(); + await releaseHistory.promise; + return result; + }; + if (ingress === "follow-up") { + const read = h.historyService.getLastMessages.bind(h.historyService); + spyOn(h.historyService, "getLastMessages").mockImplementationOnce(async (...args) => + pause(await read(...args)) + ); + } else { + const read = h.historyService.getHistoryFromLatestBoundary.bind(h.historyService); + spyOn(h.historyService, "getHistoryFromLatestBoundary").mockImplementationOnce( + async (...args) => pause(await read(...args)) + ); + } + const narrow = h.state.compactionCancellation.narrow.bind(h.state.compactionCancellation); + spyOn(h.state.compactionCancellation, "narrow").mockImplementation((...args) => { + narrowed.resolve(); + return narrow(...args); + }); + const neutralize = h.historyService.neutralizeCompactionRecoveryUnderHistoryLock.bind( + h.historyService + ); + spyOn(h.historyService, "neutralizeCompactionRecoveryUnderHistoryLock").mockImplementation( + async (...args) => { + const result = await neutralize(...args); + firstCleanup.resolve(); + return result; + } + ); + const dispatch = ( + ingress === "follow-up" + ? h.state.dispatchPendingFollowUp() + : h.session.isAutomaticSendBlocked() + ).finally(() => policySettled.resolve()); + await reading.promise; + spyOn(h.state.coordinator, "captureInterruptSettlement").mockReturnValue( + policySettled.promise + ); + const stopping = h.session.interruptStream(); + try { + await firstCleanup.promise; + releaseHistory.resolve(); + // Detect the cyclic join directly, so the red test can release its fixture without a timeout. + expect( + await Promise.race([ + dispatch.then(() => "settled"), + narrowed.promise.then(() => "joined pending Stop"), + ]) + ).toBe("settled"); + expect(await dispatch).toBe(ingress === "automatic send"); + expect(await stopping).toEqual(Ok(undefined)); + expect(h.stream).not.toHaveBeenCalled(); + } finally { + releaseHistory.resolve(); + policySettled.resolve(); + await Promise.all([dispatch, stopping]); + } + } + ); + + for (const stale of ["Stop", "caller epoch"] as const) { + test.each(["single", "batch"] as const)( + `${stale} during automatic %s append removes only stale rows`, + async (kind) => { + const h = await fixture(); + const chatPath = path.join(h.config.sessionsDir, workspaceId, "chat.jsonl"); + let stopping: ReturnType | undefined; + let foreign: ReturnType | undefined; + let epochStale = false; + const supersede = () => { + if (foreign) return; + if (stale === "Stop") stopping = h.session.cancelCompaction(); + else epochStale = true; + foreign = h.historyService.appendToHistory( + workspaceId, + createMuxMessage("foreign", "assistant", "new unrelated row") + ); + }; + if (kind === "batch") { + const atomic = atomicWrite.default; + spyOn(atomicWrite, "default").mockImplementation( + new Proxy(atomic, { + async apply(target, receiver, args: Parameters) { + const result = await Reflect.apply(target, receiver, args); + if (args[0] === chatPath) supersede(); + return result; + }, + }) + ); + } else { + const append = fileIO.appendFile; + spyOn(fileIO, "appendFile").mockImplementation( + async (...args: Parameters) => { + await append(...args); + if (args[0] === chatPath) supersede(); + } + ); + } + const result = await h.session.sendMessage("stale automatic input", options, { + acceptanceOrigin: "automatic", + admissionEpochStale: () => epochStale, + ...(kind === "batch" + ? { + preTurnMessages: [ + createMuxMessage("payload", "assistant", "stale payload", { synthetic: true }), + ], + } + : {}), + }); + expect(foreign).toBeDefined(); + await stopping; + await foreign; + expect(result.success).toBe(false); + expect(h.stream).not.toHaveBeenCalled(); + expect((await h.rows()).map((row) => row.id)).toEqual(["foreign"]); + } + ); + } + + for (const stale of ["Stop", "lease"] as const) { + test.each([false, true])( + `${stale} during edit staging preserves active and archived history (archive=%s)`, + async (archived) => { + const h = await fixture(); + for (const [id, role] of [ + ["prior", "user"], + ["edit", "user"], + ["tail", "assistant"], + ] as const) + await h.historyService.appendToHistory(workspaceId, createMuxMessage(id, role, id)); + if (archived) + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("boundary", "assistant", "summary", { + compactionBoundary: true, + compactionEpoch: 1, + compacted: "user", + }) + ); + const chatPath = path.join(h.config.sessionsDir, workspaceId, "chat.jsonl"); + const archivePath = path.join(h.config.sessionsDir, workspaceId, "chat-archive.jsonl"); + const lockPath = historyWriteLockPath(h.config.rootDir, workspaceId); + let expectedChat = await fileIO.readFile(chatPath); + let expectedArchive = archived ? await fileIO.readFile(archivePath) : undefined; + const atomic = atomicWrite.default; + let intervened = false; + let stopping: ReturnType | undefined; + spyOn(atomicWrite, "default").mockImplementation( + new Proxy(atomic, { + async apply(target, receiver, args: Parameters) { + const result = await Reflect.apply(target, receiver, args); + if (String(args[0]).startsWith(`${chatPath}.publication-`) && !intervened) { + intervened = true; + if (stale === "Stop") stopping = h.session.cancelCompaction(); + else { + await fileIO.writeFile(lockPath, "foreign-owner"); + expectedChat = Buffer.from("foreign active history\n"); + await fileIO.writeFile(chatPath, expectedChat); + if (archived) { + expectedArchive = Buffer.from("foreign archive history\n"); + await fileIO.writeFile(archivePath, expectedArchive); + } + } + } + return result; + }, + }) + ); + try { + const result = await h.session.sendMessage("edited input", { + ...options, + editMessageId: "edit", + }); + expect(intervened).toBe(true); + await stopping; + expect(result.success).toBe(false); + expect(h.stream).not.toHaveBeenCalled(); + expect(await fileIO.readFile(chatPath)).toEqual(expectedChat); + if (archived) expect(await fileIO.readFile(archivePath)).toEqual(expectedArchive!); + } finally { + if (stale === "lease" && intervened) await fileIO.rm(lockPath, { force: true }); + } + } + ); + } + + test("manual input recovers an oversized follow-up narrowing debt without releasing Stop early", async () => { + const h = await fixture(); + const pendingFollowUp = { + text: "large continuation ".repeat(SESSION_HISTORY_MAX_LINE_BYTES / 10), + model: options.model, + agentId: "exec", + }; + const summary = createMuxMessage("large-summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { type: "compaction-summary", pendingFollowUp }, + }); + expect((await h.historyService.appendToHistory(workspaceId, summary)).success).toBe(true); + await h.session.cancelCompaction(); + const original = await h.storage.read(); + assert(original); + await nodeAssert.rejects( + h.state.compactionCancellation.narrow(original.nonce, { + id: summary.id, + sequence: summary.metadata?.historySequence, + pendingFollowUp, + }), + CompactionCancellationReadRefusedError + ); + expect(await h.session.isAutomaticSendBlocked()).toBe(true); + expect(await h.state.compactionRecoveryBlocked()).toBe(true); + expect(h.stream).not.toHaveBeenCalled(); + expect( + (await h.session.sendMessage("invalid replacement", { ...options, model: "invalid" })).success + ).toBe(false); + const retained = await h.storage.read(); + expect(retained).toMatchObject({ retainUntilReplacement: true, scope: { kind: "unresolved" } }); + expect(retained?.nonce).not.toBe(original.nonce); + expect(await h.session.isAutomaticSendBlocked()).toBe(true); + expect(await h.session.sendMessage("valid replacement", options)).toEqual(Ok(undefined)); + expect((await h.rows()).at(-1)?.metadata?.compactionReplacementNonce).toBe(retained?.nonce); + expect(await h.storage.read()).toBeNull(); + }); + + test.each(["future", "oversized"] as const)( + "explicit replacement recovers a %s cancellation record without dropping its retention floor", + async (kind) => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("prior", "user", "prior") + ); + const unsupported = + kind === "future" + ? JSON.stringify({ version: 2, nonce: "future", scope: { kind: "unresolved" } }) + : " ".repeat(SESSION_HISTORY_MAX_LINE_BYTES + 1); + await fileIO.writeFile(h.storage.path, unsupported); + expect( + (await h.session.sendMessage("invalid replacement", { ...options, model: "invalid" })) + .success + ).toBe(false); + expect(await h.storage.read()).toMatchObject({ version: 1, retainUntilReplacement: true }); + expect((await h.rows()).map((row) => row.id)).toEqual(["prior"]); + const stop = await h.storage.read(); + expect(await h.session.sendMessage("accepted replacement", options)).toEqual(Ok(undefined)); + expect((await h.rows()).at(-1)?.metadata?.compactionReplacementNonce).toBe(stop?.nonce); + expect(await h.storage.read()).toBeNull(); + } + ); + + test.each(["ordinary", "retained", "foreign Stop"] as const)( + "restart reconciles already-cleared scoped follow-up debt (%s)", + async (kind) => { + const h = await fixture(); + // Seed the pre-repair crash shape: a legacy writer persisted after the Stop fence. + await h.session.cancelCompaction(); + const summary = createMuxMessage("summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "old continuation", model: options.model, agentId: "exec" }, + }, + }); + await h.historyService.appendToHistory(workspaceId, summary); + + const stop = await h.storage.read(); + assert(stop); + const metadata = summary.metadata?.muxMetadata; + assert(metadata?.type === "compaction-summary" && metadata.pendingFollowUp); + await h.state.compactionCancellation.narrow(stop.nonce, { + id: summary.id, + sequence: summary.metadata?.historySequence, + pendingFollowUp: { ...metadata.pendingFollowUp }, + }); + expect( + await h.historyService.cleanupCompactionFollowUp(workspaceId, summary, "clear", () => true) + ).toEqual(Ok("applied")); + // Crash window: cleanup is durable; the ordinary cancellation sidecar has not retired. + // A retained floor must survive the same disk shape, including a later automatic row. + if (kind === "retained") + await fileIO.writeFile( + h.storage.path, + JSON.stringify({ ...(await h.storage.read()), retainUntilReplacement: true }) + ); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("later", "assistant", "later output") + ); + await h.session.dispose(); + const fresh = await createAgentSessionHarness({ + workspaceId, + historyService: new HistoryService(h.config), + config: h.config, + }); + fixtures.push(fresh); + const freshStream = spyOn(fresh.aiService, "streamMessage"); + const state = fresh.session as unknown as Internals; + let successorNonce: string | undefined; + if (kind === "foreign Stop") { + const cleanup = fresh.historyService.cleanupCompactionFollowUp.bind(fresh.historyService); + spyOn(fresh.historyService, "cleanupCompactionFollowUp").mockImplementationOnce( + async (...args) => { + const result = await cleanup(...args); + const foreign = new CompactionCancellation( + new FileCompactionCancellationStorage(new HistoryService(h.config), workspaceId) + ); + await foreign.cancel(); + successorNonce = (await foreign.read())?.nonce; + return result; + } + ); + } + expect(await state.dispatchPendingFollowUp()).toBe(false); + expect(freshStream).not.toHaveBeenCalled(); + expect((await h.storage.read()) !== null).toBe(kind !== "ordinary"); + if (kind === "foreign Stop") { + expect(successorNonce).toBeDefined(); + expect((await h.storage.read())?.nonce).toBe(successorNonce); + expect(successorNonce).not.toBe(stop.nonce); + } + expect(await state.compactionRecoveryBlocked()).toBe(kind !== "ordinary"); + } + ); + + test("manual repair clears stale usage before replacement admission", async () => { + const h = await fixture(); + await h.historyService.appendToHistory(workspaceId, createMuxMessage("prior", "user", "prior")); + await fileIO.writeFile(h.storage.path, "{malformed cancellation"); + const clear = spyOn(h.session, "clearUsageState"); + expect(await h.session.sendMessage("replacement", options)).toEqual(Ok(undefined)); + expect(clear).toHaveBeenCalled(); + expect(await h.storage.read()).toBeNull(); + }); + + test.each([false, true])( + "Stop blocks automatic admission synchronously and disposal joins its pending publication (retained=%s)", + async (retained) => { + const h = await fixture(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const flushing = Promise.withResolvers(); + const mutate = FileCompactionCancellationStorage.prototype.mutate; // eslint-disable-line @typescript-eslint/unbound-method -- invoked with the original receiver below + spyOn(FileCompactionCancellationStorage.prototype, "mutate").mockImplementationOnce( + async function (this: FileCompactionCancellationStorage, ...args) { + entered.resolve(); + await release.promise; + return mutate.call(this, ...args); + } + ); + const cancel = h.session.cancelCompaction(retained); + await entered.promise; + expect( + (await h.session.sendMessage("automatic", options, { acceptanceOrigin: "automatic" })) + .success + ).toBe(false); + const flush = h.state.compactionCancellation.flush.bind(h.state.compactionCancellation); + spyOn(h.state.compactionCancellation, "flush").mockImplementationOnce(() => { + flushing.resolve(); + return flush(); + }); + let disposed = false; + const closing = h.session.dispose().then(() => { + disposed = true; + }); + try { + await flushing.promise; + expect(disposed).toBe(false); + } finally { + release.resolve(); + expect(await cancel).toEqual(Ok(undefined)); + await closing; + } + expect((await h.storage.read())?.retainUntilReplacement === true).toBe(retained); + expect(h.stream).not.toHaveBeenCalled(); + } + ); + + test.each(["recover", "observe"] as const)( + "Stop after the %s gate read cannot start a producer under the new generation", + async (kind) => { + const h = await fixture(); + const gate = h.state.compactionRecoveryBlocked.bind(h.state); + spyOn(h.state, "compactionRecoveryBlocked").mockImplementationOnce(async () => { + const blocked = await gate(); + expect(blocked).toBe(false); + await h.session.cancelCompaction(); + return blocked; + }); + const recover = spyOn(h.state.continuousCompactor, "recover").mockResolvedValue(true); + const observe = spyOn(h.state.continuousCompactor, "observe").mockResolvedValue("applied"); + expect( + kind === "recover" + ? await h.state.recoverCompaction() + : await h.state.observeCompaction(95, { + enabled: true, + model: options.model, + contextWindowTokens: 100_000, + thresholdPercent: 85, + phase: "on-send", + }) + ).toBe(kind === "recover" ? false : "none"); + expect(recover).not.toHaveBeenCalled(); + expect(observe).not.toHaveBeenCalled(); + expect(await h.storage.read()).not.toBeNull(); + } + ); + + test.each([false, true])( + "only manual acceptance retires Stop (synthetic=%s)", + async (synthetic) => { + const h = await fixture(); + expect(await h.session.cancelCompaction(true)).toEqual(Ok(undefined)); + const stop = await h.storage.read(); + assert(stop); + expect( + ( + await h.session.sendMessage("automatic", options, { + acceptanceOrigin: "automatic", + synthetic, + }) + ).success + ).toBe(false); + expect(await h.rows()).toEqual([]); + expect((await h.storage.read())?.nonce).toBe(stop.nonce); + expect(await h.session.sendMessage("manual", options, { synthetic })).toEqual(Ok(undefined)); + expect((await h.rows()).at(-1)?.metadata?.compactionReplacementNonce).toBe(stop.nonce); + expect(await h.storage.read()).toBeNull(); + } + ); + + test("Stop while legacy compaction waits for the stream cannot launch its old compaction request", async () => { + const h = await fixture(); + const completion = Promise.withResolvers(); + h.stream.mockResolvedValueOnce(Ok({ messageId: "original", completion: completion.promise })); + expect(await h.session.sendMessage("original input", options)).toEqual(Ok(undefined)); + const before = await h.rows(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(h.aiService, "stopStream").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + completion.resolve({ status: "aborted", abortReason: "system" }); + return Ok(undefined); + }); + const compact = h.state.interruptForCompaction(); + try { + await entered.promise; + await h.session.cancelCompaction(); + release.resolve(); + await compact; + expect(h.stream).toHaveBeenCalledTimes(1); + expect(await h.rows()).toEqual(before); + expect(await h.storage.read()).not.toBeNull(); + } finally { + completion.resolve({ status: "aborted", abortReason: "system" }); + release.resolve(); + await compact; + } + }); + + test("Stop during follow-up history inspection cannot relabel the old continuation as fresh input", async () => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "old continuation", model: options.model, agentId: "exec" }, + }, + }) + ); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = h.historyService.getLastMessages.bind(h.historyService); + spyOn(h.historyService, "getLastMessages").mockImplementationOnce(async (...args) => { + const result = await read(...args); + entered.resolve(); + await release.promise; + return result; + }); + const dispatch = h.state.dispatchPendingFollowUp(); + try { + await entered.promise; + await h.session.cancelCompaction(); + release.resolve(); + expect(await dispatch).toBe(false); + expect((await h.rows()).map((row) => row.id)).toEqual(["summary"]); + expect(h.stream).not.toHaveBeenCalled(); + expect(await h.storage.read()).not.toBeNull(); + } finally { + release.resolve(); + await dispatch; + } + }); + + test("fresh automatic input preserves Stop and its hidden canceled summary across restart", async () => { + const h = await fixture(); + // Seed the pre-repair crash shape: a legacy writer persisted after the Stop fence. + await h.session.cancelCompaction(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("canceled-summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "old continuation", model: options.model, agentId: "exec" }, + }, + }) + ); + + const stop = await h.storage.read(); + assert(stop); + expect( + await h.session.sendMessage("fresh attention", options, { + acceptanceOrigin: "automatic", + synthetic: true, + }) + ).toEqual(Ok(undefined)); + expect((await h.rows()).at(-1)?.metadata?.compactionReplacementNonce).toBeUndefined(); + expect(await h.storage.read()).toMatchObject({ + nonce: stop.nonce, + scope: { kind: "summary", id: "canceled-summary", sequence: 0 }, + }); + await h.session.dispose(); + const fresh = await createAgentSessionHarness({ + workspaceId, + historyService: new HistoryService(h.config), + config: h.config, + }); + fixtures.push(fresh); + const freshStream = spyOn(fresh.aiService, "streamMessage"); + const state = fresh.session as unknown as Internals; + expect(await fresh.session.resumeStream(options, { acceptanceOrigin: "automatic" })).toEqual( + Ok({ started: false }) + ); + expect(await state.scheduleStartupAutoRetryIfNeeded()).toBe("completed"); + expect(await state.dispatchPendingFollowUp()).toBe(false); + expect(freshStream).not.toHaveBeenCalled(); + const summary = (await h.rows())[0].metadata?.muxMetadata; + assert(summary?.type === "compaction-summary"); + expect(summary.pendingFollowUp).toBeUndefined(); + expect(await h.storage.read()).toBeNull(); + }); + + test("queued manual input invalidates explicit resume while pricing is suspended", async () => { + const entered = Promise.withResolvers(); + const pricing = Promise.withResolvers>>(); + const h = await fixture({ + assertPricedModelForBudgetedGoal: mock(() => { + entered.resolve(); + return pricing.promise; + }), + } as unknown as WorkspaceGoalService); + await h.historyService.appendToHistory(workspaceId, createMuxMessage("user", "user", "prior")); + await h.session.cancelCompaction(true); + const stop = await h.storage.read(); + const resuming = h.session.resumeStream(options); + await entered.promise; + h.session.queueMessage("new manual", options); + pricing.resolve(Ok(undefined)); + expect(await resuming).toEqual(Ok({ started: false })); + expect((await h.storage.read())?.nonce).toBe(stop?.nonce); + expect((await h.rows())[0].metadata?.compactionReplacementNonce).toBeUndefined(); + expect(h.stream).not.toHaveBeenCalled(); + }); + + test.each(["send", "resume"] as const)( + "Stop during %s capture refuses the old request", + async (kind) => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("user", "user", "prior") + ); + const capture = h.historyService.captureCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "captureCompactionReplacement").mockImplementationOnce( + async (...args) => { + const result = await capture(...args); + await h.session.cancelCompaction(true); + return result; + } + ); + const result = + kind === "send" + ? await h.session.sendMessage("replacement", options) + : await h.session.resumeStream(options); + if (kind === "send") expect(result.success).toBe(false); + else expect(result).toEqual(Ok({ started: false })); + expect(await h.storage.read()).not.toBeNull(); + expect((await h.rows()).map((row) => row.id)).toEqual(["user"]); + expect(h.stream).not.toHaveBeenCalled(); + } + ); + + test("Stop from a queued PREPARING observer cannot be adopted as replacement authority", async () => { + const h = await fixture(); + let cancellation: ReturnType | undefined; + h.session.onChatEvent(({ message }) => { + if (message.type === "stream-lifecycle" && message.phase === "preparing") + cancellation ??= h.session.cancelCompaction(true); + }); + const settled = Promise.withResolvers(); + const send = h.session.sendMessage.bind(h.session); + spyOn(h.session, "sendMessage").mockImplementation(async (...args) => { + try { + return await send(...args); + } finally { + settled.resolve(); + } + }); + h.session.queueMessage("manual", options); + h.session.sendQueuedMessages(); + await settled.promise; + await cancellation; + expect(await h.rows()).toEqual([]); + expect(await h.storage.read()).not.toBeNull(); + expect(h.stream).not.toHaveBeenCalled(); + }); + + test.each([false, true])( + "an edit refreshes its own truncate fence but refuses a newer Stop (%s)", + async (newerStop) => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("user", "user", "prior") + ); + await h.session.cancelCompaction(true); + const stop = await h.storage.read(); + if (newerStop) { + const truncate = h.historyService.truncateAfterMessage.bind(h.historyService); + spyOn(h.historyService, "truncateAfterMessage").mockImplementationOnce(async (...args) => { + const result = await truncate(...args); + await h.session.cancelCompaction(true); + return result; + }); + } + const result = await h.session.sendMessage("edited", { ...options, editMessageId: "user" }); + if (newerStop) { + expect(result.success).toBe(false); + expect((await h.storage.read())?.nonce).not.toBe(stop?.nonce); + expect(h.stream).not.toHaveBeenCalled(); + return; + } + expect(result).toEqual(Ok(undefined)); + expect((await h.rows()).at(-1)?.metadata?.compactionReplacementNonce).toBe(stop?.nonce); + expect(await h.storage.read()).toBeNull(); + } + ); + + test.each(["user", "assistant", "partial"] as const)( + "explicit resume stamps the actual %s tail before notices", + async (kind) => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("user", "user", "prior") + ); + if (kind === "assistant") + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant", "assistant", "answer") + ); + if (kind === "partial") + await h.historyService.writePartial( + workspaceId, + createMuxMessage("partial", "assistant", "interrupted answer", { historySequence: 1 }) + ); + await h.session.cancelCompaction(true); + const stop = await h.storage.read(); + const detect = h.state.fileChangeTracker.getChangedAttachments.bind( + h.state.fileChangeTracker + ); + const notice = spyOn(h.state.fileChangeTracker, "getChangedAttachments").mockImplementation( + async () => { + expect((await h.rows()).at(-1)).toMatchObject({ + id: kind, + metadata: { compactionReplacementNonce: stop?.nonce }, + }); + expect(await h.storage.read()).toBeNull(); + return detect(); + } + ); + expect(await h.session.resumeStream(options)).toEqual(Ok({ started: true })); + expect(notice).toHaveBeenCalledTimes(1); + expect(h.stream).toHaveBeenCalledTimes(1); + } + ); + + test.each(["before truncate", "after truncate"] as const)( + "an edit cannot adopt a foreign reset %s", + async (phase) => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("user", "user", "prior") + ); + const foreign = new HistoryService(h.config); + const reset = async () => { + await foreign.clearHistory(workspaceId); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("foreign-reset", "assistant", "", { contextBoundaryKind: "reset" }) + ); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("foreign", "user", "new context") + ); + }; + const truncate = h.historyService.truncateAfterMessage.bind(h.historyService); + spyOn(h.historyService, "truncateAfterMessage").mockImplementationOnce(async (...args) => { + if (phase === "before truncate") await reset(); + const result = await truncate(...args); + if (phase === "after truncate") await reset(); + return result; + }); + expect( + (await h.session.sendMessage("stale edit", { ...options, editMessageId: "user" })).success + ).toBe(false); + expect((await h.rows()).map((row) => row.id)).toEqual(["foreign-reset", "foreign"]); + expect(h.stream).not.toHaveBeenCalled(); + } + ); + + test("an assistant notice cannot make an earlier rejected request resumable or retire Stop", async () => { + const h = await fixture(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("rejected", "user", "rejected request", { contextBudgetRejected: true }) + ); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("notice", "assistant", "later notice", { synthetic: true }) + ); + await h.session.cancelCompaction(true); + const stop = await h.storage.read(); + const stamp = spyOn(h.historyService, "acceptCompactionReplacement"); + expect(await h.session.resumeStream(options)).toMatchObject({ + success: false, + error: { type: "context_budget_blocked" }, + }); + expect(stamp).not.toHaveBeenCalled(); + expect((await h.storage.read())?.nonce).toBe(stop?.nonce); + expect(h.stream).not.toHaveBeenCalled(); + }); + + test.each([false, true])( + "skipped canceled follow-up cleanup retains Stop until a replacement accepts (fails=%s)", + async (fails) => { + const h = await fixture(); + // Seed the pre-repair crash shape: a legacy writer persisted after the Stop fence. + await h.session.cancelCompaction(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "old follow-up", model: options.model, agentId: "exec" }, + }, + }) + ); + + const stop = await h.storage.read(); + const send = h.session.sendMessage.bind(h.session); + let sending: ReturnType | undefined; + spyOn(h.session, "sendMessage").mockImplementation((...args) => (sending = send(...args))); + const cleanup = h.historyService.cleanupCompactionFollowUp.bind(h.historyService); + const cleanupSpy = spyOn( + h.historyService, + "cleanupCompactionFollowUp" + ).mockImplementationOnce(async (...args) => { + const held = Promise.withResolvers(); + const release = Promise.withResolvers(); + const holding = h.historyService.withCompactionStorageLock(workspaceId, async () => { + held.resolve(); + await release.promise; + }); + await held.promise; + const cleaning = cleanup(...args); + try { + h.session.queueMessage("new manual", fails ? { ...options, model: "invalid" } : options); + h.session.sendQueuedMessages(); + } finally { + release.resolve(); + await holding; + } + const result = await cleaning; + expect(result).toEqual(Ok("skipped")); + return result; + }); + expect(await h.state.dispatchPendingFollowUp()).toBe(false); + assert(sending); + expect((await sending).success).toBe(!fails); + cleanupSpy.mockRestore(); + if (fails) expect((await h.storage.read())?.nonce).toBe(stop?.nonce); + else expect(await h.storage.read()).toBeNull(); + await h.session.dispose(); + const fresh = await createAgentSessionHarness({ + workspaceId, + historyService: new HistoryService(h.config), + config: h.config, + }); + fixtures.push(fresh); + const freshStream = spyOn(fresh.aiService, "streamMessage"); + expect(await (fresh.session as unknown as Internals).dispatchPendingFollowUp()).toBe(false); + expect(freshStream).not.toHaveBeenCalled(); + } + ); + + test.each(["empty", "snapshot", "rejected", "system"] as const)( + "explicit resume refuses %s tail without inventing a trigger", + async (kind) => { + const h = await fixture(); + if (kind !== "empty") { + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("user", "user", "prior") + ); + const tail: MuxMessage = + kind === "system" + ? { id: "tail", role: "system", parts: [{ type: "text", text: "system" }] } + : createMuxMessage( + "tail", + "user", + "ineligible", + kind === "snapshot" + ? { synthetic: true, fileAtMentionSnapshot: ["@input.ts"] } + : { contextBudgetRejected: true } + ); + await h.historyService.appendToHistory(workspaceId, tail); + } + await h.session.cancelCompaction(true); + const stop = await h.storage.read(); + const before = await h.rows(); + const notices = spyOn(h.state.fileChangeTracker, "getChangedAttachments"); + const result = await h.session.resumeStream(options); + if (kind === "empty" || kind === "rejected") expect(result.success).toBe(false); + else expect(result).toEqual(Ok({ started: false })); + expect(await h.rows()).toEqual(before); + expect((await h.storage.read())?.nonce).toBe(stop?.nonce); + expect(notices).not.toHaveBeenCalled(); + expect(h.stream).not.toHaveBeenCalled(); + } + ); + + test("a failed unlink preserves one replacement until cleanup permits an explicit retry", async () => { + const h = await fixture(); + await h.session.cancelCompaction(true); + const stop = await h.storage.read(); + assert(stop); + const unlink = fs.rmSync; + const failedUnlink = spyOn(fs, "rmSync").mockImplementation((file, options) => { + if (file === h.storage.path) throw new Error("injected unlink failure"); + return unlink(file, options); + }); + expect(await h.session.sendMessage("first", options)).toEqual(Ok(undefined)); + await h.session.interruptStream({ preserveCompactionIntent: true }); + const accepted = await h.rows(); + expect(h.stream).toHaveBeenCalledTimes(1); + // Failed ancillary cleanup keeps the first acceptance; it cannot authorize a + // second row stamped with the same Stop nonce. + expect((await h.session.sendMessage("second", options)).success).toBe(false); + expect(await h.rows()).toEqual(accepted); + expect(h.stream).toHaveBeenCalledTimes(1); + const fresh = new HistoryService(h.config); + expect(await fresh.findCompactionReplacementWitness(workspaceId, stop.nonce)).toEqual( + Ok({ nonce: stop.nonce }) + ); + const restartedHistory = await fresh.getHistoryFromLatestBoundary(workspaceId); + expect(restartedHistory).toEqual(Ok(accepted)); + expect( + accepted.filter((row) => row.metadata?.compactionReplacementNonce === stop.nonce) + ).toHaveLength(1); + expect((await h.storage.read())?.nonce).toBe(stop.nonce); + + failedUnlink.mockRestore(); + expect(await h.state.compactionCancellation.retry()).toBe("applied"); + expect(await h.storage.read()).toBeNull(); + expect(await h.state.compactionRecoveryBlocked()).toBe(false); + expect(await h.session.sendMessage("second", options)).toEqual(Ok(undefined)); + expect(h.stream).toHaveBeenCalledTimes(2); + const retried = await h.rows(); + expect( + retried + .filter((row) => row.role === "user") + .map((row) => row.parts.map((part) => (part.type === "text" ? part.text : part.type))) + ).toEqual([["first"], ["second"]]); + expect( + retried.filter((row) => row.metadata?.compactionReplacementNonce === stop.nonce) + ).toHaveLength(1); + expect(retried.find((row) => row.id === accepted[0].id)).toEqual(accepted[0]); + }); + + test.each([false, true])( + "restart blocks recovery and consumes canceled follow-up (retained=%s)", + async (retained) => { + const h = await fixture(); + // Seed the pre-repair crash shape: a legacy writer persisted after the Stop fence. + await h.session.cancelCompaction(retained); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "continue", model: options.model, agentId: "exec" }, + }, + }) + ); + + await h.session.dispose(); + const fresh = await createAgentSessionHarness({ + workspaceId, + historyService: new HistoryService(h.config), + config: h.config, + }); + fixtures.push(fresh); + const freshStream = spyOn(fresh.aiService, "streamMessage"); + const state = fresh.session as unknown as Internals; + const recovery = spyOn(state.continuousCompactor, "recover"); + expect(await state.recoverCompaction()).toBe(false); + expect(await state.scheduleStartupAutoRetryIfNeeded()).toBe("completed"); + expect(await state.dispatchPendingFollowUp()).toBe(false); + expect(recovery).not.toHaveBeenCalled(); + expect(freshStream).not.toHaveBeenCalled(); + const summary = (await h.rows())[0].metadata?.muxMetadata; + assert(summary?.type === "compaction-summary"); + expect(summary.pendingFollowUp).toBeUndefined(); + expect((await h.storage.read()) !== null).toBe(retained); + } + ); +}); diff --git a/src/node/services/agentSession.continuousCompaction.test.ts b/src/node/services/agentSession.continuousCompaction.test.ts index 3046154e48d..aaabe8e4bf1 100644 --- a/src/node/services/agentSession.continuousCompaction.test.ts +++ b/src/node/services/agentSession.continuousCompaction.test.ts @@ -1115,14 +1115,18 @@ describe("AgentSession continuous compaction wiring", () => { return read(...args); }); } else { - const update = h.historyService.cleanupCompactionFollowUp.bind(h.historyService); - spyOn(h.historyService, "cleanupCompactionFollowUp").mockImplementationOnce( - async (...args) => { - entered.resolve(); - await release.promise; - return update(...args); - } + // Hard Stop now owns the durable follow-up clear before completion dispatch. + const update = h.historyService.neutralizeCompactionRecoveryUnderHistoryLock.bind( + h.historyService ); + spyOn( + h.historyService, + "neutralizeCompactionRecoveryUnderHistoryLock" + ).mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); await h.session.interruptStream({ abandonPartial: true }); } return true; diff --git a/src/node/services/agentSession.disposeRace.test.ts b/src/node/services/agentSession.disposeRace.test.ts index fedb9eb83d4..d5350920773 100644 --- a/src/node/services/agentSession.disposeRace.test.ts +++ b/src/node/services/agentSession.disposeRace.test.ts @@ -12,6 +12,7 @@ import type { BackgroundProcessManager } from "./backgroundProcessManager"; import type { Result } from "@/common/types/result"; import { Err, Ok } from "@/common/types/result"; import { createMuxMessage } from "@/common/types/message"; +import * as branchSummary from "./branchSummary"; import { clearPendingBranchSummary, startAbandonedBranchSummaryInBackground, @@ -55,9 +56,6 @@ describe("AgentSession disposal race conditions", () => { const history = await createTestHistoryService(); const historyService = history.historyService; - // Keep the write gate while exercising real history and journal lifecycle methods. - const appendDeferred = createDeferred>(); - spyOn(historyService, "appendToHistory").mockImplementation(() => appendDeferred.promise); const initStateManager: InitStateManager = { on(_eventName: string | symbol, _listener: (...args: unknown[]) => void) { @@ -106,12 +104,12 @@ describe("AgentSession disposal race conditions", () => { expect(inFlight).toBeDefined(); - // Dispose while sendMessage() is awaiting appendToHistory. + // Dispose during queued admission, before a replacement can become durable. session.beginDispose(); - appendDeferred.resolve(Ok(undefined)); const result = await (inFlight as Promise>); - expect(result.success).toBe(true); + expect(result.success).toBe(false); + expect(await historyService.getLastMessages("ws", 1)).toEqual(Ok([])); // We should not attempt to stream once disposal has begun. expect(streamMessage).toHaveBeenCalledTimes(0); @@ -169,6 +167,15 @@ describe("AgentSession disposal race conditions", () => { const workspaceId = "ws-branch-summary-dispose"; const sessionDir = path.join(config.sessionsDir, workspaceId); + const summaryEntered = Promise.withResolvers(); + const awaitSummary = branchSummary.awaitPendingBranchSummary; + const summaryWait = spyOn(branchSummary, "awaitPendingBranchSummary").mockImplementation( + (...args) => { + const pending = awaitSummary(...args); + summaryEntered.resolve(); + return pending; + } + ); try { const session = new AgentSession({ workspaceId, @@ -217,10 +224,14 @@ describe("AgentSession disposal race conditions", () => { model: "anthropic:claude-sonnet-4-5", agentId: "exec", }); - // Let the send reach the pending-summary await: while the gate is closed - // it is the only unresolved promise in the send's path, and nothing may - // have been appended yet — on disk, not in a mock ledger. - await new Promise((resolve) => setTimeout(resolve, 10)); + // Compaction admission now does disk I/O first; signal the actual await instead of + // assuming it has been reached after a timer on a loaded CI runner. + await Promise.race([ + summaryEntered.promise, + sendPromise.then(() => { + throw new Error("send settled before awaiting the branch summary"); + }), + ]); expect(existsSync(nodePath.join(sessionDir, "chat.jsonl"))).toBe(false); // Mirror removeWorkspace: dispose the session, cancel + drain the @@ -247,6 +258,7 @@ describe("AgentSession disposal race conditions", () => { expect(readBack.data).toHaveLength(0); } } finally { + summaryWait.mockRestore(); await cleanup(); } }); @@ -526,11 +538,15 @@ describe("AgentSession disposal race conditions", () => { test("skips handle-less startup-failure recovery when disposal begins mid-startup", async () => { const commitDeferred = createDeferred>(); + const commitEntered = Promise.withResolvers(); const { session, historyService, cleanup } = await createAgentSessionHarness({ workspaceId: "ws-dispose-startup-failure", }); try { - spyOn(historyService, "commitPartial").mockReturnValueOnce(commitDeferred.promise); + spyOn(historyService, "commitPartial").mockImplementationOnce(() => { + commitEntered.resolve(); + return commitDeferred.promise; + }); const errorSink = session as unknown as { handleStreamError: (data: unknown) => Promise; handleStreamFailureForAutoRetry: (failure: unknown) => Promise; @@ -542,8 +558,13 @@ describe("AgentSession disposal race conditions", () => { model: "anthropic:claude-3-5-sonnet-latest", agentId: "exec", }); - // Let the resume park on the pending commitPartial before disposing. - await new Promise((resolve) => setTimeout(resolve, 10)); + // Wait for the intended startup failure seam, including any earlier admission I/O. + await Promise.race([ + commitEntered.promise, + resumePromise.then(() => { + throw new Error("resume settled before committing its partial"); + }), + ]); session.beginDispose(); commitDeferred.resolve(Err("workspace removed mid-startup")); diff --git a/src/node/services/agentSession.editMessageId.test.ts b/src/node/services/agentSession.editMessageId.test.ts index c2b37bae12c..221b24b2465 100644 --- a/src/node/services/agentSession.editMessageId.test.ts +++ b/src/node/services/agentSession.editMessageId.test.ts @@ -92,7 +92,7 @@ describe("AgentSession.sendMessage (editMessageId)", () => { it("treats missing edit target as no-op (allows recovery after compaction)", async () => { const { session, historyService, streamMessage } = await createSessionHarness("ws-test"); const truncateAfterMessage = spyOn(historyService, "truncateAfterMessage"); - const appendToHistory = spyOn(historyService, "appendToHistory"); + const acceptance = spyOn(historyService, "acceptCompactionReplacement"); const result = await session.sendMessage("hello", { model: TEST_MODEL, @@ -102,7 +102,7 @@ describe("AgentSession.sendMessage (editMessageId)", () => { expect(result.success).toBe(true); expect(truncateAfterMessage.mock.calls).toHaveLength(1); - expect(appendToHistory.mock.calls).toHaveLength(1); + expect(acceptance.mock.calls).toHaveLength(1); await session.waitForIdle(); expect(streamMessage.mock.calls).toHaveLength(1); @@ -140,7 +140,7 @@ describe("AgentSession.sendMessage (editMessageId)", () => { createMuxMessage("assistant-original", "assistant", "reply", { historySequence: 1 }) ); const truncateAfterMessage = spyOn(historyService, "truncateAfterMessage"); - const appendToHistory = spyOn(historyService, "appendToHistory"); + const acceptance = spyOn(historyService, "acceptCompactionReplacement"); const result = await session.sendMessage("edited", { model: "invalid-model", @@ -153,7 +153,7 @@ describe("AgentSession.sendMessage (editMessageId)", () => { expect(result.error.type).toBe("invalid_model_string"); } expect(truncateAfterMessage).not.toHaveBeenCalled(); - expect(appendToHistory).not.toHaveBeenCalled(); + expect(acceptance).not.toHaveBeenCalled(); expect(streamMessage).not.toHaveBeenCalled(); const history = await historyService.getHistoryFromLatestBoundary(workspaceId); @@ -172,7 +172,7 @@ describe("AgentSession.sendMessage (editMessageId)", () => { const originalMessageId = "user-message-with-image"; await seedImageMessage(workspaceId, historyService, originalMessageId); const truncateAfterMessage = spyOn(historyService, "truncateAfterMessage"); - const appendToHistory = spyOn(historyService, "appendToHistory"); + const acceptance = spyOn(historyService, "acceptCompactionReplacement"); const result = await session.sendMessage("edited", { model: TEST_MODEL, @@ -183,9 +183,11 @@ describe("AgentSession.sendMessage (editMessageId)", () => { expect(result.success).toBe(true); expect(truncateAfterMessage.mock.calls).toHaveLength(1); - expect(appendToHistory.mock.calls).toHaveLength(1); + expect(acceptance.mock.calls).toHaveLength(1); - const appendedMessage = appendToHistory.mock.calls[0][1]; + const persisted = await historyService.getLastMessages(workspaceId, 1); + if (!persisted.success) throw new Error(persisted.error); + const appendedMessage = persisted.data[0]; const appendedFileParts = appendedMessage.parts.filter( (part) => part.type === "file" ) as Array<{ type: "file"; url: string; mediaType: string }>; @@ -199,7 +201,7 @@ describe("AgentSession.sendMessage (editMessageId)", () => { const originalMessageId = "user-message-with-image"; const originalImageUrl = await seedImageMessage(workspaceId, historyService, originalMessageId); const truncateAfterMessage = spyOn(historyService, "truncateAfterMessage"); - const appendToHistory = spyOn(historyService, "appendToHistory"); + const acceptance = spyOn(historyService, "acceptCompactionReplacement"); const result = await session.sendMessage("edited", { model: TEST_MODEL, agentId: "exec", @@ -208,9 +210,11 @@ describe("AgentSession.sendMessage (editMessageId)", () => { expect(result.success).toBe(true); expect(truncateAfterMessage.mock.calls).toHaveLength(1); - expect(appendToHistory.mock.calls).toHaveLength(1); + expect(acceptance.mock.calls).toHaveLength(1); - const appendedMessage = appendToHistory.mock.calls[0][1]; + const persisted = await historyService.getLastMessages(workspaceId, 1); + if (!persisted.success) throw new Error(persisted.error); + const appendedMessage = persisted.data[0]; const appendedFileParts = appendedMessage.parts.filter( (part) => part.type === "file" ) as Array<{ type: "file"; url: string; mediaType: string }>; @@ -300,8 +304,6 @@ describe("AgentSession.sendMessage (editMessageId)", () => { workspaceId, streamHandler ); - const appendToHistory = spyOn(historyService, "appendToHistory"); - try { const firstSendPromise = session.sendMessage("original", { model: TEST_MODEL, @@ -313,9 +315,9 @@ describe("AgentSession.sendMessage (editMessageId)", () => { ); expect(sawPreparingTurn).toBe(true); - const originalMessage = appendToHistory.mock.calls - .map((call) => call[1]) - .find((message) => message.role === "user" && message.parts[0]?.type === "text"); + const persisted = await historyService.getLastMessages(workspaceId, 1); + if (!persisted.success) throw new Error(persisted.error); + const originalMessage = persisted.data[0]; const originalMessageId = originalMessage?.id; expect(typeof originalMessageId).toBe("string"); @@ -382,10 +384,10 @@ describe("AgentSession.sendMessage (editMessageId)", () => { }); const observed: { busyDuringTruncate: boolean | null } = { busyDuringTruncate: null }; const realTruncate = historyService.truncateAfterMessage.bind(historyService); - spyOn(historyService, "truncateAfterMessage").mockImplementation(async (wsId, messageId) => { + spyOn(historyService, "truncateAfterMessage").mockImplementation(async (...args) => { observed.busyDuringTruncate = session.isBusy(); await truncateGate; - return realTruncate(wsId, messageId); + return realTruncate(...args); }); const sendPromise = session.sendMessage("edited", { diff --git a/src/node/services/agentSession.mcpPromptSnapshot.test.ts b/src/node/services/agentSession.mcpPromptSnapshot.test.ts index b3e6c4fb3bf..714f07b7650 100644 --- a/src/node/services/agentSession.mcpPromptSnapshot.test.ts +++ b/src/node/services/agentSession.mcpPromptSnapshot.test.ts @@ -164,7 +164,7 @@ describe("AgentSession MCP prompt snapshots", () => { } }); - test("rolls back persisted snapshot rows when the user row append fails", async () => { + test("publishes no snapshot rows when manual trigger acceptance fails", async () => { const getPrompt = mock(() => Promise.resolve({ text: "Expanded prompt" })); const harness = await createAgentSessionHarness({ workspaceId: "workspace", @@ -172,13 +172,10 @@ describe("AgentSession MCP prompt snapshots", () => { }); try { - const realAppend = harness.historyService.appendToHistory.bind(harness.historyService); - const appendToHistory = spyOn(harness.historyService, "appendToHistory").mockImplementation( - async (workspaceId: string, message: MuxMessage) => { - if (message.metadata?.mcpPromptSnapshot) return realAppend(workspaceId, message); - return Err("disk full"); - } - ); + const acceptance = spyOn( + harness.historyService, + "acceptCompactionReplacement" + ).mockResolvedValueOnce(Err("disk full")); const result = await harness.session.sendMessage("Using MCP prompt coder/review: src", { model: "anthropic:claude-3-5-sonnet-latest", @@ -186,7 +183,7 @@ describe("AgentSession MCP prompt snapshots", () => { muxMetadata: promptMetadata(), }); expect(result.success).toBe(false); - appendToHistory.mockRestore(); + acceptance.mockRestore(); const history = await harness.historyService.getLastMessages("workspace", 10); expect(history.success).toBe(true); diff --git a/src/node/services/agentSession.pinnedBudget.test.ts b/src/node/services/agentSession.pinnedBudget.test.ts index 7656ae736b1..1bb7f57ce49 100644 --- a/src/node/services/agentSession.pinnedBudget.test.ts +++ b/src/node/services/agentSession.pinnedBudget.test.ts @@ -541,7 +541,7 @@ describe("pinned full-payload rollover admission", () => { } = fixture; const beginStart = spyOn(manager, "beginStreamStart"); const accepted = mock(() => undefined); - spyOn(historyService, "appendManyToHistory").mockResolvedValueOnce( + spyOn(historyService, "acceptCompactionReplacement").mockResolvedValueOnce( Err("injected rollover append failure") ); try { @@ -923,18 +923,23 @@ describe("pinned full-payload rollover admission", () => { } ); test.each([false, true])( - "cancellation during rollover append follows the durable rollback outcome (rollback fails=%s)", + "cancellation after rollover publication follows the durable rollback outcome (rollback fails=%s)", async (rollbackFails) => { const fixture = await setup("small"); const { h, historyService, before, start, assembly, modelCleanup } = fixture; const entered = Promise.withResolvers(); const release = Promise.withResolvers(); - const append = historyService.appendManyToHistory.bind(historyService); - spyOn(historyService, "appendManyToHistory").mockImplementationOnce(async (...args) => { - entered.resolve(); - await release.promise; - return append(...args); - }); + const publish = historyService.acceptCompactionReplacement.bind(historyService); + spyOn(historyService, "acceptCompactionReplacement").mockImplementationOnce( + async (...args) => { + const result = await publish(...args); + expect(result).toEqual(Ok({ kind: "accepted", witness: null })); + // Hold after the real receipt: cancellation must exercise rollback of durable rows. + entered.resolve(); + await release.promise; + return result; + } + ); const rollback = spyOn(historyService, "deleteMessages"); if (rollbackFails) rollback.mockResolvedValueOnce(Err("injected durable rollback failure")); const controller = new AbortController(); @@ -945,6 +950,7 @@ describe("pinned full-payload rollover admission", () => { "Wake retained when rollback fails", { model, agentId: "exec", experiments: { tokenBudget: true } }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelSignal: controller.signal, diff --git a/src/node/services/agentSession.preTurnMessages.test.ts b/src/node/services/agentSession.preTurnMessages.test.ts index 81be237ec97..115ca1222a8 100644 --- a/src/node/services/agentSession.preTurnMessages.test.ts +++ b/src/node/services/agentSession.preTurnMessages.test.ts @@ -75,7 +75,12 @@ describe("AgentSession.sendMessage (preTurnMessages)", () => { const result = await session.sendMessage( "family trigger", { model: TEST_MODEL, agentId: "exec" }, - { synthetic: true, agentInitiated: true, preTurnMessages: [payload] } + { + acceptanceOrigin: "automatic", + synthetic: true, + agentInitiated: true, + preTurnMessages: [payload], + } ); expect(result.success).toBe(true); @@ -113,7 +118,12 @@ describe("AgentSession.sendMessage (preTurnMessages)", () => { const result = await session.sendMessage( "family trigger", { model: TEST_MODEL, agentId: "exec" }, - { synthetic: true, agentInitiated: true, preTurnMessages: [payload] } + { + acceptanceOrigin: "automatic", + synthetic: true, + agentInitiated: true, + preTurnMessages: [payload], + } ); expect(result.success).toBe(false); diff --git a/src/node/services/agentSession.preparationAdmission.test.ts b/src/node/services/agentSession.preparationAdmission.test.ts index 6ed8290ccf2..c42e29bfbd4 100644 --- a/src/node/services/agentSession.preparationAdmission.test.ts +++ b/src/node/services/agentSession.preparationAdmission.test.ts @@ -47,7 +47,11 @@ describe("preparation admission", () => { await release.promise; return Ok(createStartedTurnHandle(h.session.closingSignal)); }); - h.session.queueMessage("head", { ...options, muxMetadata: metadata }, { synthetic: true }); + h.session.queueMessage( + "head", + { ...options, muxMetadata: metadata }, + { acceptanceOrigin: "automatic", synthetic: true } + ); h.session.queueMessage("tail", options); let observed = false; h.session.onChatEvent(({ message }) => { @@ -81,7 +85,11 @@ describe("preparation admission", () => { started.resolve(); return Promise.resolve(Ok(createStartedTurnHandle(h.session.closingSignal))); }); - h.session.queueMessage("removed", options, { synthetic: true, onCanceled: canceled }); + h.session.queueMessage("removed", options, { + acceptanceOrigin: "automatic", + synthetic: true, + onCanceled: canceled, + }); let removed = false; h.session.onChatEvent(({ message }) => { if (message.type !== "stream-lifecycle" || message.phase !== "preparing" || removed) return; @@ -188,6 +196,7 @@ describe("preparation admission", () => { return Promise.resolve(Ok(createStartedTurnHandle(h.session.closingSignal))); }); h.session.queueMessage("failed", options, { + acceptanceOrigin: "automatic", synthetic: true, onAcceptedPreStreamFailure: async () => { if (++cleanups === 1) throw new Error("cleanup transient"); @@ -263,7 +272,7 @@ describe("preparation admission", () => { } ); - test("a superseded direct append rolls back only its own row and preserves replacement thinking and retry state", async () => { + test("a superseded automatic append rolls back only its own row and preserves replacement thinking and retry state", async () => { const h = await harness("superseded-direct-state"); const appended = Promise.withResolvers(); const releaseAppend = Promise.withResolvers(); @@ -281,7 +290,7 @@ describe("preparation admission", () => { await releaseProvider.promise; return Ok(createStartedTurnHandle(h.session.closingSignal)); }); - const oldSend = h.session.sendMessage("old", options); + const oldSend = h.session.sendMessage("old", options, { acceptanceOrigin: "automatic" }); await appended.promise; const replacement = h.session.sendMessage("replacement", { ...options, @@ -310,7 +319,7 @@ describe("preparation admission", () => { } }); - test("a stale on-send compaction append is rolled back before its request can be replayed", async () => { + test("a stale automatic on-send compaction append is rolled back before its request can be replayed", async () => { const h = await harness("stale-compaction-append"); const monitor = (h.session as unknown as { compactionMonitor: CompactionMonitor }) .compactionMonitor; @@ -332,6 +341,7 @@ describe("preparation admission", () => { }); const stream = spyOn(h.aiService, "streamMessage"); const result = await h.session.sendMessage("deferred question", options, { + acceptanceOrigin: "automatic", admissionStale: () => stale, }); expect(result.success).toBe(false); diff --git a/src/node/services/agentSession.preparedHistory.test.ts b/src/node/services/agentSession.preparedHistory.test.ts index 0094d50d7bb..c8626ae496c 100644 --- a/src/node/services/agentSession.preparedHistory.test.ts +++ b/src/node/services/agentSession.preparedHistory.test.ts @@ -62,7 +62,7 @@ async function fixture() { assert(result.success); return result.data; }; - return { ...h, inputs, skills, prompts, rows }; + return { ...h, inputs, skills, prompts, rows, stream: spyOn(h.aiService, "streamMessage") }; } afterEach(async () => { @@ -106,7 +106,7 @@ describe("prepared history publication", () => { (["result", "rejection"] as const).map((outcome) => ({ failure, outcome })) ) )( - "a failed $failure append ($outcome) rolls back only this attempt's previously published rows", + "a failed automatic $failure append ($outcome) rolls back only this attempt's previously published rows", async ({ failure, outcome }) => { const h = await fixture(); const foreign = createMuxMessage("foreign", "assistant", "concurrent input"); @@ -131,7 +131,7 @@ describe("prepared history publication", () => { else appends.mockRejectedValueOnce(new Error("injected write failure")); const start = spyOn(h.aiService, "streamMessage"); const result = await h.session - .sendMessage("inspect input", options) + .sendMessage("inspect input", options, { acceptanceOrigin: "automatic" }) .catch((error: unknown) => error); expect(appends.mock.calls.slice(0, -1).map(([, row]) => row.id)).toEqual(earlier); expect(foreign.metadata?.historySequence).toBe(1); @@ -143,7 +143,7 @@ describe("prepared history publication", () => { ); test.each(["file", "skill", "prompt", "trigger"])( - "cancellation after %s publication still runs the existing rollback checkpoint", + "automatic cancellation after %s publication still runs the existing rollback checkpoint", async (after) => { const h = await fixture(); const controller = new AbortController(); @@ -159,6 +159,7 @@ describe("prepared history publication", () => { const start = spyOn(h.aiService, "streamMessage"); expect( await h.session.sendMessage("inspect input", options, { + acceptanceOrigin: "automatic", cancelSignal: controller.signal, onCanceled: canceled, onAccepted: accepted, @@ -171,8 +172,87 @@ describe("prepared history publication", () => { } ); - test("on-send compaction publishes only the request carrying the deferred user input", async () => { + test.each(["skill", "prompt"] as const)( + "manual %s materialization failure leaves no orphaned prefixes or accepted Stop", + async (failure) => { + const h = await fixture(); + await h.session.cancelCompaction(true); + const storage = h.historyService.getCompactionCancellationStorage(workspaceId); + const stop = await storage.read(); + h[failure === "skill" ? "skills" : "prompts"].mockRejectedValueOnce( + new Error("materialization failed") + ); + expect((await h.session.sendMessage("inspect input", options)).success).toBe(false); + expect(await h.rows()).toEqual([]); + expect((await storage.read())?.nonce).toBe(stop?.nonce); + expect(h.stream).not.toHaveBeenCalled(); + } + ); + + test("manual prefixes commit with the trigger and survive cancellation in the durable receipt", async () => { + const h = await fixture(); + await h.session.cancelCompaction(true); + const storage = h.historyService.getCompactionCancellationStorage(workspaceId); + const stop = await storage.read(); + const controller = new AbortController(); + const canceled = mock(() => undefined); + const accepted = mock(() => undefined); + const publish = h.historyService.acceptCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementation( + async (id, capture, operation, observer) => { + expect(await h.rows()).toEqual([]); + assert(operation.kind === "append"); + expect(operation.messages.slice(0, -1).map((row) => row.id)).toEqual([ + "file", + "skill", + "prompt", + ]); + return publish(id, capture, operation, { + ...observer, + onCommitted: (receipt) => { + observer.onCommitted(receipt); + controller.abort(); + }, + }); + } + ); + expect( + await h.session.sendMessage("inspect input", options, { + cancelSignal: controller.signal, + onCanceled: canceled, + onAccepted: accepted, + }) + ).toEqual(Ok(undefined)); + const rows = await h.rows(); + expect(rows).toHaveLength(4); + expect(rows.at(-1)?.metadata?.compactionReplacementNonce).toBe(stop?.nonce); + expect(accepted).toHaveBeenCalledTimes(1); + expect(canceled).not.toHaveBeenCalled(); + expect(await storage.read()).toBeNull(); + }); + + test("Stop before a manual batch commits leaves every prefix and trigger unpublished", async () => { + const h = await fixture(); + const publish = h.historyService.acceptCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementationOnce( + async (...args) => { + await h.session.cancelCompaction(true); + return publish(...args); + } + ); + expect((await h.session.sendMessage("inspect input", options)).success).toBe(false); + expect(await h.rows()).toEqual([]); + expect( + await h.historyService.getCompactionCancellationStorage(workspaceId).read() + ).not.toBeNull(); + expect(h.stream).not.toHaveBeenCalled(); + }); + + test("on-send compaction accepts only the request carrying the deferred user input", async () => { const h = await fixture(); + await h.session.cancelCompaction(true); + const storage = h.historyService.getCompactionCancellationStorage(workspaceId); + const stop = await storage.read(); spyOn(h.inputs.compactionMonitor, "checkBeforeSend").mockReturnValue({ shouldShowWarning: true, shouldForceCompact: true, @@ -185,6 +265,8 @@ describe("prepared history publication", () => { expect(await h.session.sendMessage("inspect input", options)).toEqual(Ok(undefined)); const rows = await h.rows(); expect(rows).toHaveLength(1); + expect(rows[0].metadata?.compactionReplacementNonce).toBe(stop?.nonce); + expect(await storage.read()).toBeNull(); const request = rows[0].metadata?.muxMetadata; assert(request?.type === "compaction-request"); expect(request.parsed.followUpContent?.text).toBe("inspect input"); diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 0fdd316ca9c..1f0542ce03e 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -3,6 +3,8 @@ import { runSessionTerminalPolicy } from "./agentSession.testHarness"; import { describe, expect, mock, spyOn, test } from "bun:test"; import { EventEmitter } from "node:events"; import * as fsPromises from "node:fs/promises"; +import * as nodeFs from "node:fs"; +import * as path from "node:path"; import { createDisplayUsage } from "@/common/utils/tokens/displayUsage"; import { getTotalCost } from "@/common/utils/tokens/usageAggregator"; @@ -13,6 +15,11 @@ import { createAgentSessionHarness, createStartedTurnHandle } from "./agentSessi import type { AIService } from "./aiService"; import type { CompactionMonitor } from "./compactionMonitor"; import type { TurnCompletion } from "./streamManager"; +import { + CompactionCancellation, + type CompactionReplacementCapture, + FileCompactionCancellationStorage, +} from "./compactionCancellation"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; const WORKSPACE_TURN_CORRELATION = { @@ -66,6 +73,458 @@ async function waitForCondition(condition: () => boolean, timeoutMs = 500): Prom } describe("AgentSession queued message tool-call dispatch", () => { + test.each(["before write", "failed flush"] as const)( + "Stop restores only unpublished queued input when held at %s", + async (phase) => { + const workspaceId = `queue-visible-publication-${phase}`; + const h = await createAgentSessionHarness({ workspaceId, captureEvents: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const storage = h.historyService.getCompactionCancellationStorage(workspaceId); + const chatPath = path.join(path.dirname(storage.path), "chat.jsonl"); + const open = fsPromises.open; + let appendFd: number | undefined; + let failed = false; + const opening = spyOn(fsPromises, "open").mockImplementation( + async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && args[1] === "a" && appendFd === undefined) { + appendFd = handle.fd; + if (phase === "before write") { + entered.resolve(); + await release.promise; + } else { + const close = handle.close.bind(handle); + spyOn(handle, "close").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + await close(); + }); + } + } + return handle; + } + ); + const sync = nodeFs.fsyncSync; + const flushing = spyOn(nodeFs, "fsyncSync").mockImplementation((fd) => { + if (phase === "failed flush" && fd === appendFd && !failed) { + failed = true; + throw new Error("queued input flush failed"); + } + sync(fd); + }); + const accepted = mock(() => undefined); + const failedPreparation = mock((error: unknown) => { + entered.reject(error); + }); + const stream = spyOn(h.aiService, "streamMessage"); + let stopping: ReturnType | undefined; + try { + h.session.queueMessage( + "first input", + { model: TEST_MODEL, agentId: "exec" }, + { + onAccepted: accepted, + onAcceptedPreStreamFailure: failedPreparation, + } + ); + h.session.queueMessage("later input", { model: TEST_MODEL, agentId: "exec" }); + h.session.sendQueuedMessages(); + await entered.promise; + expect(failed).toBe(phase === "failed flush"); + const visible = await fsPromises.readFile(chatPath, "utf8"); + expect(visible.includes("first input")).toBe(phase === "failed flush"); + stopping = h.session.interruptStream(); + release.resolve(); + expect(await stopping).toEqual(Ok(undefined)); + await h.session.waitForIdle(); + h.session.restoreQueueToInput(); + expect( + h.events.filter((event) => event.type === "restore-to-input").map((event) => event.text) + ).toEqual([phase === "failed flush" ? "later input" : "first input\nlater input"]); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.filter((row) => row.role === "user")).toHaveLength( + phase === "failed flush" ? 1 : 0 + ); + expect(accepted).not.toHaveBeenCalled(); + expect(failedPreparation).toHaveBeenCalledTimes(1); + expect(stream).not.toHaveBeenCalled(); + expect(await storage.read()).not.toBeNull(); + } finally { + release.resolve(); + opening.mockRestore(); + flushing.mockRestore(); + await stopping; + await h.session.dispose(); + await h.cleanup(); + } + } + ); + + test.each(["local", "foreign"] as const)( + "Send Now cannot replace a later %s Stop during workspace cleanup", + async (kind) => { + const workspaceId = `send-now-owned-stop-${kind}`; + const h = await createAgentSessionHarness({ workspaceId, captureEvents: true }); + const failed = Promise.withResolvers(); + let admission = h.session.captureCompactionAdmission("manual"); + let capture: CompactionReplacementCapture | undefined; + const stream = spyOn(h.aiService, "streamMessage"); + try { + h.session.queueMessage( + "owned draft", + { model: TEST_MODEL, agentId: "exec" }, + { + compactionAdmissionStale: () => admission(), + refreshCompactionAdmission: (isStale) => { + admission = isStale; + }, + onAcceptedPreStreamFailure: () => failed.resolve(), + } + ); + const first = h.session.interruptStream({ + onCompactionCanceled: (receipt) => { + capture = receipt; + }, + }); + const ownAdmission = h.session.captureCompactionAdmission("manual"); + expect(await first).toEqual(Ok(undefined)); + expect(capture).toBeDefined(); + if (kind === "local") expect(await h.session.interruptStream()).toEqual(Ok(undefined)); + else + await new CompactionCancellation( + h.historyService.getCompactionCancellationStorage(workspaceId) + ).cancel(); + const successor = await h.historyService + .getCompactionCancellationStorage(workspaceId) + .read(); + h.session.sendNextUserQueuedMessage({ isStale: ownAdmission, readCapture: () => capture }); + expect( + await waitForCondition( + () => + stream.mock.calls.length > 0 || + h.events.some((event) => event.type === "restore-to-input") + ) + ).toBe(true); + expect(stream).not.toHaveBeenCalled(); + await failed.promise; + await h.session.waitForIdle(); + expect( + h.events.filter((event) => event.type === "restore-to-input").map((event) => event.text) + ).toEqual(["owned draft"]); + expect(await h.historyService.getHistoryFromLatestBoundary(workspaceId)).toEqual(Ok([])); + expect(await h.historyService.getCompactionCancellationStorage(workspaceId).read()).toEqual( + successor + ); + expect(stream).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } + ); + + test.each(["Stop retry", "capture"] as const)( + "Send Now restores unpublished input after a fresh admission fails at %s", + async (failure) => { + const workspaceId = `queue-send-now-failure-${failure}`; + const h = await createAgentSessionHarness({ workspaceId, captureEvents: true }); + const accepted = mock(() => undefined); + const failed = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage"); + let admission = h.session.captureCompactionAdmission("manual"); + let refreshed = false; + // Record the queue's original frontier before injecting failure into Send Now's + // explicit fresh acquisition; eager queue capture must not consume the fault. + const original = await h.historyService.captureCompactionReplacement(workspaceId); + const injection = + failure === "Stop retry" + ? spyOn(FileCompactionCancellationStorage.prototype, "mutate").mockImplementation(() => + Promise.reject(new Error("Stop persistence unavailable")) + ) + : spyOn(h.historyService, "captureCompactionReplacement").mockResolvedValueOnce( + Err("capture unavailable") + ); + try { + expect(original.success).toBe(true); + h.session.queueMessage( + "send now draft", + { model: TEST_MODEL, agentId: "exec" }, + { + readCompactionAdmission: () => Promise.resolve(original), + compactionAdmissionStale: () => admission(), + refreshCompactionAdmission: () => { + admission = h.session.captureCompactionAdmission("manual"); + refreshed = true; + }, + onAccepted: accepted, + onAcceptedPreStreamFailure: () => failed.resolve(), + } + ); + expect((await h.session.cancelCompaction()).success).toBe(failure !== "Stop retry"); + expect(h.session.sendNextUserQueuedMessage()).toBe(true); + await failed.promise; + await h.session.waitForIdle(); + expect(refreshed).toBe(failure !== "capture"); + expect(admission()).toBe(failure === "capture"); + expect( + h.events.filter((event) => event.type === "restore-to-input").map((event) => event.text) + ).toEqual(["send now draft"]); + expect(await h.historyService.getHistoryFromLatestBoundary(workspaceId)).toEqual(Ok([])); + expect(accepted).not.toHaveBeenCalled(); + expect(stream).not.toHaveBeenCalled(); + } finally { + injection.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } + } + ); + + test.each([false, true])( + "queued gate rejection restores only an unwritten row (write fails=%s)", + async (writeFails) => { + const workspaceId = `queue-rejected-publication-${writeFails}`; + const h = await createAgentSessionHarness({ + workspaceId, + captureEvents: true, + workspaceGoalService: { + assertPricedModelForBudgetedGoal: () => + Promise.resolve(Err({ type: "unknown", raw: "pricing refused" })), + } as unknown as WorkspaceGoalService, + }); + const failed = Promise.withResolvers(); + const goalSafety = spyOn( + h.session as unknown as { + applyManualUserMessageGoalSafety(): Promise; + }, + "applyManualUserMessageGoalSafety" + ).mockResolvedValue(undefined); + const append = writeFails + ? spyOn(h.historyService, "acceptCompactionReplacement").mockResolvedValueOnce( + Err("disk unavailable") + ) + : undefined; + try { + h.session.queueMessage( + "rejected draft", + { model: TEST_MODEL, agentId: "exec" }, + { + onAcceptedPreStreamFailure: () => failed.resolve(), + } + ); + h.session.sendQueuedMessages(); + await failed.promise; + await h.session.waitForIdle(); + expect( + h.events.filter((event) => event.type === "restore-to-input").map((event) => event.text) + ).toEqual(writeFails ? ["rejected draft"] : []); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.filter((row) => row.role === "user")).toHaveLength( + writeFails ? 0 : 1 + ); + } finally { + append?.mockRestore(); + goalSafety.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } + } + ); + + test.each([false, true])( + "Send Now refreshes the queued capture and restores only a later Stop (stopped=%s)", + async (stopped) => { + const workspaceId = `queue-send-now-stop-${stopped}`; + const h = await createAgentSessionHarness({ workspaceId, captureEvents: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const capture = h.historyService.captureCompactionReplacement.bind(h.historyService); + const held = spyOn(h.historyService, "captureCompactionReplacement").mockImplementationOnce( + async (...args) => { + const result = await capture(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + const stream = spyOn(h.aiService, "streamMessage"); + const accepted = Promise.withResolvers(); + let admission = h.session.captureCompactionAdmission("manual"); + try { + h.session.queueMessage( + "send now input", + { model: TEST_MODEL, agentId: "exec" }, + { + compactionAdmissionStale: () => admission(), + refreshCompactionAdmission: () => { + admission = h.session.captureCompactionAdmission("manual"); + }, + onAccepted: () => accepted.resolve(), + } + ); + expect(await h.session.cancelCompaction()).toEqual(Ok(undefined)); + expect(h.session.sendNextUserQueuedMessage()).toBe(true); + await entered.promise; + if (stopped) { + expect(await h.session.interruptStream()).toEqual(Ok(undefined)); + h.session.restoreQueueToInput(); + } + release.resolve(); + if (stopped) await h.session.waitForIdle(); + else { + await accepted.promise; + // Once the row is durable, a later Stop must not offer it as unsent input again. + expect(await h.session.cancelCompaction()).toEqual(Ok(undefined)); + h.session.restoreQueueToInput(); + } + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.filter((row) => row.role === "user")).toHaveLength( + stopped ? 0 : 1 + ); + expect( + h.events.filter((event) => event.type === "restore-to-input").map((event) => event.text) + ).toEqual(stopped ? ["send now input"] : []); + if (stopped) expect(stream).not.toHaveBeenCalled(); + } finally { + release.resolve(); + held.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } + } + ); + + test.each(["automatic-visible", "automatic-hidden", "caller-canceled", "caller-stale"] as const)( + "Stop does not restore a dequeued %s candidate over later manual input", + async (kind) => { + const workspaceId = `queue-stop-control-${kind}`; + const h = await createAgentSessionHarness({ workspaceId, captureEvents: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const controller = new AbortController(); + let stale = false; + const send = h.session.sendMessage.bind(h.session); + const held = spyOn(h.session, "sendMessage").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return send(...args); + }); + const stream = spyOn(h.aiService, "streamMessage"); + try { + h.session.queueMessage( + "revoked candidate", + { model: TEST_MODEL, agentId: "exec" }, + { + acceptanceOrigin: kind.startsWith("automatic") ? "automatic" : "manual", + synthetic: kind === "automatic-hidden", + cancelSignal: controller.signal, + admissionStale: () => stale, + } + ); + h.session.queueMessage("later manual", { model: TEST_MODEL, agentId: "exec" }); + h.session.sendQueuedMessages(); + await entered.promise; + if (kind === "caller-canceled") controller.abort(); + if (kind === "caller-stale") stale = true; + expect(await h.session.interruptStream()).toEqual(Ok(undefined)); + h.session.restoreQueueToInput(); + release.resolve(); + await h.session.waitForIdle(); + expect(h.events.filter((event) => event.type === "restore-to-input")).toMatchObject([ + { text: "later manual", fileParts: [] }, + ]); + expect(await h.historyService.getHistoryFromLatestBoundary(workspaceId)).toEqual(Ok([])); + expect(stream).not.toHaveBeenCalled(); + } finally { + release.resolve(); + held.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } + } + ); + + test.each(["held", "refused", "raw command"] as const)( + "Stop restores dequeued manual input and later queued input together (%s)", + async (restoreAt) => { + const workspaceId = `queue-stop-restore-${restoreAt}`; + const h = await createAgentSessionHarness({ workspaceId, captureEvents: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const capture = h.historyService.captureCompactionReplacement.bind(h.historyService); + const held = spyOn(h.historyService, "captureCompactionReplacement").mockImplementationOnce( + async (...args) => { + const result = await capture(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + const stream = spyOn(h.aiService, "streamMessage"); + const accepted = mock(() => undefined); + const fileParts = [{ url: "data:image/png;base64,aGVsbG8=", mediaType: "image/png" }]; + const laterFileParts = [{ url: "data:text/plain;base64,dGFpbA==", mediaType: "text/plain" }]; + const reviews = [ + { filePath: "src/file.ts", lineRange: "1", selectedCode: "call()", userNote: "check this" }, + ]; + try { + h.session.queueMessage( + "first\nsecond", + { + model: TEST_MODEL, + agentId: "exec", + fileParts, + muxMetadata: + restoreAt === "raw command" + ? { + type: "agent-skill", + rawCommand: "/init", + skillName: "init", + scope: "built-in", + reviews, + } + : { type: "normal", reviews }, + }, + { + onAccepted: accepted, + } + ); + h.session.queueMessage("later input", { + model: TEST_MODEL, + agentId: "exec", + fileParts: laterFileParts, + }); + h.session.sendQueuedMessages(); + await entered.promise; + expect(h.session.queuedMessageEntryCount()).toBe(1); + expect(await h.session.interruptStream()).toEqual(Ok(undefined)); + if (restoreAt === "held") h.session.restoreQueueToInput(); + release.resolve(); + await h.session.waitForIdle(); + if (restoreAt === "refused") h.session.restoreQueueToInput(); + expect(h.events.filter((event) => event.type === "restore-to-input")).toEqual([ + { + type: "restore-to-input", + workspaceId, + text: `${restoreAt === "raw command" ? "/init" : "first\nsecond"}\nlater input`, + fileParts: [...fileParts, ...laterFileParts], + reviews, + }, + ]); + expect(await h.historyService.getHistoryFromLatestBoundary(workspaceId)).toEqual(Ok([])); + expect(stream).not.toHaveBeenCalled(); + expect(accepted).not.toHaveBeenCalled(); + expect(h.session.hasQueuedMessages()).toBe(false); + } finally { + release.resolve(); + held.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } + } + ); + test("a queued provider startup failure drains its successor after accepted-turn cleanup", async () => { const successor = Promise.withResolvers(); let calls = 0; @@ -85,7 +544,7 @@ describe("AgentSession queued message tool-call dispatch", () => { session.queueMessage( "failed startup", { model: TEST_MODEL, agentId: "exec" }, - { synthetic: true } + { acceptanceOrigin: "automatic", synthetic: true } ); session.queueMessage("successor", { model: TEST_MODEL, agentId: "exec" }); session.sendQueuedMessages(); @@ -127,6 +586,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "failed head", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, onAcceptedPreStreamFailure: async (error) => { failures.push(error); @@ -214,7 +674,7 @@ describe("AgentSession queued message tool-call dispatch", () => { await session.sendMessage( "start", { model: TEST_MODEL, agentId: "exec" }, - { synthetic: true, agentInitiated: true } + { acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true } ) ).success ).toBe(true); @@ -326,7 +786,7 @@ describe("AgentSession queued message tool-call dispatch", () => { session.queueMessage( "queued continuation", { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, - { synthetic: true } + { acceptanceOrigin: "automatic", synthetic: true } ); expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(false); expect(session.hasQueuedOrDispatchingEntry(differentCorrelation)).toBe(true); @@ -334,7 +794,7 @@ describe("AgentSession queued message tool-call dispatch", () => { session.queueMessage( "second queued continuation", { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, - { synthetic: true } + { acceptanceOrigin: "automatic", synthetic: true } ); expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(false); @@ -342,6 +802,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "unrelated predecessor", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, } ); @@ -373,12 +834,12 @@ describe("AgentSession queued message tool-call dispatch", () => { session.queueMessage( "manual message", { model: TEST_MODEL, agentId: "exec" }, - { synthetic: true } + { acceptanceOrigin: "automatic", synthetic: true } ); session.queueMessage( "workspace-turn follow-up", { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, - { synthetic: true } + { acceptanceOrigin: "automatic", synthetic: true } ); // Queued stage: the manual head entry is the candidate (no metadata). @@ -409,7 +870,7 @@ describe("AgentSession queued message tool-call dispatch", () => { session.queueMessage( "workspace-turn follow-up", { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, - { synthetic: true } + { acceptanceOrigin: "automatic", synthetic: true } ); // Force the dequeue-to-stream-start window with PREPARING already // released (a background send can resolve before stream-start): the @@ -444,7 +905,7 @@ describe("AgentSession queued message tool-call dispatch", () => { muxMetadata: WORKSPACE_TURN_CORRELATION, queueDispatchMode: "turn-end", }, - { synthetic: true } + { acceptanceOrigin: "automatic", synthetic: true } ); const cutter = session.getQueueCutCutter(); @@ -573,6 +1034,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "tool-end" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelSignal: controller.signal, @@ -621,6 +1083,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec", queueDispatchMode: withdrawnMode }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelSignal: controller.signal, @@ -710,7 +1173,7 @@ describe("AgentSession queued message tool-call dispatch", () => { const dispatchMode = session.queueMessage( "[Scheduled heartbeat] check in", { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "turn-end" }, - { synthetic: true, dedupeKey: "heartbeat-request" } + { acceptanceOrigin: "automatic", synthetic: true, dedupeKey: "heartbeat-request" } ); expect(dispatchMode).toBe("turn-end"); expect(session.hasQueuedDedupeKey("heartbeat-request")).toBe(true); @@ -720,7 +1183,7 @@ describe("AgentSession queued message tool-call dispatch", () => { session.queueMessage( "[Scheduled heartbeat] check in", { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "turn-end" }, - { synthetic: true, dedupeKey: "heartbeat-request" } + { acceptanceOrigin: "automatic", synthetic: true, dedupeKey: "heartbeat-request" } ) ).toBeNull(); @@ -767,7 +1230,7 @@ describe("AgentSession queued message tool-call dispatch", () => { session.queueMessage( "hidden predecessor", { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "turn-end" }, - { synthetic: true, agentInitiated: true } + { acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true } ); session.queueMessage("my queued follow-up", { model: TEST_MODEL, @@ -806,7 +1269,7 @@ describe("AgentSession queued message tool-call dispatch", () => { session.queueMessage( "[Scheduled heartbeat] check in", { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "turn-end" }, - { synthetic: true, dedupeKey: "heartbeat-request" } + { acceptanceOrigin: "automatic", synthetic: true, dedupeKey: "heartbeat-request" } ); expect(session.hasQueuedMessages()).toBe(true); @@ -866,6 +1329,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, onCanceled: (reason) => { @@ -926,6 +1390,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelState, @@ -1012,6 +1477,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelState, @@ -1097,6 +1563,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelState, @@ -1183,6 +1650,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelState, @@ -1279,6 +1747,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelSignal: controller.signal, @@ -1371,6 +1840,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelSignal: controller.signal, @@ -1431,6 +1901,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelState, @@ -1490,6 +1961,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "Background monitor wake", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true, cancelState, @@ -1583,6 +2055,7 @@ describe("AgentSession queued message tool-call dispatch", () => { "queued peer trigger", { model: TEST_MODEL, agentId: "exec" }, { + acceptanceOrigin: "automatic", synthetic: true, // Peer sends refund their family-message reservation through this hook; a dispatch // that REJECTS (throws) instead of returning Err must reach it just like the diff --git a/src/node/services/agentSession.scopedLifetimes.test.ts b/src/node/services/agentSession.scopedLifetimes.test.ts index 14f9df35f55..4d5a27ec28f 100644 --- a/src/node/services/agentSession.scopedLifetimes.test.ts +++ b/src/node/services/agentSession.scopedLifetimes.test.ts @@ -17,6 +17,42 @@ const workspaceId = "scoped-turn"; const options = { model: "openai:gpt-4o", agentId: "exec" }; describe("AgentSession scoped turn lifetimes", () => { + test("cleared direct queue admission remains supervised until its disk capture settles", async () => { + const appFiberScope = Scope.makeUnsafe("parallel"); + const h = await createAgentSessionHarness({ workspaceId, appFiberScope }); + const stream = spyOn(h.aiService, "streamMessage"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const capture = h.historyService.captureCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "captureCompactionReplacement").mockImplementationOnce( + async (...args) => { + const result = await capture(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + h.session.queueMessage("clear before capture returns", options); + await entered.promise; + h.session.clearQueue(); + let closed = false; + const closing = runner.runPromise(Scope.close(appFiberScope, Exit.void)).then(() => { + closed = true; + }); + try { + await runner.runPromise(Effect.yieldNow); + expect(closed).toBe(false); + release.resolve(); + await closing; + expect(stream).not.toHaveBeenCalled(); + } finally { + release.resolve(); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + }); + test("queue clear publication cannot outrun its cancellation refund", async () => { const appFiberScope = Scope.makeUnsafe("parallel"); const entered = Promise.withResolvers(); @@ -104,6 +140,7 @@ describe("AgentSession scoped turn lifetimes", () => { }); }; h.session.queueMessage("queued", options, { + acceptanceOrigin: "automatic", synthetic: true, onAcceptedPreStreamFailure: () => { entered.resolve(); @@ -143,10 +180,10 @@ describe("AgentSession scoped turn lifetimes", () => { const entered = Promise.withResolvers(); const release = Promise.withResolvers(); const h = await createAgentSessionHarness({ workspaceId, appFiberScope }); - const append = h.historyService.appendToHistory.bind(h.historyService); - spyOn(h.historyService, "appendToHistory").mockImplementation(async (id, message) => { - const result = await append(id, message); - if (message.role === "user") { + const append = h.historyService.acceptCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementation(async (...args) => { + const result = await append(...args); + if (result.success && result.data.kind === "accepted") { entered.resolve(); await release.promise; } @@ -166,9 +203,11 @@ describe("AgentSession scoped turn lifetimes", () => { release.resolve(); await Promise.all([send, closing]); expect(spyOn(h.aiService, "streamMessage")).not.toHaveBeenCalled(); - // The joined append is then rolled back: the refused turn leaves no row for startup recovery. + // The history receipt accepted the user input before shutdown; retain it for explicit resume. const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history).toEqual(Ok([])); + expect(history.success && history.data.at(-1)?.parts).toMatchObject([ + { type: "text", text: "persist before preparing" }, + ]); } finally { release.resolve(); await send; @@ -278,19 +317,26 @@ describe("AgentSession scoped turn lifetimes", () => { (await goalService.setGoal({ workspaceId, objective: "Continue until interrupted" })) .success ).toBe(true); - const append = h.historyService.appendToHistory.bind(h.historyService); - spyOn(h.historyService, "appendToHistory").mockImplementation(async (id, message) => { - if (message.metadata?.contextBudgetRejected && heldWrite === "history") { - entered.resolve(); - await release.promise; - } - const result = await append(id, message); - if (message.metadata?.contextBudgetRejected) { - writes.push("history"); - if (closed) writesAfterDrain.push("history"); + const accept = h.historyService.acceptCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementation( + async (...args) => { + const result = await accept(...args); + const operation = args[2]; + const rejected = + operation.kind === "append" && + operation.messages.some((message) => message.metadata?.contextBudgetRejected); + // Hold a real publication receipt: close before the CAS correctly refuses the row. + if (rejected && heldWrite === "history") { + entered.resolve(); + await release.promise; + } + if (rejected) { + writes.push("history"); + if (closed) writesAfterDrain.push("history"); + } + return result; } - return result; - }); + ); const setGoal = goalService.setGoal.bind(goalService); spyOn(goalService, "setGoal").mockImplementation(async (input) => { if (input.status === "paused" && heldWrite === "goal") { diff --git a/src/node/services/agentSession.startupAutoRetry.test.ts b/src/node/services/agentSession.startupAutoRetry.test.ts index 14d798f66a3..b1146781122 100644 --- a/src/node/services/agentSession.startupAutoRetry.test.ts +++ b/src/node/services/agentSession.startupAutoRetry.test.ts @@ -276,7 +276,7 @@ describe("AgentSession startup auto-retry recovery", () => { }); test.each(["materialize", "append"] as const)( - "beginShutdown inside the %s await rolls the unaccepted turn back instead of leaving a row", + "beginShutdown inside the %s await preserves only accepted user input", async (window) => { const workspaceId = `startup-retry-shutdown-mid-${window}`; const streamMessage = mock(() => @@ -307,9 +307,9 @@ describe("AgentSession startup auto-retry recovery", () => { return snapshots; }; } else { - const append = historyService.appendToHistory.bind(historyService); - spyOn(historyService, "appendToHistory").mockImplementation(async (id, message) => { - const result = await append(id, message); + const append = historyService.acceptCompactionReplacement.bind(historyService); + spyOn(historyService, "acceptCompactionReplacement").mockImplementation(async (...args) => { + const result = await append(...args); session.beginShutdown(); return result; }); @@ -322,9 +322,14 @@ describe("AgentSession startup auto-retry recovery", () => { expect(sendResult.success).toBe(false); expect(streamMessage).not.toHaveBeenCalled(); const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success ? history.data.map((row) => row.id) : ["unexpected"]).toEqual( - seeded.map((row) => row.id) + expect(history.success ? history.data.slice(0, seeded.length) : ["unexpected"]).toMatchObject( + seeded ); + expect(history.success && history.data.length).toBe( + seeded.length + (window === "append" ? 1 : 0) + ); + if (window === "append" && history.success) + expect(history.data.at(-1)?.parts).toMatchObject([{ type: "text", text: "hello" }]); await session.dispose(); } @@ -364,9 +369,9 @@ describe("AgentSession startup auto-retry recovery", () => { return snapshots; }; } else { - const append = historyService.appendToHistory.bind(historyService); - spyOn(historyService, "appendToHistory").mockImplementation(async (id, message) => { - const result = await append(id, message); + const append = historyService.acceptCompactionReplacement.bind(historyService); + spyOn(historyService, "acceptCompactionReplacement").mockImplementation(async (...args) => { + const result = await append(...args); session.beginShutdown(); return result; }); diff --git a/src/node/services/agentSession.tokenBudget.test.ts b/src/node/services/agentSession.tokenBudget.test.ts index 4e46adb8ef0..2e45a8b7cc0 100644 --- a/src/node/services/agentSession.tokenBudget.test.ts +++ b/src/node/services/agentSession.tokenBudget.test.ts @@ -12,6 +12,7 @@ import { createMuxMessage, type MuxMessage } from "@/common/types/message"; import type { SendMessageError } from "@/common/types/errors"; import type { FrontendWorkspaceMetadata } from "@/common/types/workspace"; import { Err, Ok } from "@/common/types/result"; +import assert from "@/common/utils/assert"; import { prepareProviderRequestMessages } from "./turnContextAssembler"; import { MuxMessageSchema } from "@/common/orpc/schemas/message"; import { sliceMessagesForProviderFromLatestContextBoundary } from "@/common/utils/messages/compactionBoundary"; @@ -23,6 +24,8 @@ import { FLUSH_MAX_OUTPUT_TOKENS, } from "@/common/constants/contextBudget"; import type { AgentSessionAIService } from "./agentSession"; +import { CompactionCancellation } from "./compactionCancellation"; +import { HistoryService } from "./historyService"; import { createAgentSessionHarness, type AgentSessionHarness } from "./agentSession.testHarness"; import { createTurnCompletionController, type SettledStepBudget } from "./streamManager"; import { createRolloverPrefix, type ContextWindowRollover } from "./contextWindowRollover"; @@ -365,6 +368,22 @@ describe("AgentSession token-budget lifecycle", () => { } ); + test("queued context-budget rejection persists once without restoring a duplicate draft", async () => { + const h = await setup(); + const failed = Promise.withResolvers(); + h.session.queueMessage("oversized ".repeat(60_000), options, { + onAcceptedPreStreamFailure: () => failed.resolve(), + }); + h.session.sendQueuedMessages(); + await failed.promise; + await h.session.waitForIdle(); + const rows = await allRows(h); + expect(rows).toHaveLength(1); + expect(rows[0].metadata?.contextBudgetRejected).toBe(true); + expect(h.events.filter((event) => event.type === "restore-to-input")).toHaveLength(0); + expect(h.requests).toHaveLength(0); + }); + test.each([false, true])( "a rejected tail never retries the older completed turn after restart (legacy=%s)", async (legacy) => { @@ -414,11 +433,17 @@ describe("AgentSession token-budget lifecycle", () => { h.session.setAutoCompactionThreshold(1); await seedHistory(h, 20_000); const before = await allRows(h); - const append = spyOn(h.historyService, "appendToHistory"); + const append = spyOn(h.historyService, "acceptCompactionReplacement"); const batch = spyOn(h.historyService, "appendManyToHistory"); expect((await h.session.sendMessage("Ordinary next request", options)).success).toBe(true); expect(batch).not.toHaveBeenCalled(); - expect(append.mock.calls.some(([, row]) => text(row) === "Ordinary next request")).toBe(true); + expect( + append.mock.calls.some( + ([, , operation]) => + operation.kind === "append" && + operation.messages.some((row) => text(row) === "Ordinary next request") + ) + ).toBe(true); expect((await allRows(h)).slice(0, before.length)).toEqual(before); expect(h.requests).toHaveLength(1); }); @@ -426,7 +451,7 @@ describe("AgentSession token-budget lifecycle", () => { test("a failed single-user append preserves old history and does not dispatch", async () => { const h = await setup(); const before = await allRows(h); - spyOn(h.historyService, "appendToHistory").mockResolvedValueOnce(Err("disk full")); + spyOn(h.historyService, "acceptCompactionReplacement").mockResolvedValueOnce(Err("disk full")); expect((await h.session.sendMessage("Not durably accepted", options)).success).toBe(false); expect(await allRows(h)).toEqual(before); expect(h.requests).toHaveLength(0); @@ -447,6 +472,7 @@ describe("AgentSession token-budget lifecycle", () => { expect( ( await h.session.sendMessage("Cancel after persistence", options, { + acceptanceOrigin: "automatic", cancelSignal: controller.signal, cancelState, }) @@ -552,11 +578,13 @@ describe("AgentSession token-budget lifecycle", () => { await cleanup(); }); } else { - const append = h.historyService.appendManyToHistory.bind(h.historyService); - spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce(async (id, rows) => { - replaceRegistration(); - return append(id, rows); - }); + const append = h.historyService.acceptCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementationOnce( + async (...args) => { + replaceRegistration(); + return append(...args); + } + ); } try { expect((await h.session.sendMessage("Admitted turn", options)).success).toBe(true); @@ -850,7 +878,7 @@ describe("AgentSession token-budget lifecycle", () => { runtimeConfig: { type: "local" }, } as FrontendWorkspaceMetadata) ); - const append = spyOn(h.historyService, "appendManyToHistory"); + const append = spyOn(h.historyService, "acceptCompactionReplacement"); const result = await h.session.sendMessage("Do the requested work", { ...options, muxMetadata: { @@ -876,7 +904,9 @@ describe("AgentSession token-budget lifecycle", () => { expect(text(user)).toBe("Do the requested work"); expect(user.metadata?.muxMetadata?.type).toBe("agent-skill"); expect(append.mock.calls).toHaveLength(1); - expect(append.mock.calls[0][1].map((row) => row.id)).toEqual( + const operation = append.mock.calls[0][2]; + assert(operation.kind === "append"); + expect(operation.messages.map((row) => row.id)).toEqual( rows.slice(boundaryIndex).map((row) => row.id) ); expect(h.requests).toHaveLength(1); @@ -1412,12 +1442,7 @@ describe("AgentSession token-budget lifecycle", () => { await fs.writeFile(pendingPath, before); await cleanup(...args); }); - const append = h.historyService.appendManyToHistory.bind(h.historyService); - spyOn(h.historyService, "appendManyToHistory").mockImplementation(async (id, rows) => { - const rollover = rolloverRows(rows).length > 0; - if (rollover && outcome === "append-failed") return Err("injected rollover append failure"); - const result = await append(id, rows); - if (!rollover || !result.success) return result; + const afterPublication = async () => { if (outcome === "successor") { const pending = new CompactionPendingState( pendingPath, @@ -1456,9 +1481,17 @@ describe("AgentSession token-budget lifecycle", () => { if (outcome === "ack-failed" || outcome === "ack-and-cleanup-failed") { throw new Error(acknowledgmentFailure); } + }; + // Both paths use the guarded history receipt; retain real disk publication. + const accept = h.historyService.acceptCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementation(async (...args) => { + const operation = args[2]; + const rollover = operation.kind === "append" && rolloverRows(operation.messages).length > 0; + if (rollover && outcome === "append-failed") return Err("injected rollover append failure"); + const result = await accept(...args); + if (rollover && result.success && result.data.kind === "accepted") await afterPublication(); return result; }); - const sent = await h.session.sendMessage("Roll over this window", options); if (cleanupFails) { const causes = reportedErrors.mock.calls.flatMap((args) => @@ -1510,7 +1543,7 @@ describe("AgentSession token-budget lifecycle", () => { const h = await setup(); await seedHistory(h, 110_000); const cleanup = spyOn(h.session, "applyContextResetSideEffects"); - const append = spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce( + const append = spyOn(h.historyService, "acceptCompactionReplacement").mockImplementationOnce( async () => { expect(cleanup).toHaveBeenCalledTimes(1); await Promise.resolve(); @@ -1520,7 +1553,9 @@ describe("AgentSession token-budget lifecycle", () => { expect((await h.session.sendMessage("Retry me", options)).success).toBe(false); expect(rolloverRows(await allRows(h))).toHaveLength(0); expect(h.requests).toHaveLength(0); - const failedRollover = append.mock.calls[0][1][0].metadata?.muxMetadata; + const failedOperation = append.mock.calls[0][2]; + assert(failedOperation.kind === "append"); + const failedRollover = failedOperation.messages[0].metadata?.muxMetadata; expect((await h.session.sendMessage("Retry me", options)).success).toBe(true); const rows = await allRows(h); expect(rolloverRows(rows)).toHaveLength(1); @@ -1531,10 +1566,10 @@ describe("AgentSession token-budget lifecycle", () => { test("a published rollover is not repeated when its append acknowledgment fails", async () => { const h = await setup(); await seedHistory(h, 110_000); - const append = h.historyService.appendManyToHistory.bind(h.historyService); - spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce( - async (workspace, rows) => { - const result = await append(workspace, rows); + const append = h.historyService.acceptCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementationOnce( + async (...args) => { + const result = await append(...args); if (!result.success) throw new Error(result.error); throw new Error("directory sync failed after publication"); } @@ -1864,6 +1899,102 @@ describe("AgentSession token-budget lifecycle", () => { expect(text(rows.at(-1)!)).toBe("Later question"); }); + test.each([false, true])( + "separate queued inputs survive their own replacement retirement (reset=%s)", + async (reset) => { + const h = await setup(); + await seedHistory(h, reset ? 110_000 : 20_000); + const foreign = new CompactionCancellation( + new HistoryService(h.config).getCompactionCancellationStorage(workspaceId) + ); + await foreign.cancel({ retainUntilReplacement: true }); + for (const message of ["First after Stop", "Second after Stop"]) + h.session.queueMessage(message, options, { onAccepted: () => undefined }); + h.session.sendQueuedMessages(); + await h.waitForRequest(1); + h.settleStream(0, { finishReason: "stop" }); + await h.waitForRequest(2); + const rows = await allRows(h); + expect( + rows.filter((row) => ["First after Stop", "Second after Stop"].includes(text(row))) + ).toHaveLength(2); + expect(rolloverRows(rows)).toHaveLength(reset ? 1 : 0); + } + ); + + test("cancellation before an owned reset receipt releases its automatic caller without publication", async () => { + const h = await setup(); + await seedHistory(h, 110_000); + const controller = new AbortController(); + const canceled = mock(() => undefined); + const failed = mock(() => undefined); + const accept = h.historyService.acceptCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementation(async (...args) => { + if (args[2].kind === "append" && rolloverRows(args[2].messages).length > 0) + controller.abort(); + return accept(...args); + }); + expect( + ( + await h.session.sendMessage("Automatic reset", options, { + acceptanceOrigin: "automatic", + synthetic: true, + cancelSignal: controller.signal, + onCanceled: canceled, + onAcceptedPreStreamFailure: failed, + }) + ).success + ).toBe(true); + expect(canceled).toHaveBeenCalledTimes(1); + expect(failed).not.toHaveBeenCalled(); + expect(h.requests).toHaveLength(0); + expect(rolloverRows(await allRows(h))).toHaveLength(0); + }); + + test.each(["before", "after"] as const)( + "owned rollover cannot reauthorize queued input across a foreign Stop %s its receipt", + async (phase) => { + const h = await setup(); + await seedHistory(h, 110_000); + h.session.queueMessage("Queued before foreign Stop", options); + const foreign = new CompactionCancellation( + new HistoryService(h.config).getCompactionCancellationStorage(workspaceId) + ); + let stopped: Awaited> = null; + const stop = async () => { + await foreign.cancel({ retainUntilReplacement: true }); + stopped = await foreign.read(); + }; + const accept = h.historyService.acceptCompactionReplacement.bind(h.historyService); + let intercepted = false; + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementation(async (...args) => { + const reset = args[2].kind === "append" && rolloverRows(args[2].messages).length > 0; + if (reset && phase === "before") await stop(); + const result = await accept(...args); + if (reset) { + intercepted = true; + if (phase === "after") await stop(); + } + return result; + }); + const result = await h.session.sendMessage("Reset this window", options); + expect(result.success).toBe(false); + // A reset receipt retains accepted input, but cannot authorize provider startup + // after another backend's Stop supersedes the original request frontier. + expect(h.requests).toHaveLength(0); + await h.session.waitForIdle(); + expect(intercepted).toBe(true); + const rows = await allRows(h); + expect(rows.some((row) => text(row) === "Queued before foreign Stop")).toBe(false); + expect(rolloverRows(rows)).toHaveLength(phase === "before" ? 0 : 1); + expect(rows.filter((row) => text(row) === "Reset this window")).toHaveLength( + phase === "before" ? 0 : 1 + ); + expect(stopped).not.toBeNull(); + expect(await foreign.read()).toEqual(stopped); + } + ); + test("a user message queued behind the flush pair lands in the fresh window", async () => { const h = await setup(); expect((await h.session.sendMessage("Work", options)).success).toBe(true); @@ -2150,24 +2281,29 @@ describe("AgentSession token-budget lifecycle", () => { expect(rolloverRows(await allRows(h))).toHaveLength(0); }); - test("a Stop during flush admission degrades the flush instead of running it unsealed", async () => { + test("a Stop during flush admission refuses the automatic flush and continuation", async () => { const h = await setup(); expect((await h.session.sendMessage("Work", options)).success).toBe(true); expect(await h.requests[0].onStepSettled?.(step(110_000))).toBe("rollover"); const capture = h.aiService.captureRequestAssemblySnapshot!.bind(h.aiService); + const stopped = Promise.withResolvers(); // interruptStream clears the pending reset and its paired continuation while the flush // dispatch is still awaiting its admission checks. spyOn(h.aiService, "captureRequestAssemblySnapshot").mockImplementationOnce(async (id) => { expect((await h.session.interruptStream()).success).toBe(true); + stopped.resolve(); return capture(id); }); - await h.finishAndDispatch(); + h.settleStream(0); + await stopped.promise; + await h.session.waitForIdle(); const rows = await allRows(h); expect(warningRows(rows)).toHaveLength(0); - const trigger = rows.at(-1)!; - expect(text(trigger)).toBe("Continue"); - expect(trigger.metadata?.muxMetadata).not.toHaveProperty("contextBudgetFlush"); - expect(h.requests[1].muxMetadata).not.toHaveProperty("contextBudgetFlush"); + expect(h.requests).toHaveLength(1); + expect(rows.some((row) => row.metadata?.muxMetadata?.contextBudgetFlush)).toBe(false); + expect( + await h.historyService.getCompactionCancellationStorage(workspaceId).read() + ).not.toBeNull(); expect(h.session.hasQueuedDedupeKey(CONTEXT_CONTINUE_DEDUPE_KEY)).toBe(false); }); @@ -3568,16 +3704,15 @@ describe("AgentSession token-budget lifecycle", () => { await fs.writeFile(mentioned, "accepted original bytes\n"); await fs.utimes(mentioned, new Date(1000), new Date(1000)); await seedHistory(h, 20000); - const append = h.historyService.appendManyToHistory.bind(h.historyService); - spyOn(h.historyService, "appendManyToHistory").mockImplementation(async (id, rows) => { - const rollover = rows.some( - (row) => row.metadata?.muxMetadata?.type === "context-window-rollover" - ); + const accept = h.historyService.acceptCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "acceptCompactionReplacement").mockImplementation(async (...args) => { + const operation = args[2]; + const rollover = operation.kind === "append" && rolloverRows(operation.messages).length > 0; if (rollover) { expect(trackedFilePaths(h)).toEqual([]); if (failure === "append-failure") return Err("injected emergency append failure"); } - const result = await append(id, rows); + const result = await accept(...args); if (rollover && failure === "shutdown-after-append") h.session.beginShutdown(); return result; }); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index b646dbb26ca..b3602b09835 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1,3 +1,5 @@ +import type { CompactionHistoryDeletion } from "./compactionCancellation"; +import type { ContinuousCompactionPublication } from "./continuousCompactionJournal"; import type { QueuedInputStopCause } from "@/common/types/streamStopCause"; import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { STARTUP_RECOVERY_PROBE_TIMEOUT_MS } from "@/constants/startupRecovery"; @@ -72,6 +74,13 @@ export type { StreamErrorRecoveryOutcome } from "./turnCoordinator"; import type { StreamMessageOptions } from "@/node/services/turnRequestBuilder"; import type { HistoryService } from "@/node/services/historyService"; import type { TurnAcceptanceOrigin } from "./taskWorkspaceSeam"; +import { + CompactionCancellation, + matchesCompactionCancellation, + type CompactionCancellationReplacementWitness, + type CompactionCancellationSummary, + type CompactionReplacementCapture, +} from "./compactionCancellation"; import type { SessionUsageService } from "@/node/services/sessionUsageService"; import type { InitStateManager } from "@/node/services/initStateManager"; import type { MCPServerManager } from "@/node/services/mcpServerManager"; @@ -179,7 +188,7 @@ import { createRuntimeForWorkspace, } from "@/node/runtime/runtimeHelpers"; import { MessageQueue, cancelReasonBeforeAcceptance } from "./messageQueue"; -import type { QueueCutCutter } from "./messageQueue"; +import type { QueueCutCutter, QueuedInput } from "./messageQueue"; import { copyStreamLifecycleSnapshot, type RuntimeStatusEvent, @@ -285,6 +294,9 @@ type AgentSessionResult = | { success: true; data: T } | { success: false; error: SendMessageError; failureHandled?: true }; +// Durability failure must not hide a successful abort from the service's hard-stop cleanup. +type AgentSessionInterruptResult = Result & { streamStopped?: true }; + /** * Tracked file state for detecting external edits. * Uses timestamp-based polling with diff injection. @@ -623,6 +635,8 @@ export async function clearProviderConfigFixableAbandonMarkers( export const CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE = "Workspace history is being cleared or reset. Please wait and try again."; const SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE = "Xum is shutting down; the message was not sent."; +const EMPTY_RESUME_HISTORY_ERROR = + "Cannot resume stream: workspace history is empty. Send a new message instead."; export interface AgentSessionChatEvent { workspaceId: string; @@ -775,6 +789,7 @@ interface CachedMemoryContext { } interface SendMessageInternalOptions { + readCompactionAdmission?: () => Promise>; acceptanceOrigin?: TurnAcceptanceOrigin; preparation?: PreparationAttempt; /** A dequeued send keeps its admission owner through acceptance and startup failure. */ @@ -859,16 +874,41 @@ interface SendMessageInternalOptions { admissionStale?: () => boolean; } +function pendingCompactionSummary(message: MuxMessage): CompactionCancellationSummary | undefined { + const metadata = message.metadata?.muxMetadata; + return message.role === "assistant" && + isCompactionSummaryMetadata(metadata) && + metadata.pendingFollowUp + ? { + id: message.id, + sequence: message.metadata?.historySequence, + pendingFollowUp: { ...metadata.pendingFollowUp }, + } + : undefined; +} + +export interface CompactionStopAdmission { + isStale: () => boolean; + readCapture: () => CompactionReplacementCapture | undefined; +} + // Enqueueing creates no preparation attempt. Once dispatched, Promise success alone cannot // distinguish cancellation, a background transfer, and delivery to terminal policy. interface PreparationAttempt { + intent: "send" | "resume"; acceptanceOrigin: TurnAcceptanceOrigin; + compactionAdmissionStale: () => boolean; + admissionCapture?: CompactionReplacementCapture; + resumeReplacement?: CompactionReplacementCapture; + queuedStopAdmission?: CompactionStopAdmission; preparedRequest?: PreparedStreamMessage; owner?: TurnId; expectedTurn: TurnId; editReservation?: ReturnType; outcome: "preparing" | "background" | "delivered" | "canceled"; durability: "rollback-eligible" | "durable" | "accepted"; + /** HistoryService copies the assigned sequence here at visible publication, before durability. */ + inputPublication?: MuxMessage; queued: boolean; failureNotified: boolean; failureAttempts?: number; @@ -953,6 +993,9 @@ export class AgentSession { private readonly compactionHandler: CompactionHandler; private readonly compactionMonitor: CompactionMonitor; private readonly continuousCompactor: ContinuousCompactor; + private readonly compactionCancellation: CompactionCancellation; + private compactionStopGeneration = 0; + private pendingResumeIntent?: AbortController; private readonly retryManager: RetryManager; private lastAutoRetryResumeRequest?: AutoRetryResumeRequest; @@ -963,7 +1006,7 @@ export class AgentSession { steps: [ () => this.runStartupRecoveryStep(() => this.requireGoalAcknowledgmentForCrashRecoveredPartial()), - () => this.runStartupRecoveryStep(() => this.continuousCompactor.recover()), + () => this.runStartupRecoveryStep(() => this.recoverCompaction()), () => this.runStartupRecoveryStep(() => this.dispatchPendingFollowUp()), () => this.runStartupRecoveryStep(() => @@ -1112,12 +1155,17 @@ export class AgentSession { */ private dispatchingQueuedEntry = false; private dispatchingQueuedEntryMuxMetadata?: unknown; + private preparingQueuedInput?: { + attempt: PreparationAttempt; + read: () => QueuedInput | undefined; + }; /** Correlation of the direct send currently in the PREPARING phase, if any. */ private preparingWorkspaceTurnMetadata?: WorkspaceTurnMuxMetadata; /** Context needed to retry the current stream (cleared on stream end/abort/error). */ private activeStreamContext?: { + admissionCapture?: CompactionReplacementCapture; modelString: string; contextBudgetRetried?: boolean; requestAssemblySnapshot?: RequestAssemblySnapshot; @@ -1134,6 +1182,8 @@ export class AgentSession { }; private activeCompactionRequest?: { + admissionCapture?: CompactionReplacementCapture; + publication?: ContinuousCompactionPublication; id: string; modelString: string; options?: SendMessageOptions; @@ -1181,6 +1231,9 @@ export class AgentSession { this.workspaceId = trimmedWorkspaceId; this.config = config; this.historyService = historyService; + this.compactionCancellation = new CompactionCancellation( + historyService.getCompactionCancellationStorage(trimmedWorkspaceId) + ); this.aiService = aiService; const streamManagerCandidate = streamManager ?? aiService; assert( @@ -1412,6 +1465,7 @@ export class AgentSession { .then(async () => { cleanupExecution[Symbol.dispose](); await cleanup("drain", () => this.coordinator.drain()); + await cleanup("compaction cancellation", () => this.compactionCancellation.flush()); // Raw bridges stay attached through the attempt fence. Destructive disposal suppresses // recovery policy, but still presents its captured terminal exactly once below. for (const { event, handler } of this.aiListeners) this.aiService.off(event, handler); @@ -1624,6 +1678,7 @@ export class AgentSession { if (this.coordinator.closing || !isCurrent()) { return; } + if (await this.compactionRecoveryBlocked()) return; // Load persisted preference before scheduling retries so an on-disk opt-out is // honored even when the first failure happens before startup recovery runs. @@ -2571,6 +2626,7 @@ export class AgentSession { ): Promise { if (this.coordinator.closing) return "completed"; using _execution = this.coordinator.enterExecution(); + if (await this.compactionRecoveryBlocked()) return "completed"; const turn = this.coordinator.turnId; const generation = this.retryManager.captureGeneration(); const isCurrent = () => @@ -2754,11 +2810,21 @@ export class AgentSession { } private async readStartupRecoveryState(signal: AbortSignal): Promise { + // Child startup now uses this probe instead of the root recovery scheduler. A durable + // Stop remains authoritative here too; uncertain publication must stay retryable. + const generation = this.compactionStopGeneration; + const canceled = () => + signal.aborted || + generation !== this.compactionStopGeneration || + this.compactionCancellation.blocksRecovery; + const cancellation = await this.readCompactionCancellation(); + if (canceled()) return "blocked"; + if (cancellation) return "stopped"; await this.loadAutoRetryState(); - if (signal.aborted) return "blocked"; + if (canceled()) return "blocked"; if (this.autoRetryEnabledPreference === false) return "stopped"; const [partial, history] = (await this.readStartupTail(true)) ?? []; - if (!history?.success || partial === undefined) return "blocked"; + if (canceled() || !history?.success || partial === undefined) return "blocked"; const abandon = this.startupAutoRetryAbandon; if (abandon?.reason === "aborted") { // Accepted synthetic guidance is new intent too; snapshots/notices are not. @@ -3309,8 +3375,19 @@ export class AgentSession { return Err(createUnknownSendMessageError(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE)); if (internal?.preparation) return this.prepareMessage(message, options, internal, internal.preparation); + if (!internal?.readCompactionAdmission) { + const admission = this.historyService.captureCompactionReplacement(this.workspaceId, { + onRepaired: () => this.clearUsageState(), + replaceUnreadable: (internal?.acceptanceOrigin ?? "manual") === "manual", + }); + internal = { ...internal, readCompactionAdmission: () => admission }; + } const attempt: PreparationAttempt = { + intent: "send", acceptanceOrigin: internal?.acceptanceOrigin ?? "manual", + compactionAdmissionStale: this.captureCompactionAdmission( + internal?.acceptanceOrigin ?? "manual" + ), owner: internal?.turnReservation, expectedTurn: this.coordinator.turnId, outcome: "preparing", @@ -3348,6 +3425,20 @@ export class AgentSession { throw error; } finally { try { + if (this.preparingQueuedInput?.attempt === attempt) { + // A dequeued manual send can fail even after Send Now refreshes its Stop admission. + // Restore unpublished input before IDLE lets its successor overwrite the draft. + if ( + !this.coordinator.closing && + attempt.durability === "rollback-eligible" && + (attempt.compactionAdmissionStale() || + (attempt.failure != null && + attempt.inputPublication?.metadata?.historySequence === undefined && + this.preparingQueuedInput.read() != null)) + ) + this.restoreQueueToInput(); + if (this.preparingQueuedInput?.attempt === attempt) this.preparingQueuedInput = undefined; + } if (attempt.outcome !== "background" && attempt.owner != null) this.coordinator.finishPreparation(attempt.owner); } finally { @@ -3406,9 +3497,11 @@ export class AgentSession { assert(typeof message === "string", "sendMessage requires a string message"); const isManualUserMessage = internal?.synthetic !== true; + const manualReplacement = attempt.acceptanceOrigin === "manual"; // Single admission-staleness predicate for all three turn-admission gates below. const isAdmissionStale = () => + attempt.compactionAdmissionStale() || internal?.admissionEpochStale?.() === true || internal?.admissionStale?.() === true || !this.coordinator.isCurrentTurn(attempt.owner ?? attempt.expectedTurn) || @@ -3422,14 +3515,70 @@ export class AgentSession { const cancelSignal = internal?.cancelSignal; const persistedCancelableMessageIds: string[] = []; - // All prepared rows share publication bookkeeping while retaining their current write - // order and rollback checkpoints. Prefixes alone never establish trigger acceptance. + const stagedPrefixes: MuxMessage[] = []; + let replacementCapture: CompactionReplacementCapture | undefined; + let replacementCommitted = false; + // Manual prefixes and their trigger form one accepted replacement. Optional context alone + // must never retire Stop, and cancellation after the receipt cannot erase the user's row. const publishPreparedHistory = async ( publication: | { kind: "prefix"; message: MuxMessage } | { kind: "trigger"; messages: MuxMessage[] } ): Promise> => { const messages = publication.kind === "prefix" ? [publication.message] : publication.messages; + const resetBatch = [...stagedPrefixes, ...messages].some( + (row) => row.metadata?.contextBoundaryKind === "reset" + ); + // Only reset prefixes wait for their actual trigger. Ordinary A prefixes retain their + // existing publication/rollback path; a reset alone cannot authorize replacement. + if (manualReplacement || resetBatch) { + if (publication.kind === "prefix") { + stagedPrefixes.push(...messages); + return Ok(undefined); + } + attempt.inputPublication = messages.at(-1); + const capture = replacementCapture ?? attempt.admissionCapture; + assert(capture, "Publication requires its original admission capture"); + const batch = [...stagedPrefixes, ...messages]; + const accepted = await this.historyService + .acceptCompactionReplacement( + this.workspaceId, + capture, + { + kind: "append", + messages: batch, + ...(!manualReplacement ? { preserveCancellation: true as const } : {}), + }, + { + isCurrent: () => + !isAdmissionStale() && !shutdownRefusesBeforePersist() && !cancelSignal?.aborted, + onContextResetCommitted: (predecessor, successor) => { + this.advanceOwnedCompactionAdmission(predecessor, successor); + if (attempt.admissionCapture) Object.assign(attempt.admissionCapture, successor); + }, + onCommitted: () => { + if (manualReplacement) { + replacementCommitted = true; + attempt.durability = "durable"; + } else persistedCancelableMessageIds.push(...batch.map((row) => row.id)); + return undefined; + }, + } + ) + .catch((error: unknown) => Err(getErrorMessage(error))); + if (!accepted.success) return accepted; + if (accepted.data.kind !== "accepted") { + if (await cancelBeforeAcceptance()) return Ok(undefined); + return Err(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE); + } + // Retirement takes the same lock; join it only after publication releases that lock. + if (manualReplacement) { + await this.retireCompactionReplacement(accepted.data.witness, attempt.admissionCapture); + if ((internal?.preTurnMessages?.length ?? 0) > 0) internal?.onPreTurnRowsPersisted?.(); + } + return Ok(undefined); + } + if (await this.isAutomaticSendBlocked()) return Err(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE); // Preserve the same rollback checkpoints for unexpected service rejection as for // ordinary Result failures; earlier prefixes may already be durable. const result = await ( @@ -3450,6 +3599,7 @@ export class AgentSession { * counting against the sender's budget. */ const rollbackPersistedTurnRows = async (): Promise => { + if (replacementCommitted) return false; if (persistedCancelableMessageIds.length === 0) return true; this.continuousCompactor.reset("delete-messages"); const rollbackResult = await this.historyService.deleteMessages( @@ -3510,10 +3660,39 @@ export class AgentSession { return true; }; + const frontier = await (internal?.readCompactionAdmission?.() ?? + Promise.resolve( + attempt.admissionCapture + ? Ok(attempt.admissionCapture) + : Err("Preparation has no original admission frontier.") + )); + if (!frontier.success) return Err(createUnknownSendMessageError(frontier.error)); + attempt.admissionCapture = frontier.data; + if (await cancelBeforeAcceptance()) { return Ok(undefined); } + if (manualReplacement) { + await this.readCompactionCancellation("manual"); + const stopAdmission = attempt.queuedStopAdmission; + const captured = stopAdmission + ? (() => { + const capture = stopAdmission.readCapture(); + return capture ? Ok(capture) : Err("The initiating Stop has no publication receipt."); + })() + : frontier; + if (!captured.success) return Err(createUnknownSendMessageError(captured.error)); + replacementCapture = captured.data; + attempt.admissionCapture = replacementCapture; + } else if (await this.isAutomaticSendBlocked()) { + return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + } + if (isAdmissionStale()) + return refuseBeforeAcceptance( + createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + ); + // Last-line-of-defence pricing gate: every dispatch path (initial sends, // sendQueuedMessages, dispatchPendingFollowUp, // post-compaction follow-ups) lands here, so a budgeted goal that became @@ -3548,6 +3727,9 @@ export class AgentSession { message, options, pricingGate.error, + attempt, + replacementCapture, + isAdmissionStale, internal?.enqueuedAtMs ); // The user has explicitly intervened, so the goal-safety contract @@ -3759,7 +3941,10 @@ export class AgentSession { if (this.coordinator.phase !== "completing") { // MUST use abandonPartial=true to prevent handleAbort from performing partial compaction // with mismatched history (since we're about to truncate it). - const stopResult = await this.interruptStream({ abandonPartial: true }); + const stopResult = await this.interruptStream({ + abandonPartial: true, + preserveCompactionIntent: true, + }); if (!stopResult.success) { log.warn("Failed to interrupt stream before edit", { workspaceId: this.workspaceId, @@ -3809,9 +3994,23 @@ export class AgentSession { const truncateTargetId = await this.getEditTruncateTargetId(editMessageId); this.clearUsageState(); + const editCapture = replacementCapture; const truncateResult = await this.historyService.truncateAfterMessage( this.workspaceId, - truncateTargetId + truncateTargetId, + editCapture + ? { + replacement: { + capture: editCapture, + isCurrent: () => !isAdmissionStale(), + // Only this edit's held-lock fence can refresh its original capture. + onGenerationAdvanced: (generation) => { + replacementCapture = { ...editCapture, generation }; + attempt.admissionCapture = replacementCapture; + }, + }, + } + : undefined ); if (!truncateResult.success) { const isMissingEditTarget = @@ -3829,7 +4028,8 @@ export class AgentSession { } else { return Err(createUnknownSendMessageError(truncateResult.error)); } - } else { + } + if (truncateResult.success) { editTailTruncated = true; // RLM mode: summarize the truncated tail into a durable labeled row // BEFORE the edited user message is appended and this turn's request is @@ -3964,6 +4164,9 @@ export class AgentSession { message, options, error, + attempt, + replacementCapture, + isAdmissionStale, internal?.enqueuedAtMs ); // Rejection does not cancel the user's intervention; match the pricing gate's safety. @@ -4011,7 +4214,7 @@ export class AgentSession { const providersConfigForCompaction = this.getProvidersConfigSafe(); // Recover before measuring pressure so the old pre-swap usage cannot force another fold. - if (await this.continuousCompactor.recover()) this.clearUsageState(); + if (await this.recoverCompaction()) this.clearUsageState(); const compactionResult = this.compactionMonitor.checkBeforeSend({ model: modelForStream, usage: this.getUsageState(), @@ -4030,7 +4233,7 @@ export class AgentSession { ); if (!continuousContext.enabled) this.continuousCompactor.reset("disabled"); const continuousResult = continuousContext.enabled - ? await this.continuousCompactor.observe(compactionResult.usagePercentage, { + ? await this.observeCompaction(compactionResult.usagePercentage, { ...continuousContext, phase: "on-send", }) @@ -4045,7 +4248,13 @@ export class AgentSession { (continuousContext.enabled ? continuousResult === "fallback" && compactionResult.shouldForceCompact : compactionResult.usagePercentage >= compactionResult.thresholdPercentage); - if (shouldCompactBeforeSend) { + // A new boundary would hide the summary needed to retire scoped Stop debt. + // Keep ordinary input flowing, but defer legacy compaction until cleanup succeeds. + // An explicit replacement instead publishes its witness before compaction can hide debt. + if ( + shouldCompactBeforeSend && + (manualReplacement || !(await this.compactionRecoveryBlocked())) + ) { this.continuousCompactor.reset("legacy-fallback"); const followUpFileParts = effectiveFileParts?.map((part) => ({ url: part.url, @@ -4369,6 +4578,7 @@ export class AgentSession { } catch (error) { return Err(createUnknownSendMessageError(getErrorMessage(error))); } + if (await cancelBeforeAcceptance()) return Ok(undefined); if (contextRollover) { const sequences = [batch[0], batch[1], userMessage].map( (row) => row.metadata?.historySequence @@ -4418,10 +4628,7 @@ export class AgentSession { // leaves no trace for a later human resume to replay into provider context. Past this point // rollback is forbidden by design (goal sync observes the durable row), so a Stop landing in // the remaining pre-stream awaits refuses the turn at the PREPARING gate with rows retained. - if ( - internal?.admissionStale?.() === true || - !this.coordinator.isCurrentTurn(attempt.owner ?? attempt.expectedTurn) - ) { + if (isAdmissionStale()) { const rolledBack = await rollbackPersistedTurnRows(); // Probe-carrying sends are peer messages whose caller already returned success when the // entry was queued — the cancellation hook is their only way to observe this refusal and @@ -4784,10 +4991,12 @@ export class AgentSession { options: SendMessageOptions, internal?: { acceptanceOrigin?: TurnAcceptanceOrigin; + readCompactionAdmission?: () => Promise>; agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string; retrySignal?: AbortSignal; + preparationSignal?: AbortSignal; requestAssemblySnapshot?: RequestAssemblySnapshot; contextBudgetRetried?: boolean; } @@ -4796,13 +5005,23 @@ export class AgentSession { if (this.coordinator.closing || internal?.retrySignal?.aborted) return Ok({ started: false }); using _execution = this.coordinator.enterExecution(); const expectedTurnId = this.coordinator.turnId; + const manualReplacement = (internal?.acceptanceOrigin ?? "manual") === "manual"; + using resumeIntent = + manualReplacement && !internal?.preparationSignal ? this.beginResumeIntent() : undefined; + const preparationSignal = internal?.preparationSignal ?? resumeIntent?.signal; + const stopGeneration = this.compactionStopGeneration; // Cancel retry startup through pricing/history/provider preparation, then detach the link. // Disabling backoff after delivery must not abort the already-running stream. - const startupController = internal?.retrySignal ? new AbortController() : undefined; - const cancelStartup = () => startupController?.abort(); + const startupController = new AbortController(); + const cancelStartup = () => startupController.abort(); internal?.retrySignal?.addEventListener("abort", cancelStartup, { once: true }); + preparationSignal?.addEventListener("abort", cancelStartup, { once: true }); + if (preparationSignal?.aborted) cancelStartup(); using _retryCancellation = { - [Symbol.dispose]: () => internal?.retrySignal?.removeEventListener("abort", cancelStartup), + [Symbol.dispose]: () => { + internal?.retrySignal?.removeEventListener("abort", cancelStartup); + preparationSignal?.removeEventListener("abort", cancelStartup); + }, }; assert(options, "resumeStream requires options"); @@ -4819,6 +5038,18 @@ export class AgentSession { return Ok({ started: false }); } + const admission = await (internal?.readCompactionAdmission?.() ?? + this.historyService.captureCompactionReplacement(this.workspaceId, { + onRepaired: () => this.clearUsageState(), + replaceUnreadable: (internal?.acceptanceOrigin ?? "manual") === "manual", + })); + if (!admission.success) return Err(createUnknownSendMessageError(admission.error)); + let replacementCapture: CompactionReplacementCapture | undefined; + if (manualReplacement) { + await this.readCompactionCancellation("manual"); + replacementCapture = admission.data; + } else if (await this.compactionRecoveryBlocked()) return Ok({ started: false }); + if (this.workspaceGoalService) { const pricingGate = await this.workspaceGoalService.assertPricedModelForBudgetedGoal( this.workspaceId, @@ -4839,16 +5070,21 @@ export class AgentSession { if ( this.coordinator.admissionBlocked || this.coordinator.closing || + stopGeneration !== this.compactionStopGeneration || startupController?.signal.aborted ) { return Ok({ started: false }); } const attempt: PreparationAttempt = { + intent: "resume", acceptanceOrigin: internal?.acceptanceOrigin ?? "manual", + admissionCapture: admission.data, + resumeReplacement: replacementCapture, + compactionAdmissionStale: () => stopGeneration !== this.compactionStopGeneration, expectedTurn: expectedTurnId, outcome: "preparing", - durability: "accepted", + durability: manualReplacement ? "rollback-eligible" : "accepted", queued: false, failureNotified: false, }; @@ -5121,10 +5357,53 @@ export class AgentSession { } } + private advanceOwnedCompactionAdmission( + predecessor: CompactionReplacementCapture, + successor: CompactionReplacementCapture, + preparing?: CompactionReplacementCapture + ): void { + this.messageQueue.advanceCompactionAdmission(predecessor, successor); + for (const capture of [ + preparing, + this.activeStreamContext?.admissionCapture, + this.activeCompactionRequest?.admissionCapture, + ]) { + if (capture?.nonce === predecessor.nonce && capture.generation === predecessor.generation) + Object.assign(capture, successor); + } + } + private async appendContextRolloverRows( rows: MuxMessage[], - publish: () => Promise> = () => - this.historyService.appendManyToHistory(this.workspaceId, rows) + publish: () => Promise> = async () => { + const context = this.activeStreamContext; + const capture = context?.admissionCapture; + const turn = this.coordinator.turnId; + const operation = this.coordinator.operationId; + if (!capture) return Err("Rollover has no original admission frontier."); + const accepted = await this.historyService.acceptCompactionReplacement( + this.workspaceId, + capture, + { kind: "append", messages: rows, preserveCancellation: true }, + { + isCurrent: () => + this.activeStreamContext === context && + this.coordinator.isCurrentTurn(turn) && + this.coordinator.isCurrentOperation(operation) && + !this.coordinator.closing && + !this.coordinator.admissionBlocked, + onCommitted: () => undefined, + onContextResetCommitted: (predecessor, successor) => { + this.advanceOwnedCompactionAdmission(predecessor, successor); + }, + } + ); + return !accepted.success + ? accepted + : accepted.data.kind === "accepted" + ? Ok(undefined) + : Err(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE); + } ): Promise> { let appended: Result; try { @@ -5966,6 +6245,7 @@ export class AgentSession { /** Sealed tool-end continuation shared by the mid-stream and resume-hydration paths. */ private enqueueContextBudgetContinuation(args: { + admissionCapture?: CompactionReplacementCapture; text: string; dedupeKey: string; options: SendMessageOptions; @@ -5985,6 +6265,13 @@ export class AgentSession { args.dedupeKey, { acceptanceOrigin: "automatic", + // This is continuation of the admitted stream, not newly authored input. + readCompactionAdmission: () => + Promise.resolve( + args.admissionCapture + ? Ok(args.admissionCapture) + : Err("Continuation has no original admission frontier.") + ), synthetic: true, agentInitiated: true, sealed: true, @@ -6133,6 +6420,7 @@ export class AgentSession { // `contextBudgetFlush` flag, independent of these send options. const enqueue = (text: string, dedupeKey: string, flush: boolean) => this.enqueueContextBudgetContinuation({ + admissionCapture: context.admissionCapture, text, dedupeKey, options: streamOptions, @@ -6194,6 +6482,9 @@ export class AgentSession { message: string, options: (SendMessageOptions & { fileParts?: FilePart[] }) | undefined, rejection: SendMessageError, + attempt: PreparationAttempt, + capture: CompactionReplacementCapture | undefined, + isAdmissionStale: () => boolean, enqueuedAtMs?: number ): Promise { if (this.coordinator.disposed) { @@ -6234,10 +6525,35 @@ export class AgentSession { rejection.type === "context_budget_blocked" || rejection.type === "context_budget_exceeded" ? createContextBudgetRejectedMessage(userMessage) : userMessage; - const appendResult = await this.historyService.appendToHistory( - this.workspaceId, - persistedMessage - ); + // A sequence is allocated before append opens the file. Only the publication receipt + // distinguishes a visible rejected input from a draft that still needs restoration. + attempt.inputPublication = persistedMessage; + let appendResult: Result; + if (capture) { + const accepted = await this.historyService.acceptCompactionReplacement( + this.workspaceId, + capture, + { kind: "append", messages: [persistedMessage], preserveCancellation: true }, + { + isCurrent: () => !isAdmissionStale() && !this.coordinator.closing, + onCommitted: () => { + attempt.durability = "durable"; + return undefined; + }, + } + ); + appendResult = accepted.success + ? accepted.data.kind === "accepted" + ? Ok(undefined) + : Err(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + : accepted; + } else { + appendResult = await this.historyService.appendToHistory( + this.workspaceId, + persistedMessage + ); + } + if (appendResult.success) attempt.durability = "durable"; if (!appendResult.success) { log.warn("Failed to persist user message after pre-stream gate rejection", { workspaceId: this.workspaceId, @@ -6644,7 +6960,7 @@ export class AgentSession { ), providersConfig: this.getProvidersConfigSafe(), }); - const result = await this.continuousCompactor.observe(usage.usagePercentage, { + const result = await this.observeCompaction(usage.usagePercentage, { ...context, phase: "stream-end", }); @@ -6856,6 +7172,7 @@ export class AgentSession { return; } using _execution = this.coordinator.enterExecution(); + const admissionStale = this.captureCompactionAdmission("automatic"); const streamContext = this.activeStreamContext; if (!streamContext?.modelString || !streamContext.options) { @@ -6881,7 +7198,7 @@ export class AgentSession { } await this.waitForIdle(); - if (this.coordinator.disposed) { + if (this.coordinator.disposed || admissionStale()) { return; } @@ -6902,6 +7219,7 @@ export class AgentSession { reason: "mid-stream", }); + if (admissionStale()) return; const autoCompactionRequest = this.buildAutoCompactionRequest({ followUpContent, baseOptions: streamContext.options, @@ -6916,10 +7234,12 @@ export class AgentSession { }, { acceptanceOrigin: "automatic", + admissionStale, synthetic: true, agentInitiated: autoCompactionRequest.agentInitiated, } ); + if (admissionStale()) return; if (!sendResult.success) { log.warn("Failed to dispatch mid-stream compaction request", { workspaceId: this.workspaceId, @@ -6974,11 +7294,143 @@ export class AgentSession { }; } + /** Capture before service pricing awaits; a Stop must refuse that older request. */ + captureCompactionAdmission(origin: TurnAcceptanceOrigin): () => boolean { + if (origin === "manual") this.pendingResumeIntent?.abort(); + const generation = this.compactionStopGeneration; + return () => generation !== this.compactionStopGeneration; + } + + beginResumeIntent(): { signal: AbortSignal; [Symbol.dispose](): void } { + this.pendingResumeIntent?.abort(); + const controller = (this.pendingResumeIntent = new AbortController()); + return { + signal: controller.signal, + [Symbol.dispose]: () => { + if (this.pendingResumeIntent === controller) this.pendingResumeIntent = undefined; + }, + }; + } + + async cancelCompaction( + retainUntilReplacement = false, + settled?: Promise, + options?: { + fullHistoryDeletion?: CompactionHistoryDeletion; + onCaptured?: (capture: CompactionReplacementCapture) => void; + } + ): Promise> { + this.compactionStopGeneration++; + this.pendingResumeIntent?.abort(); + this.coordinator.abandonCompaction(); + this.continuousCompactor.reset("user-interrupt"); + // cancel installs the blocking debt synchronously, before interruption or storage awaits. + try { + await this.compactionCancellation.cancel({ + retainUntilReplacement, + settled, + fullHistoryDeletion: options?.fullHistoryDeletion, + onCaptured: options?.onCaptured, + }); + return Ok(undefined); + } catch (error) { + return Err(getErrorMessage(error)); + } + } + + private async retireCompactionReplacement( + witness: CompactionCancellationReplacementWitness | null, + preparing?: CompactionReplacementCapture + ): Promise { + if (!witness) return; + await this.compactionCancellation + .retireReplacement(witness, (predecessor, successor) => { + this.advanceOwnedCompactionAdmission(predecessor, successor, preparing); + }) + .catch((error: unknown) => { + log.warn("Accepted replacement retains compaction cancellation cleanup debt", { error }); + }); + } + + private async readCompactionCancellation(origin: TurnAcceptanceOrigin = "automatic") { + // A witnessed unlink failure is ancillary, but later admissions still retry its cleanup. + if (this.compactionCancellation.needsPersistence && !this.compactionCancellation.blocksRecovery) + await this.compactionCancellation.retry().catch((error: unknown) => { + log.warn("Compaction cancellation cleanup retry failed", { error }); + }); + const revision = this.compactionCancellation.repairRevision; + const record = await (origin === "manual" + ? this.compactionCancellation.readForReplacement() + : this.compactionCancellation.read()); + if (revision !== this.compactionCancellation.repairRevision) this.clearUsageState(); + if (origin === "manual") return record; + if (!record || this.compactionCancellation.blocksRecovery) return record; + const witness = await this.historyService.findCompactionReplacementWitness( + this.workspaceId, + record.nonce + ); + if (!witness.success) throw new Error(witness.error); + if (!witness.data) return record; + await this.retireCompactionReplacement(witness.data); + // A newer Stop during verification/cleanup still owns recovery admission. + return this.compactionCancellation.read(); + } + + async isAutomaticSendBlocked(): Promise { + const record = await this.readCompactionCancellation(); + if (this.compactionCancellation.blocksRecovery || record?.retainUntilReplacement) return true; + // Scoped Stop can distinguish its canceled handoff from later automatic input. + // Keep that handoff identifiable before any fresh request can hide the summary. + if (record?.scope.kind === "unresolved") { + const history = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); + if (!history.success) throw new Error(history.error); + // A Stop admitted during the read can be waiting for this caller's terminal policy. + if (this.compactionCancellation.blocksRecovery) return true; + const summary = history.data.map(pendingCompactionSummary).findLast((entry) => entry != null); + if (summary) await this.compactionCancellation.narrow(record.nonce, summary); + // An unresolved Stop has no durable settlement proof. Refuse fresh automatic input + // before acceptance so it cannot create a compaction continuation that recovery blocks. + // Explicit manual replacement remains the recovery path for this conservative layer. + else return true; + } + return this.compactionCancellation.blocksRecovery; + } + + private async compactionRecoveryBlocked(): Promise { + const record = await this.readCompactionCancellation(); + return this.compactionCancellation.blocksRecovery || record !== null; + } + + private async recoverCompaction(): Promise { + const generation = this.compactionStopGeneration; + return ( + !(await this.compactionRecoveryBlocked()) && + generation === this.compactionStopGeneration && + this.continuousCompactor.recover() + ); + } + + private async observeCompaction(...args: Parameters) { + const generation = this.compactionStopGeneration; + if ((await this.compactionRecoveryBlocked()) || generation !== this.compactionStopGeneration) + return "none" as const; + return this.continuousCompactor.observe(...args); + } + async interruptStream(options?: { soft?: boolean; abandonPartial?: boolean; - }): Promise> { + preserveCompactionIntent?: boolean; + onCompactionCanceled?: (capture: CompactionReplacementCapture) => void; + }): Promise { this.assertNotDisposed("interruptStream"); + const settled = Promise.withResolvers(); + const cancellation = + options?.soft || options?.preserveCompactionIntent + ? undefined + : this.cancelCompaction(false, settled.promise, { + onCaptured: options?.onCompactionCanceled, + }); // Send-now callers may replace the turn immediately after this returns. Capture // its settlement before any await so the old abort reaches accounting and the // renderer before replacement PREPARING invalidates its operation identity. @@ -6999,15 +7451,22 @@ export class AgentSession { this.activeToolCallIds.clear(); } - const stopResult = await this.streamManager.stopStream(this.workspaceId, { - ...options, - abortReason: "user", - }); + const stopResult = await this.streamManager + .stopStream(this.workspaceId, { + ...options, + abortReason: "user", + }) + .then(async (result) => { + if (result.success) await interruptedPolicy; + return result; + }) + .finally(() => settled.resolve()); + const canceled = await cancellation; if (!stopResult.success) { return Err(stopResult.error); } - await interruptedPolicy; + if (canceled && !canceled.success) return { ...canceled, streamStopped: true }; return Ok(undefined); } @@ -7092,9 +7551,11 @@ export class AgentSession { preparation?: PreparationAttempt, contextBudgetRetried = false, requestAssemblySnapshot?: RequestAssemblySnapshot, - admittedRequest?: PreparedStreamMessage + admittedRequest?: PreparedStreamMessage, + admissionCapture = preparation?.admissionCapture ?? this.activeStreamContext?.admissionCapture ): Promise> { const preparedRequest = admittedRequest ?? preparation?.preparedRequest; + const previousCompactionRequest = this.activeCompactionRequest; const fail = ( error: SendMessageError, acpPromptId?: string, @@ -7108,10 +7569,18 @@ export class AgentSession { preStartErrors, preparation ); + const refuseRejectedResume = (message: MuxMessage) => { + this.activeStreamUserMessageId = message.id; + return fail({ + type: "context_budget_blocked", + message: "Cannot retry a rejected request. Edit it or send a new message instead.", + }); + }; // Re-read at every pre-stream checkpoint below: dispose or shutdown can land while a // recovery-initiated stream (which carries no abortSignal) awaits commitPartial, file-change // detection, or history reads, and must not reach the provider afterwards. const isStreamStartAborted = (): boolean => + preparation?.compactionAdmissionStale() === true || !this.coordinator.isCurrentTurn(turn) || this.coordinator.closing || abortSignal?.aborted === true; @@ -7143,6 +7612,7 @@ export class AgentSession { this.activeStreamHadPostCompactionInjection = false; const providersConfig = this.getProvidersConfigSafe(); this.activeStreamContext = { + admissionCapture, modelString, contextBudgetRetried, requestAssemblySnapshot, @@ -7164,6 +7634,42 @@ export class AgentSession { return Ok(undefined); } + if (preparation?.resumeReplacement) { + // Stamp the actual committed tail before notices or a CONTINUE sentinel can invent + // something resumable. Never search backward past an ineligible final row. + const tail = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); + if (!tail.success) return await fail(createUnknownSendMessageError(tail.error)); + const retryRequest = this.findLastRetryUserMessage(tail.data); + if (retryRequest?.metadata?.contextBudgetRejected) + return await refuseRejectedResume(retryRequest); + const target = tail.data.at(-1); + if (!target) return await fail(createUnknownSendMessageError(EMPTY_RESUME_HISTORY_ERROR)); + if (target.metadata?.contextBudgetRejected) return await refuseRejectedResume(target); + if (target.role !== "assistant" && !this.shouldUseUserMessageForRetry(target)) + return Ok(undefined); + const accepted = await this.historyService.acceptCompactionReplacement( + this.workspaceId, + preparation.resumeReplacement, + { kind: "resume", message: target }, + { + isCurrent: () => !isStreamStartAborted(), + onCommitted: () => { + preparation.durability = "accepted"; + return undefined; + }, + } + ); + if (!accepted.success) return await fail(createUnknownSendMessageError(accepted.error)); + if (accepted.data.kind !== "accepted") return Ok(undefined); + await this.retireCompactionReplacement(accepted.data.witness, preparation.admissionCapture); + } else if ( + preparation?.acceptanceOrigin === "automatic" && + (await (preparation.intent === "resume" + ? this.compactionRecoveryBlocked() + : this.isAutomaticSendBlocked())) + ) + return Ok(undefined); + // Detect external file edits (timestamp-based polling) BEFORE reading history // and append the notification as a durable row. The // provider request is built purely from chat.jsonl, so anything the model @@ -7203,11 +7709,7 @@ export class AgentSession { const lastUserMessage = this.findLastRetryUserMessage(historyResult.data); if (lastUserMessage?.metadata?.contextBudgetRejected) { - this.activeStreamUserMessageId = lastUserMessage.id; - return await fail({ - type: "context_budget_blocked", - message: "Cannot retry a rejected request. Edit it or send a new message instead.", - }); + return await refuseRejectedResume(lastUserMessage); } let resumedFlushCannotWrite = false; @@ -7284,6 +7786,7 @@ export class AgentSession { if (this.messageQueue.isEmpty()) { const { contextBudgetFlush: _flush, ...continuationMetadata } = flushMuxMetadata; this.enqueueContextBudgetContinuation({ + admissionCapture, text: "Continue", dedupeKey: CONTEXT_CONTINUE_DEDUPE_KEY, options, @@ -7310,11 +7813,7 @@ export class AgentSession { let requestMessages = filterOrphanedMcpPromptSnapshots(historyResult.data); if (requestMessages.length === 0) { - return await fail( - createUnknownSendMessageError( - "Cannot resume stream: workspace history is empty. Send a new message instead." - ) - ); + return await fail(createUnknownSendMessageError(EMPTY_RESUME_HISTORY_ERROR)); } // Structural invariant: API requests must not end with a non-partial assistant message. @@ -7355,6 +7854,16 @@ export class AgentSession { modelString, options ); + if (this.activeCompactionRequest) { + this.activeCompactionRequest.admissionCapture = admissionCapture; + // Completion must retain the request's original publication generation, including + // captured absence. Retrying the same request cannot adopt a later Stop's frontier. + this.activeCompactionRequest.publication = admissionCapture + ? { generation: admissionCapture.generation } + : previousCompactionRequest?.id === this.activeCompactionRequest.id + ? previousCompactionRequest.publication + : undefined; + } if (isStreamStartAborted()) { return Ok(undefined); @@ -7463,7 +7972,43 @@ export class AgentSession { const startRequest = preparedRequest ? preparedRequest.start.bind(preparedRequest) : this.aiService.streamMessage.bind(this.aiService); + // Revalidate the recorded admission; this read never grants a newer Stop's authority. + // The same check travels past runtime preparation to the engine's provider-start gate. + const assertAdmissionCurrent = admissionCapture + ? async () => { + const current = await this.historyService.captureCompactionReplacement( + this.workspaceId + ); + if (isStreamStartAborted()) return; + if (!current.success) throw new Error(current.error); + if ( + current.data.nonce !== admissionCapture.nonce || + current.data.generation !== admissionCapture.generation + ) + throw new Error(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE); + } + : undefined; + const withAdmissionCurrent: StreamMessageOptions["withAdmissionCurrent"] = admissionCapture + ? async (construct) => { + const result = await this.historyService.runWithCompactionAdmission( + this.workspaceId, + admissionCapture, + () => { + if (!isStreamStartAborted()) construct(); + } + ); + if (!result.success && !isStreamStartAborted()) throw new Error(result.error); + } + : undefined; + try { + await assertAdmissionCurrent?.(); + } catch (error) { + return await fail(createUnknownSendMessageError(getErrorMessage(error))); + } + if (isStreamStartAborted()) return Ok(undefined); const streamResult = await startRequest({ + assertAdmissionCurrent, + withAdmissionCurrent, messages: requestMessages, workspaceId: this.workspaceId, modelString, @@ -7887,7 +8432,13 @@ export class AgentSession { retryAgentInitiated, undefined, retryGoalKind, - retryGoalId + retryGoalId, + undefined, + undefined, + undefined, + undefined, + undefined, + context.admissionCapture ); } finally { if (this.coordinator.isCurrentTurn(preparedTurn)) { @@ -8021,7 +8572,9 @@ export class AgentSession { undefined, undefined, context.contextBudgetRetried, - context.requestAssemblySnapshot + context.requestAssemblySnapshot, + undefined, + context.admissionCapture ); } finally { if (this.coordinator.isCurrentTurn(preparedTurn)) { @@ -8240,7 +8793,8 @@ export class AgentSession { undefined, true, rolled.data.snapshot, - rolled.data.request + rolled.data.request, + context.admissionCapture ); } finally { if (this.coordinator.isCurrentTurn(preparedTurn)) { @@ -8311,10 +8865,16 @@ export class AgentSession { this.clearQueue(); } - await this.handleStreamFailureForAutoRetry({ - type: failureType, - message: data.error, - }); + try { + await this.handleStreamFailureForAutoRetry({ + type: failureType, + message: data.error, + }); + } catch (error) { + // Uncertain cancellation forbids this retry, but terminal error cleanup must still finish. + // Startup callers keep the rejection so their recovery checkpoint remains retryable. + log.warn("Terminal auto-retry unavailable", { workspaceId: this.workspaceId, error }); + } if (!this.coordinator.isCurrentTurn(turn) || !this.coordinator.isCurrentOperation(operation)) return; await this.updateStartupAutoRetryAbandonFromFailure(failureType, failedUserMessageId); @@ -8543,7 +9103,9 @@ export class AgentSession { const handled = await this.compactionHandler.handleCompletion( streamEndPayload, completedCompactionRequest?.id, - () => this.coordinator.isCurrentTurn(turn) && this.coordinator.isCurrentOperation(operation) + () => + this.coordinator.isCurrentTurn(turn) && this.coordinator.isCurrentOperation(operation), + completedCompactionRequest?.publication ); if (!this.coordinator.isCurrentTurn(turn) || !this.coordinator.isCurrentOperation(operation)) return; @@ -8866,7 +9428,7 @@ export class AgentSession { ) return; await this.runContinuousCompactionObservation(async (token) => { - const result = await this.continuousCompactor.observe(0, { + const result = await this.observeCompaction(0, { ...this.getContinuousCompactionContext(context.modelString, context.options), phase: "mid-stream", }); @@ -8938,7 +9500,7 @@ export class AgentSession { // One usage handler owns the eventual resume; observe itself shares its // latch result, which must not dispatch the continuation twice. const observed = await this.runContinuousCompactionObservation(async (token) => { - const result = await this.continuousCompactor.observe(usagePercent, { + const result = await this.observeCompaction(usagePercent, { ...continuousContext, phase: "mid-stream", }); @@ -9250,9 +9812,30 @@ export class AgentSession { onPreTurnRowsPersisted?: () => void; /** Caller staleness probe re-checked at this entry's dispatch admission. */ admissionStale?: () => boolean; + compactionAdmissionStale?: () => boolean; + readCompactionAdmission?: () => Promise>; + /** Refresh only this entry's Stop capture for an explicit manual Send now. */ + refreshCompactionAdmission?: ( + isStale: () => boolean, + capture?: CompactionReplacementCapture + ) => void; } ): "tool-end" | "turn-end" | null { this.assertNotDisposed("queueMessage"); + if (internal?.dedupeKey != null && this.messageQueue.hasDedupeKey(internal.dedupeKey)) + return null; + if (!internal?.readCompactionAdmission) { + const admission = (async () => { + // Direct queue callers own disk acquisition even if the entry is cleared before + // dispatch. Disposal must join its repair/write before releasing the session. + using _capture = this.coordinator.enterExecution(); + return await this.historyService.captureCompactionReplacement(this.workspaceId, { + onRepaired: () => this.clearUsageState(), + replaceUnreadable: (internal?.acceptanceOrigin ?? "manual") === "manual", + }); + })().catch((error: unknown) => Err(getErrorMessage(error))); + internal = { ...internal, readCompactionAdmission: () => admission }; + } const didEnqueue = internal?.dedupeKey != null ? this.messageQueue.addOnce(message, options, internal.dedupeKey, internal) @@ -9260,6 +9843,8 @@ export class AgentSession { if (!didEnqueue) { return null; } + // A newly authored manual turn replaces a resume still waiting on preflight I/O. + if ((internal?.acceptanceOrigin ?? "manual") === "manual") this.pendingResumeIntent?.abort(); this.emitQueuedMessageChanged(); // Signal to bash_output that it should return early to process queued messages // only for tool-end dispatches. Return the same mode so the caller's foreground @@ -9647,28 +10232,35 @@ export class AgentSession { */ restoreQueueToInput(): void { this.assertNotDisposed("restoreQueueToInput"); - if (this.messageQueue.isEmpty()) { - return; - } - - const queuedMessages = this.messageQueue.getVisibleMessages(); - const displayText = this.messageQueue.getVisibleDisplayText(); - const fileParts = this.messageQueue.getVisibleFileParts(); - const reviews = this.messageQueue.getVisibleReviews(); - const hasVisibleContent = - queuedMessages.length > 0 || fileParts.length > 0 || (reviews?.length ?? 0) > 0; + const preparing = this.preparingQueuedInput; + const interrupted = + preparing?.attempt.durability === "rollback-eligible" && + // Complete bytes may survive a failed flush without granting a durable acceptance receipt. + preparing.attempt.inputPublication?.metadata?.historySequence === undefined && + (preparing.attempt.compactionAdmissionStale() || preparing.attempt.failure != null) + ? preparing.read() + : undefined; + if (interrupted) this.preparingQueuedInput = undefined; + const inputs = [interrupted, this.messageQueue.getInputForRestore()].filter( + (input) => input != null + ); + if (this.messageQueue.isEmpty() && inputs.length === 0) return; // Clear everything: synthetic wake callbacks need cancellation so their durable // records do not retry after the user explicitly interrupted the workspace. this.clearQueue(); - if (hasVisibleContent) { + if (inputs.length > 0) { + const reviews = inputs.flatMap((input) => input.reviews ?? []); this.emitChatEvent({ type: "restore-to-input", workspaceId: this.workspaceId, - text: displayText, - fileParts, - reviews, + text: inputs + .map((input) => input.text) + .filter((text) => text.length > 0) + .join("\n"), + fileParts: inputs.flatMap((input) => input.fileParts ?? []), + reviews: reviews.length > 0 ? reviews : undefined, }); } } @@ -9691,12 +10283,12 @@ export class AgentSession { * Dispatch the next user-authored queued entry immediately. Hidden synthetic * entries remain queued behind it and resume through the normal drain lifecycle. */ - sendNextUserQueuedMessage(): boolean { + sendNextUserQueuedMessage(stopAdmission?: CompactionStopAdmission): boolean { this.assertNotDisposed("sendNextUserQueuedMessage"); if (!this.messageQueue.prioritizeNextUserEntry()) { return false; } - this.sendQueuedMessages("send-immediately"); + this.sendQueuedMessages("send-immediately", stopAdmission); return true; } @@ -9722,7 +10314,10 @@ export class AgentSession { * Send queued messages if any exist. * Called when the current turn ends or the user chooses to send immediately. */ - sendQueuedMessages(trigger: QueueDrainTrigger = "terminal"): void { + sendQueuedMessages( + trigger: QueueDrainTrigger = "terminal", + stopAdmission?: CompactionStopAdmission + ): void { if ( this.coordinator.closing || this.coordinator.editBlocked() || @@ -9737,7 +10332,9 @@ export class AgentSession { } const expectedTurnId = this.coordinator.turnId; const attempt: PreparationAttempt = { + intent: "send", acceptanceOrigin: candidate.acceptanceOrigin, + compactionAdmissionStale: this.captureCompactionAdmission(candidate.acceptanceOrigin), expectedTurn: expectedTurnId, outcome: "preparing", durability: "rollback-eligible", @@ -9762,6 +10359,7 @@ export class AgentSession { if (this.messageQueue.peekNext()?.identity !== candidate.identity) return Ok(undefined); attempt.queued = true; const { message, options, internal, enqueuedAtMs } = this.messageQueue.dequeueNext(); + this.preparingQueuedInput = { attempt, read: candidate.inputForRestore }; attempt.acceptanceOrigin = internal?.acceptanceOrigin ?? "manual"; attempt.onFailure = internal?.onAcceptedPreStreamFailure; this.dispatchingQueuedEntry = true; @@ -9774,6 +10372,24 @@ export class AgentSession { createUnknownSendMessageError("Queued preparation was retired before dispatch.") ); } + if (trigger === "send-immediately" && attempt.acceptanceOrigin === "manual") { + // Send now replaces the Stop it just issued, while caller cancellation and + // automatic queue entries keep their original authority. This attempt's own + // capture still refuses a second Stop during PREPARING or publication. + const admission = stopAdmission?.isStale ?? attempt.compactionAdmissionStale; + attempt.compactionAdmissionStale = admission; + attempt.queuedStopAdmission = stopAdmission; + const captured = stopAdmission?.readCapture(); + if (captured) internal?.refreshCompactionAdmission?.(admission, captured); + else if (!stopAdmission) { + const fresh = await this.historyService.captureCompactionReplacement(this.workspaceId, { + onRepaired: () => this.clearUsageState(), + replaceUnreadable: (internal?.acceptanceOrigin ?? "manual") === "manual", + }); + if (!fresh.success) return Err(createUnknownSendMessageError(fresh.error)); + internal?.refreshCompactionAdmission?.(admission, fresh.data); + } + } this.backgroundProcessManager.setMessageQueued( this.workspaceId, this.messageQueue.getNextDispatchableMode() === "tool-end" @@ -9873,7 +10489,20 @@ export class AgentSession { return false; } using _execution = this.coordinator.enterExecution(); - + // Recovery keeps the Stop identity from entry; its later send must not appear fresh. + const stopGeneration = this.compactionStopGeneration; + const resumeCanceled = () => + stopGeneration !== this.compactionStopGeneration || cancelResume?.() === true; + + const canceled = await this.readCompactionCancellation(); + // Stop can be waiting for this policy to settle before its final cleanup. + // Do not join that same mutation from automatic continuation dispatch. + if (this.compactionCancellation.blocksRecovery) return false; + const canceledScope = canceled?.scope.kind === "summary" ? canceled.scope : undefined; + const isCanceledSummary = (message: MuxMessage) => + canceledScope?.id === message.id && + canceledScope?.sequence === message.metadata?.historySequence; + summaryMessageId ??= canceledScope?.id; let summaryMessage: MuxMessage | undefined; if (summaryMessageId) { const historyResult = await this.historyService.getHistoryFromLatestBoundary( @@ -9898,10 +10527,15 @@ export class AgentSession { const onlyTailCopiesAfterSummary = historyResult.data .slice(summaryIndex + 1) .every((message) => message.metadata?.rlmPreservedTailCopy === true); - if (!onlyTailCopiesAfterSummary) { + summaryMessage = historyResult.data[summaryIndex]; + const pending = pendingCompactionSummary(summaryMessage); + if ( + !onlyTailCopiesAfterSummary && + !(canceled && pending && matchesCompactionCancellation(canceled, pending)) && + !(pending == null && isCanceledSummary(summaryMessage)) + ) { return false; } - summaryMessage = historyResult.data[summaryIndex]; } else { // Read the last message from history — only need 1 message, avoid full-file read. // Startup recovery must retry on transient read failures, so bubble errors. @@ -9949,12 +10583,34 @@ export class AgentSession { const muxMeta = lastMessage.metadata?.muxMetadata; if (!isCompactionSummaryMetadata(muxMeta) || !muxMeta.pendingFollowUp) { + if ( + canceled && + !canceled.retainUntilReplacement && + isCanceledSummary(lastMessage) && + isCompactionSummaryMetadata(muxMeta) && + muxMeta.pendingFollowUp === undefined && + (await this.clearPendingFollowUpFromSummary(lastMessage, "confirm-cleared")) + ) + await this.compactionCancellation.retire(canceled.nonce); + return false; + } + + const summary = pendingCompactionSummary(lastMessage); + if (canceled && summary && matchesCompactionCancellation(canceled, summary)) { + // The history read may have admitted a newer Stop that is waiting for this policy. + if (this.compactionCancellation.blocksRecovery) return false; + await this.compactionCancellation.narrow(canceled.nonce, summary); + // Skipped cleanup proves neither removal nor replacement of the durable handoff. + if (await this.clearPendingFollowUpFromSummary(lastMessage)) + await this.compactionCancellation.retire(canceled.nonce); return false; } + if (this.compactionCancellation.blocksRecovery || canceled?.retainUntilReplacement) + return false; // A user can abandon after the boundary commits but before its continuation // dispatches. Keep the fold, but remove the crash-recoverable resume intent. - if (cancelResume?.()) { + if (resumeCanceled()) { await this.clearPendingFollowUpFromSummary(lastMessage); return false; } @@ -10078,13 +10734,8 @@ export class AgentSession { this.hasExternalSendPreflight?.() === true || (this.isBusy() && this.coordinator.phase !== "completing") : undefined; - const followUpAdmissionStale = - idleRuleStale != null || goalAdmissionStale != null || cancelResume != null - ? () => - idleRuleStale?.() === true || - goalAdmissionStale?.() === true || - cancelResume?.() === true - : undefined; + const followUpAdmissionStale = () => + resumeCanceled() || idleRuleStale?.() === true || goalAdmissionStale?.() === true; log.debug("Dispatching pending follow-up from compaction summary", { workspaceId: this.workspaceId, @@ -10164,7 +10815,7 @@ export class AgentSession { return false; } - if (cancelResume?.()) { + if (resumeCanceled()) { await this.clearPendingFollowUpFromSummary(lastMessage); return false; } @@ -10200,14 +10851,14 @@ export class AgentSession { admissionStale: followUpAdmissionStale, }); if (!sendResult.success) { - if (cancelResume?.()) { + if (resumeCanceled()) { await this.clearPendingFollowUpFromSummary(lastMessage); return false; } // A stale-admission refusal is the idle rule (or a goal transition) // working as intended, not a recovery failure: route it through the // same skip path as the pre-send check instead of throwing. - if (followUpAdmissionStale?.() === true) { + if (followUpAdmissionStale()) { log.info("Pending follow-up refused at send admission; skipping it", { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, @@ -10281,7 +10932,10 @@ export class AgentSession { } } - private async clearPendingFollowUpFromSummary(summaryMessage: MuxMessage): Promise { + private async clearPendingFollowUpFromSummary( + summaryMessage: MuxMessage, + action: "clear" | "confirm-cleared" = "clear" + ): Promise { assert( summaryMessage.role === "assistant", "clearPendingFollowUpFromSummary requires an assistant summary message" @@ -10293,20 +10947,21 @@ export class AgentSession { "clearPendingFollowUpFromSummary requires compaction-summary metadata" ); - if (!muxMeta.pendingFollowUp) { - return; + if (!muxMeta.pendingFollowUp && action !== "confirm-cleared") { + return false; } const turn = this.coordinator.turnId; const updateResult = await this.historyService.cleanupCompactionFollowUp( this.workspaceId, summaryMessage, - "clear", + action, () => this.coordinator.turnId === turn ); if (!updateResult.success) { throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); } + return updateResult.data === "applied"; } /** @@ -11045,6 +11700,17 @@ export class AgentSession { pendingFollowUp: CompactionFollowUpRequest; }): Promise> { this.assertNotDisposed("appendHeartbeatContextResetBoundary"); + const admissionStale = this.captureCompactionAdmission("automatic"); + const captured = await this.historyService.captureCompactionReplacement(this.workspaceId); + if (!captured.success) return Err(captured.error); + if ( + (await this.isAutomaticSendBlocked()) || + // Ordinary automatic input preserves scoped debt. A reset would archive its summary, + // making the pending handoff unreachable to active-boundary recovery after restart. + (await this.readCompactionCancellation())?.scope.kind === "summary" || + admissionStale() + ) + return Err(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE); if (this.isBusy()) { return Err("Cannot reset heartbeat context while a turn is active."); @@ -11057,7 +11723,9 @@ export class AgentSession { const result = await this.compactionHandler.appendHeartbeatContextResetBoundary({ boundaryText: params.boundaryText, pendingFollowUp: params.pendingFollowUp, + publication: { generation: captured.data.generation }, isCurrent: () => + !admissionStale() && this.coordinator.isCurrentTurn(turn) && !this.isBusy() && !this.hasQueuedMessages() && diff --git a/src/node/services/agentSession.turnCompletion.test.ts b/src/node/services/agentSession.turnCompletion.test.ts index aecdd6c8862..9fb9d5fd504 100644 --- a/src/node/services/agentSession.turnCompletion.test.ts +++ b/src/node/services/agentSession.turnCompletion.test.ts @@ -5,6 +5,7 @@ import { Exit, Scope } from "effect"; import { defaultEffectRunner as runner } from "./di/effectRunner"; import { log } from "./log"; import { EventEmitter } from "events"; +import { promises as fileIO } from "node:fs"; import * as fs from "node:fs/promises"; import * as path from "node:path"; import { CONTINUOUS_COMPACTION_GENERATION_FILE } from "@/constants/continuousCompaction"; @@ -65,7 +66,130 @@ function observePolicy(session: AgentSession) { } describe("AgentSession turn completion", () => { - test("ordinary completion drains queued input despite an unreadable compaction generation", async () => { + test("startup retries a pending continuation after a transient cancellation read failure", async () => { + const h = await createAgentSessionHarness({ workspaceId, captureEvents: true }); + const storage = h.historyService.getCompactionCancellationStorage(workspaceId); + const stream = spyOn(h.aiService, "streamMessage"); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "summary", { + compactionBoundary: true, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "pending continuation", model, agentId: "exec" }, + }, + }) + ); + const open = fileIO.open; + let failed = false; + const reading = spyOn(fileIO, "open").mockImplementation( + async (...args: Parameters) => { + if (args[0] === storage.path && !failed) { + failed = true; + throw Object.assign(new Error("temporary cancellation read failure"), { code: "EIO" }); + } + return open(...args); + } + ); + try { + await h.session.runStartupRecovery(); + expect(failed).toBe(true); + expect(stream).not.toHaveBeenCalled(); + // The failed startup step must remain retryable; the later run dispatches real pending input. + await h.session.runStartupRecovery(); + expect(stream).toHaveBeenCalledTimes(1); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.at(-1)?.parts).toMatchObject([ + { type: "text", text: "pending continuation" }, + ]); + } finally { + reading.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } + }); + + test.each(["EACCES", "EIO"])( + "terminal error settles while cancellation storage fails with %s", + async (code) => { + const completion = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const emitter = new EventEmitter(); + const h = await createAgentSessionHarness({ + workspaceId, + aiEmitter: emitter, + captureEvents: true, + aiServiceOverrides: { + streamMessage: mock(() => { + start(emitter); + return Promise.resolve( + Ok({ messageId: "assistant-1", completion: completion.promise }) + ); + }), + }, + }); + const consumer = observePolicy(h.session); + const stream = spyOn(h.aiService, "streamMessage"); + const storage = h.historyService.getCompactionCancellationStorage(workspaceId); + const failure = Object.assign(new Error(`${code}: cancellation read unavailable`), { code }); + const open = fileIO.open; + let reading: ReturnType> | undefined; + const warning = spyOn(log, "warn"); + try { + expect(await h.session.sendMessage("original", sendOptions)).toEqual(Ok(undefined)); + expect(await h.session.cancelCompaction(true)).toEqual(Ok(undefined)); + const cancellationBytes = await fileIO.readFile(storage.path); + reading = spyOn(fileIO, "open").mockImplementation( + async (...args: Parameters) => { + if (args[0] === storage.path) { + entered.resolve(); + await release.promise; + throw failure; + } + return open(...args); + } + ); + const streamError = { + messageId: "assistant-1", + error: "provider failed", + errorType: "api" as const, + }; + emitter.emit("error", { ...streamError, workspaceId }); + completion.resolve({ status: "failed", streamError }); + await entered.promise; + expect(internal(h.session).coordinator.phase).toBe("completing"); + release.resolve(); + await policyPromise(consumer); + expect(internal(h.session).coordinator.phase).toBe("idle"); + expect(await h.session.waitForPendingStreamErrorRecoveryDecision("assistant-1")).toBe( + "terminal" + ); + expect(h.events.filter((event) => event.type === "stream-error")).toMatchObject([ + { messageId: "assistant-1", error: "provider failed" }, + ]); + expect(stream).toHaveBeenCalledTimes(1); + expect(await fileIO.readFile(storage.path)).toEqual(cancellationBytes); + expect( + warning.mock.calls.some((args) => + args.some( + (arg) => + typeof arg === "object" && arg != null && "error" in arg && arg.error === failure + ) + ) + ).toBe(true); + } finally { + release.resolve(); + reading?.mockRestore(); + warning.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } + } + ); + + test("ordinary completion drains queued input despite a transient compaction generation read failure", async () => { const completion = Promise.withResolvers(); const nextStarted = Promise.withResolvers(); const emitter = new EventEmitter(); @@ -94,23 +218,43 @@ describe("AgentSession turn completion", () => { const consumer = observePolicy(h.session); const observation = spyOn(internal(h.session), "observeContinuousCompactionAtStreamEnd"); const accounting = spyOn(internal(h.session), "recordGoalAccountingFromUsage"); + let reading: ReturnType> | undefined; + let failed = false; try { - expect((await h.session.sendMessage("original", sendOptions)).success).toBe(true); - const firstPolicy = policyPromise(consumer); - h.session.queueMessage("queued follow-up", sendOptions); - // A compaction-only sidecar must not strand ordinary completion policy or its queue. + // Keep admission's real frontier stable: only the ordinary completion read fails. const generationPath = path.join( h.config.sessionsDir, workspaceId, CONTINUOUS_COMPACTION_GENERATION_FILE ); - await fs.mkdir(generationPath); + const generation = "existing-generation"; + await fs.mkdir(path.dirname(generationPath), { recursive: true }); + await fs.writeFile(generationPath, generation); + expect((await h.session.sendMessage("original", sendOptions)).success).toBe(true); + const firstPolicy = policyPromise(consumer); + const admission = await h.historyService.captureCompactionReplacement(workspaceId); + expect(admission.success).toBe(true); + h.session.queueMessage("queued follow-up", sendOptions, { + readCompactionAdmission: () => Promise.resolve(admission), + }); + // Install the transient fault only after the queue's recorded acquisition completes. + const read = fileIO.readFile; + reading = spyOn(fileIO, "readFile").mockImplementation((async ( + ...args: Parameters + ) => { + if (args[0] === generationPath && !failed) { + failed = true; + throw Object.assign(new Error("temporary generation read failure"), { code: "EIO" }); + } + return read(...args); + }) as typeof read); completion.resolve({ status: "completed", streamEnd: end() }); await firstPolicy; expect(h.session.hasQueuedMessages()).toBe(false); expect(observation).toHaveBeenCalledTimes(1); expect(accounting).toHaveBeenCalledTimes(1); await nextStarted.promise; + expect(failed).toBe(true); expect(calls).toBe(2); expect(h.session.isBusy()).toBe(true); expect(h.events.filter((event) => event.type === "stream-end")).toHaveLength(1); @@ -119,8 +263,9 @@ describe("AgentSession turn completion", () => { expect(history.data.at(-1)?.parts).toMatchObject([ { type: "text", text: "queued follow-up" }, ]); - expect((await fs.stat(generationPath)).isDirectory()).toBe(true); + expect(await fs.readFile(generationPath, "utf8")).toBe(generation); } finally { + reading?.mockRestore(); completion.resolve({ status: "completed", streamEnd: end() }); observation.mockRestore(); accounting.mockRestore(); diff --git a/src/node/services/aiService.test.ts b/src/node/services/aiService.test.ts index 6897bc32523..831c8330c59 100644 --- a/src/node/services/aiService.test.ts +++ b/src/node/services/aiService.test.ts @@ -1,3 +1,4 @@ +import nodeAssert from "node:assert/strict"; import { eventSpine, type RequestAssembleContext } from "./events/eventSpine"; // Bun test file - doesn't support Jest mocking, so we skip this test for now // These tests would need to be rewritten to work with Bun's test runner @@ -17,6 +18,7 @@ import { ProviderModelFactory, } from "./providerModelFactory"; import { HistoryService } from "./historyService"; +import { CompactionCancellation } from "./compactionCancellation"; import { InitStateManager } from "./initStateManager"; import { ProviderService } from "./providerService"; import { EXPERIMENT_IDS } from "@/common/constants/experiments"; @@ -1271,6 +1273,116 @@ describe("AIService.streamMessage compaction boundary slicing", () => { mock.restore(); }); + it.each([false, true])( + "carries final recorded admission into the engine (prepared=%s)", + async (prepared) => { + using xumHome = new DisposableTempDir("ai-service-final-admission"); + const metadata = createLocalWorkspaceMetadata("final-admission", xumHome.path); + const harness = createHarness(xumHome.path, metadata); + const previewFence = mock((_construct: () => void) => Promise.resolve()); + const finalFence = mock((_construct: () => void) => + Promise.reject(new Error("final construction frontier superseded")) + ); + const previewCheck = mock(() => Promise.resolve()); + const finalCheck = mock(() => Promise.reject(new Error("original frontier superseded"))); + const options = { + workspaceId: metadata.id, + messages: [createMuxMessage("admitted-user", "user", "continue")], + modelString: "openai:gpt-5.2", + }; + if (prepared) { + const candidate = await harness.service.prepareStreamMessage({ + ...options, + assertAdmissionCurrent: previewCheck, + withAdmissionCurrent: previewFence, + }); + if (!candidate.success) throw new Error(JSON.stringify(candidate.error)); + await using request = candidate.data; + expect( + ( + await request.start({ + ...options, + assertAdmissionCurrent: finalCheck, + withAdmissionCurrent: finalFence, + }) + ).success + ).toBe(true); + } else { + expect( + ( + await harness.service.streamMessage({ + ...options, + assertAdmissionCurrent: finalCheck, + withAdmissionCurrent: finalFence, + }) + ).success + ).toBe(true); + } + expect(harness.startStreamCalls).toHaveLength(1); + const gate = harness.startStreamCalls[0].assertAdmissionCurrent; + expect(gate).toBeDefined(); + nodeAssert(gate); + await nodeAssert.rejects(gate(), /original frontier superseded/); + expect(finalCheck).toHaveBeenCalledTimes(1); + expect(previewCheck).not.toHaveBeenCalled(); + const fence = harness.startStreamCalls[0].withAdmissionCurrent; + nodeAssert(fence); + const construct = mock(() => undefined); + await nodeAssert.rejects(fence(construct), /final construction frontier superseded/); + expect(finalFence).toHaveBeenCalledTimes(1); + expect(previewFence).not.toHaveBeenCalled(); + expect(construct).not.toHaveBeenCalled(); + } + ); + + it.each([false, true])( + "mock playback retains its awaited admission check (stale=%s)", + async (stale) => { + using xumHome = new DisposableTempDir("ai-service-mock-admission"); + const { service, historyService } = createBasicAIService(xumHome.path); + service.enableMockMode(); + const player = service.mockAiStreamPlayer; + nodeAssert(player); + const workspaceId = "mock-admission"; + const captured = await historyService.captureCompactionReplacement(workspaceId); + nodeAssert(captured.success); + if (stale) + await new CompactionCancellation( + historyService.getCompactionCancellationStorage(workspaceId) + ).cancel(); + let checked = false; + const play = spyOn(player, "play").mockImplementation(async () => { + expect(checked).toBe(true); + expect( + ( + await historyService.appendToHistory( + workspaceId, + createMuxMessage("played", "assistant", "done") + ) + ).success + ).toBe(true); + return { success: true, data: undefined }; + }); + const result = await service.streamMessage({ + workspaceId, + messages: [createMuxMessage("mock-input", "user", "hello")], + modelString: "openai:gpt-5.2", + assertAdmissionCurrent: async () => { + const current = await historyService.captureCompactionReplacement(workspaceId); + nodeAssert(current.success); + if ( + current.data.nonce !== captured.data.nonce || + current.data.generation !== captured.data.generation + ) + throw new Error("Mock admission superseded"); + checked = true; + }, + }); + expect(result.success).toBe(!stale); + expect(play).toHaveBeenCalledTimes(stale ? 0 : 1); + } + ); + it.each(["request-row", "idle-row", "agent", "send-metadata", "ordinary", "historical"] as const)( "keeps oversized compaction recovery outside token-budget preflight: %s", async (identity) => { diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 66796bd7fac..b121aa42b62 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -949,6 +949,9 @@ export class AIService extends EventEmitter { if (combinedAbortSignal.aborted) { return Ok(this.createAbortedTurnHandle(syntheticMessageId, combinedAbortSignal)); } + if (!combinedAbortSignal.aborted) await opts.assertAdmissionCurrent?.(); + if (combinedAbortSignal.aborted) + return Ok(this.createAbortedTurnHandle(syntheticMessageId, combinedAbortSignal)); const result = await this.mockAiStreamPlayer.play(messages, workspaceId, { model: modelString, agentId, @@ -999,6 +1002,9 @@ export class AIService extends EventEmitter { return buildOutcome.result; } + // Prepared candidates must use the final caller's admission, not their earlier preview. + buildOutcome.turnExecutionOptions.assertAdmissionCurrent = opts.assertAdmissionCurrent; + buildOutcome.turnExecutionOptions.withAdmissionCurrent = opts.withAdmissionCurrent; const startStreamStartedAt = Date.now(); const streamResult = await this.streamManager.startStream(buildOutcome.turnExecutionOptions); recordStartupPhaseTiming("startStreamMs", startStreamStartedAt); diff --git a/src/node/services/compactionCancellation.storage.test.ts b/src/node/services/compactionCancellation.storage.test.ts index b4eed9d0c9c..5416b7b35e4 100644 --- a/src/node/services/compactionCancellation.storage.test.ts +++ b/src/node/services/compactionCancellation.storage.test.ts @@ -1,15 +1,17 @@ import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; import * as fs from "node:fs/promises"; import * as nodeFs from "node:fs"; import callbackFs from "node:fs"; import * as path from "node:path"; -import { createMuxMessage } from "@/common/types/message"; +import { createMuxMessage, type MuxMessage } from "@/common/types/message"; import { SESSION_HISTORY_MAX_LINE_BYTES } from "@/common/constants/contextBudget"; import { CONTINUOUS_COMPACTION_GENERATION_FILE } from "@/constants/continuousCompaction"; import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; import { workspaceFileLocks } from "@/node/utils/concurrency/workspaceFileLocks"; import { HistoryService } from "./historyService"; +import { CompactionHandler } from "./compactionHandler"; import { HISTORY_APPEND_PROVENANCE_FILE } from "./historyAppendProvenance"; import { createTestHistoryService } from "./testHistoryService"; import { historyWriteLockPath, removeSessionDirUnderMemoryLocks } from "./workspaceRemoval"; @@ -252,6 +254,294 @@ describe("inactive real cancellation storage", () => { } }); + it("publishes legacy-readable neutralization before the Stop sidecar", async () => { + const summaryRow = createMuxMessage("stopped-summary", "assistant", "summary", { + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp() }, + }); + expect((await h.historyService.appendToHistory(workspaceId, summaryRow)).success).toBe(true); + expect( + (await h.historyService.writePartial(workspaceId, { ...summaryRow, id: "partial" })).success + ).toBe(true); + const committed = Promise.withResolvers(); + const release = Promise.withResolvers(); + const remove = nodeFs.promises.rm; + spyOn(nodeFs.promises, "rm").mockImplementation(async (file, options) => { + if (String(file).startsWith(`${storage.path}.continuous-`)) { + committed.resolve(); + await release.promise; + } + await remove(file, options); + }); + const stopping = state.cancel(); + try { + await committed.promise; + expect(await storage.read()).not.toBeNull(); + // Simulate downgrade's lock-free read after a crash at this publication boundary. + const rows = (await fs.readFile(path.join(sessionDir, "chat.jsonl"), "utf8")) + .trimEnd() + .split("\n"); + const last = JSON.parse(rows.at(-1)!) as MuxMessage; + const partial = JSON.parse( + await fs.readFile(path.join(sessionDir, "partial.json"), "utf8") + ) as MuxMessage; + expect(last.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + expect(partial.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + } finally { + release.resolve(); + await stopping; + } + }); + + it.each([ + ["followUpContent", true], + ["continueMessage", true], + ["followUpContent", false], + ["continueMessage", false], + ] as const)( + "late completion cannot republish stopped %s before settlement cleanup (captured=%s)", + async (field, captured) => { + const request = createMuxMessage("compact-request", "user", "summarize", { + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { [field]: followUp() }, + }, + }); + expect((await h.historyService.appendToHistory(workspaceId, request)).success).toBe(true); + const admission = await foreign.captureCompactionReplacement(workspaceId); + assert(admission.success); + const authored = await fs.readFile(path.join(sessionDir, "chat.jsonl")); + const initialCleanup = Promise.withResolvers(); + const settled = Promise.withResolvers(); + const neutralize = h.historyService.neutralizeCompactionRecoveryUnderHistoryLock.bind( + h.historyService + ); + spyOn( + h.historyService, + "neutralizeCompactionRecoveryUnderHistoryLock" + ).mockImplementationOnce(async (...args) => { + const result = await neutralize(...args); + initialCleanup.resolve(); + return result; + }); + const stopping = state.cancel({ settled: settled.promise }); + try { + await initialCleanup.promise; + const handler = new CompactionHandler({ + workspaceId, + historyService: foreign, + sessionDir, + emitter: new EventEmitter(), + }); + expect( + await handler.handleCompletion( + { + type: "stream-end", + workspaceId, + messageId: "late-summary", + parts: [{ type: "text", text: "A completed summary" }], + metadata: { model: "test:model", duration: 1 }, + }, + request.id, + () => true, + captured ? { generation: admission.data.generation } : undefined + ) + ).toBe(false); + const history = await foreign.getLastMessages(workspaceId, 1); + assert(history.success); + expect(history.data[0]?.metadata?.muxMetadata).not.toHaveProperty("pendingFollowUp"); + expect(await fs.readFile(path.join(sessionDir, "chat.jsonl"))).toEqual(authored); + } finally { + settled.resolve(); + await stopping; + } + } + ); + + it("settlement still waits and cleans late recovery", async () => { + const entered = Promise.withResolvers(); + const settled = Promise.withResolvers(); + const neutralize = h.historyService.neutralizeCompactionRecoveryUnderHistoryLock.bind( + h.historyService + ); + const cleanup = spyOn( + h.historyService, + "neutralizeCompactionRecoveryUnderHistoryLock" + ).mockImplementation(async (...args) => { + const result = await neutralize(...args); + entered.resolve(); + return result; + }); + const stopping = state.cancel({ settled: settled.promise }); + try { + await entered.promise; + expect(state.blocksRecovery).toBe(true); + expect(cleanup).toHaveBeenCalledTimes(1); + expect( + ( + await foreign.writePartial( + workspaceId, + createMuxMessage("late", "assistant", "summary", { + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp() }, + }) + ) + ).success + ).toBe(true); + expect(cleanup).toHaveBeenCalledTimes(1); + settled.resolve(); + expect(await stopping).toBe("applied"); + expect(cleanup).toHaveBeenCalledTimes(2); + expect((await foreign.readPartial(workspaceId))?.metadata?.muxMetadata).toEqual({ + type: "compaction-summary", + }); + expect(await storage.read()).toMatchObject({ version: 1, scope: { kind: "unresolved" } }); + } finally { + settled.resolve(); + await stopping; + } + }); + + it.each([ + ["nonce", false], + ["generation", false], + ["nonce", true], + ["generation", true], + ] as const)( + "post-settlement cleanup cannot overwrite a newer %s (retained=%s)", + async (successorKind, retainUntilReplacement) => { + const firstCleanup = Promise.withResolvers(); + const settled = Promise.withResolvers(); + const neutralize = h.historyService.neutralizeCompactionRecoveryUnderHistoryLock.bind( + h.historyService + ); + spyOn(h.historyService, "neutralizeCompactionRecoveryUnderHistoryLock").mockImplementation( + async (...args) => { + const result = await neutralize(...args); + firstCleanup.resolve(); + return result; + } + ); + const stopping = state.cancel({ retainUntilReplacement, settled: settled.promise }); + try { + await firstCleanup.promise; + expect(state.blocksRecovery).toBe(true); + // These real writes acquire the same locks: waiting for physical settlement must release them. + if (successorKind === "nonce") { + await new CompactionCancellation( + new FileCompactionCancellationStorage(foreign, workspaceId) + ).cancel({ retainUntilReplacement: true }); + } else { + await foreign.withCompactionStorageLock(workspaceId, async (_dir, checkLock) => { + await foreign + .getContinuousCompactionJournal(workspaceId) + .advanceGenerationUnderHistoryLock(undefined, checkLock); + }); + } + const successor = await storage.read(); + const partial = createMuxMessage("successor", "assistant", "new summary", { + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp("new request") }, + }); + expect((await foreign.writePartial(workspaceId, partial)).success).toBe(true); + const before = await foreign.readPartial(workspaceId); + settled.resolve(); + expect(await stopping).toBe("superseded"); + expect(await foreign.readPartial(workspaceId)).toEqual(before); + expect(await storage.read()).toEqual(successor); + } finally { + settled.resolve(); + await stopping; + } + } + ); + + it.each(["first absent", "first existing", "second"] as const)( + "failed cleanup preserves %s durable frontier and retries exact debt", + async (phase) => { + if (phase === "first existing") await state.cancel(); + const predecessor = await storage.read(); + const initial = Promise.withResolvers(); + const settled = Promise.withResolvers(); + const neutralize = h.historyService.neutralizeCompactionRecoveryUnderHistoryLock.bind( + h.historyService + ); + let calls = 0; + const cleanup = spyOn( + h.historyService, + "neutralizeCompactionRecoveryUnderHistoryLock" + ).mockImplementation(async (...args) => { + if (++calls === (phase === "second" ? 2 : 1)) { + initial.resolve(); + throw new Error("legacy cleanup unavailable"); + } + return await neutralize(...args); + }); + const stopping = state.cancel({ + retainUntilReplacement: true, + settled: settled.promise, + onCaptured: () => initial.resolve(), + }); + const failed = assert.rejects(stopping, /legacy cleanup unavailable/); + try { + await initial.promise; + const durable = await storage.read(); + if (phase === "second") expect(durable).toMatchObject({ retainUntilReplacement: true }); + else expect(durable).toEqual(predecessor); + expect( + ( + await foreign.writePartial( + workspaceId, + createMuxMessage("late", "assistant", "summary", { + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp() }, + }) + ) + ).success + ).toBe(true); + settled.resolve(); + await failed; + expect(state.needsPersistence).toBe(true); + expect(state.blocksRecovery).toBe(true); + expect(await storage.read()).toEqual(durable); + cleanup.mockRestore(); + expect(await state.retry()).toBe("applied"); + expect(state.needsPersistence).toBe(false); + if (phase === "second") expect(await storage.read()).toEqual(durable); + else expect((await storage.read())?.nonce).not.toBe(predecessor?.nonce); + expect((await foreign.readPartial(workspaceId))?.metadata?.muxMetadata).toEqual({ + type: "compaction-summary", + }); + } finally { + settled.resolve(); + await failed; + } + } + ); + + it("a failed prepublication cleanup cannot retry over a foreign successor", async () => { + const failure = spyOn( + h.historyService, + "neutralizeCompactionRecoveryUnderHistoryLock" + ).mockRejectedValueOnce(new Error("cleanup unavailable")); + const captured = mock(() => undefined); + await assert.rejects(state.cancel({ onCaptured: captured }), /cleanup unavailable/); + expect(captured).not.toHaveBeenCalled(); + expect(await storage.read()).toBeNull(); + failure.mockRestore(); + const successor = new CompactionCancellation( + new FileCompactionCancellationStorage(foreign, workspaceId) + ); + expect(await successor.cancel({ retainUntilReplacement: true })).toBe("applied"); + const successorBytes = await fs.readFile(storage.path); + const row = createMuxMessage("foreign-summary", "assistant", "new summary", { + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp("new request") }, + }); + expect((await foreign.appendToHistory(workspaceId, row)).success).toBe(true); + const before = await fs.readFile(path.join(sessionDir, "chat.jsonl")); + expect(await state.retry()).toBe("superseded"); + expect(await fs.readFile(storage.path)).toEqual(successorBytes); + expect(await fs.readFile(path.join(sessionDir, "chat.jsonl"))).toEqual(before); + expect(captured).not.toHaveBeenCalled(); + }); + it.each(["publish", "narrow", "confirm", "retire"] as const)( "reports the exact %s receipt before cleanup or lock release", async (phase) => { @@ -330,10 +620,15 @@ describe("inactive real cancellation storage", () => { const release = Promise.withResolvers(); const open = fs.open; let fail = true; + let targetCommitted = phase !== "publish"; + const rename = nodeFs.renameSync; + spyOn(nodeFs, "renameSync").mockImplementation((from, to) => { + rename(from, to); + if (to === storage.path) targetCommitted = true; + }); spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { const handle = await open(...args); - if (args[0] === sessionDir && fail) { - fail = false; + if (args[0] === sessionDir && fail && targetCommitted) { spyOn(handle, "sync").mockImplementation(async () => { syncing.resolve(); await release.promise; @@ -363,6 +658,7 @@ describe("inactive real cancellation storage", () => { } expect(state.needsPersistence).toBe(true); const committed = await storage.read(); + fail = false; await state.retry(); expect(state.needsPersistence).toBe(false); expect(await storage.read()).toEqual(committed); @@ -425,8 +721,30 @@ describe("inactive real cancellation storage", () => { } ); + it.each(["future", "oversized"] as const)( + "explicit Stop preserves history and installs a retained fence over %s bytes", + async (kind) => { + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const generation = await journal.captureGeneration(); + const bytes = + kind === "future" + ? JSON.stringify({ ...record("future"), version: 2 }) + : " ".repeat(SESSION_HISTORY_MAX_LINE_BYTES + 1); + await fs.writeFile(storage.path, bytes); + expect(await state.cancel()).toBe("applied"); + expect(await storage.read()).toMatchObject({ + version: 1, + retainUntilReplacement: true, + scope: { kind: "unresolved" }, + }); + expect(await journal.captureGeneration()).not.toBe(generation); + expect(await h.historyService.getHistoryFromLatestBoundary(workspaceId)).toEqual(history); + } + ); + it.each(["newer", "oversized", "oversized newer"] as const)( - "preserves %s cancellation and recovery bytes through every refusal path", + "preserves %s cancellation and recovery bytes through automatic refusal paths", async (kind) => { const bytes = " ".repeat(kind === "newer" ? 0 : SESSION_HISTORY_MAX_LINE_BYTES) + @@ -454,8 +772,7 @@ describe("inactive real cancellation storage", () => { () => true, () => mutationCommitted(null) ), - () => state.readForReplacement(), - () => state.cancel(), + () => state.read(), ]) { await assert.rejects( operation, @@ -951,6 +1268,40 @@ describe("inactive real cancellation storage", () => { expect((await storage.read())?.nonce).toBeDefined(); }); + it.each(["Stop", "generation"] as const)( + "unsupported-record publication retries cannot adopt a foreign %s", + async (change) => { + await fs.writeFile(storage.path, JSON.stringify({ ...record("future"), version: 2 })); + const mutation = publication("stale-intervention"); + const rename = nodeFs.renameSync; + const failingRename = spyOn(nodeFs, "renameSync").mockImplementation( + (source, destination) => { + if (destination === storage.path) throw new Error("cancellation rename failed"); + return rename(source, destination); + } + ); + await assert.rejects( + storage.mutate(mutation, () => true, mutationCommitted), + /cancellation rename failed/ + ); + failingRename.mockRestore(); + if (change === "Stop") { + await new CompactionCancellation( + new FileCompactionCancellationStorage(foreign, workspaceId) + ).cancel(); + } else { + await foreign.getContinuousCompactionJournal(workspaceId).advanceGeneration(); + } + const generationPath = path.join(sessionDir, CONTINUOUS_COMPACTION_GENERATION_FILE); + const before = await fs.readFile(storage.path); + const generation = await fs.readFile(generationPath); + mutation.publication.attempts++; + expect(await storage.mutate(mutation, () => true, mutationCommitted)).toBe("superseded"); + expect(await fs.readFile(storage.path)).toEqual(before); + expect(await fs.readFile(generationPath)).toEqual(generation); + } + ); + it("a locally superseded publication cannot rename its staged cancellation", async () => { let current = true; const journal = h.historyService.getContinuousCompactionJournal(workspaceId); @@ -969,7 +1320,7 @@ describe("inactive real cancellation storage", () => { ); }); - it("Stop publishes even when real truncate recovery cannot read its marker", async () => { + it("Stop preserves absence when real truncate recovery cannot read its marker", async () => { await fs.mkdir(path.join(sessionDir, "chat-archive.jsonl.truncate.json")); expect( ( @@ -979,8 +1330,11 @@ describe("inactive real cancellation storage", () => { ) ).success ).toBe(false); - expect(await state.cancel()).toBe("applied"); - expect(await storage.read()).not.toBeNull(); + await assert.rejects(state.cancel(), /EISDIR/); + expect(await storage.read()).toBeNull(); + expect(state.needsPersistence).toBe(true); + await fs.rmdir(path.join(sessionDir, "chat-archive.jsonl.truncate.json")); + expect(await state.retry()).toBe("applied"); }); it.each([ @@ -1127,7 +1481,7 @@ describe("inactive real cancellation storage", () => { }); it.each(["JSON", "schema", "privacy", "I/O"])( - "retains cancellation on unsafe partial %s", + "repairs unusable partials while retaining cancellation on protected or unreadable bytes (%s)", async (damage) => { const partialPath = path.join(sessionDir, "partial.json"); const partial = createMuxMessage("partial", "assistant", "summary", { @@ -1145,6 +1499,13 @@ describe("inactive real cancellation storage", () => { if (damage === "I/O") await fs.mkdir(partialPath); else await fs.writeFile(partialPath, contents); await fs.writeFile(storage.path, "{broken cancellation"); + if (damage === "JSON" || damage === "schema") { + expect(await state.read()).toBeNull(); + expect(state.repairRevision).toBe(1); + expect(nodeFs.existsSync(partialPath)).toBe(false); + expect(nodeFs.existsSync(storage.path)).toBe(false); + return; + } await assert.rejects(state.read()); expect(await fs.readFile(storage.path, "utf8")).toBe("{broken cancellation"); expect(state.repairRevision).toBe(0); @@ -1152,6 +1513,43 @@ describe("inactive real cancellation storage", () => { } ); + it.each(["local generation", "physical lease"] as const)( + "a corrupt-partial cleanup displaced by %s preserves successor bytes", + async (displacement) => { + const partialPath = path.join(sessionDir, "partial.json"); + const lockPath = historyWriteLockPath(h.config.rootDir, workspaceId); + await fs.writeFile(partialPath, "{broken partial"); + const successor = createMuxMessage("successor", "assistant", "new summary", { + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp("new request") }, + }); + let current = true; + let displaced = false; + const read = fs.readFile; + spyOn(fs, "readFile").mockImplementation((async (...args: Parameters) => { + const result = await read(...args); + if (args[0] === partialPath && !displaced) { + displaced = true; + // Model a successor arriving during the last read; cleanup must validate after I/O. + if (displacement === "physical lease") + nodeFs.writeFileSync(lockPath, `${process.pid}:successor-partial-holder`); + else current = false; + nodeFs.writeFileSync(partialPath, JSON.stringify(successor)); + } + return result; + }) as typeof read); + try { + const stopping = storage.mutate(publication("old-stop"), () => current, mutationCommitted); + if (displacement === "physical lease") await assert.rejects(stopping, /no longer owned/); + else expect(await stopping).toBe("superseded"); + expect(displaced).toBe(true); + expect(await fs.readFile(partialPath, "utf8")).toBe(JSON.stringify(successor)); + expect(await storage.read()).toBeNull(); + } finally { + if (displacement === "physical lease") await fs.rm(lockPath, { force: true }); + } + } + ); + it("rereads valid successors under the repair lock and guards a repair superseded during I/O", async () => { await fs.writeFile(storage.path, "{bad"); await assert.rejects(storage.read(), MalformedCompactionCancellationError); @@ -1196,11 +1594,185 @@ describe("inactive real cancellation storage", () => { await assert.rejects(state.read(), /Cannot safely neutralize/); expect(await fs.readFile(chatPath, "utf8")).toBe(raw + "\n"); expect(await fs.readFile(storage.path, "utf8")).toBe("{damaged cancellation"); - // Explicit intervention remains usable without dropping an unknown full-clear obligation. - expect(await state.readForReplacement()).toMatchObject({ retainUntilReplacement: true }); + // Failed legacy neutralization cannot publish a new Stop over the preserved predecessor. + await assert.rejects(state.readForReplacement(), /Cannot safely neutralize/); + expect(await fs.readFile(storage.path, "utf8")).toBe("{damaged cancellation"); + expect(state.blocksRecovery).toBe(true); + } + ); + + it("full deletion fences malformed rows without inheriting Stop's row-wise repair debt", async () => { + const row = createMuxMessage("damaged", "assistant", "summary", { + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp() }, + }); + const chatPath = path.join(sessionDir, "chat.jsonl"); + const raw = JSON.stringify({ ...row, parts: null }) + "\n"; + await fs.writeFile(chatPath, raw); + await assert.rejects(state.cancel(), /Cannot safely neutralize/); + expect(state.blocksRecovery).toBe(true); + const journal = foreign.getContinuousCompactionJournal(workspaceId); + const before = await journal.captureGeneration(); + expect(await state.cancel({ fullHistoryDeletion: { percentage: 1 } })).toBe("applied"); + expect(await storage.read()).toMatchObject({ retainUntilReplacement: true }); + expect(await journal.captureGeneration()).not.toBe(before); + expect(state.blocksRecovery).toBe(false); + // Full-delete authorization removes malformed bytes before publishing its fence. + await assert.rejects(fs.stat(chatPath), { code: "ENOENT" }); + // The exception belongs to one mutation and must never change ordinary Stop/repair. + await fs.writeFile(chatPath, raw); + await assert.rejects(state.cancel(), /Cannot safely neutralize/); + expect(await fs.readFile(chatPath, "utf8")).toBe(raw); + }); + + it("full deletion removes downgrade recovery and foreign partial before sidecar publication", async () => { + const row = createMuxMessage("legacy-summary", "assistant", "summary", { + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp() }, + }); + expect((await h.historyService.appendToHistory(workspaceId, row)).success).toBe(true); + const paths = ["chat.jsonl", "chat-archive.jsonl", "partial.json"].map((name) => + path.join(sessionDir, name) + ); + await fs.copyFile(paths[0], paths[1]); + expect( + (await foreign.writePartial(workspaceId, { ...row, id: "foreign-partial" })).success + ).toBe(true); + let receipt: number[] | undefined; + const rename = nodeFs.renameSync; + let checked = false; + spyOn(nodeFs, "renameSync").mockImplementation((from, to) => { + if (to === storage.path) { + checked = true; + for (const file of paths) expect(nodeFs.existsSync(file)).toBe(false); + expect(receipt).toEqual([0, 1, 0, 1]); + } + rename(from, to); + }); + expect( + await state.cancel({ + fullHistoryDeletion: { + percentage: 1, + onCommitted: (sequences) => { + receipt = sequences; + return undefined; + }, + }, + }) + ).toBe("applied"); + expect(checked).toBe(true); + const restarted = new HistoryService(h.config); + expect(await restarted.getLastMessages(workspaceId, 1)).toEqual({ success: true, data: [] }); + expect(await restarted.readPartial(workspaceId)).toBeNull(); + }); + + it("full deletion retries only the sidecar after its receipt and preserves newer ordinary history", async () => { + expect( + (await h.historyService.appendToHistory(workspaceId, createMuxMessage("old", "user", "old"))) + .success + ).toBe(true); + const onCommitted = mock(() => undefined); + const rename = nodeFs.renameSync; + const failure = spyOn(nodeFs, "renameSync").mockImplementation((from, to) => { + if (to === storage.path) throw new Error("sidecar unavailable"); + rename(from, to); + }); + await assert.rejects( + state.cancel({ fullHistoryDeletion: { percentage: 1, onCommitted } }), + /sidecar unavailable/ + ); + expect(onCommitted).toHaveBeenCalledTimes(1); + expect(await storage.read()).toBeNull(); + failure.mockRestore(); + expect( + (await foreign.appendToHistory(workspaceId, createMuxMessage("new", "user", "new"))).success + ).toBe(true); + const chatPath = path.join(sessionDir, "chat.jsonl"); + const bytes = await fs.readFile(chatPath); + expect(await state.retry()).toBe("applied"); + expect(await fs.readFile(chatPath)).toEqual(bytes); + expect(onCommitted).toHaveBeenCalledTimes(1); + }); + + it("a throwing deletion observer cannot repeat the committed transaction", async () => { + expect( + (await h.historyService.appendToHistory(workspaceId, createMuxMessage("old", "user", "old"))) + .success + ).toBe(true); + const onCommitted = mock(() => { + throw new Error("observer failed"); + }); + await assert.rejects( + state.cancel({ fullHistoryDeletion: { percentage: 1, onCommitted } }), + /observer failed/ + ); + expect(await storage.read()).toBeNull(); + expect( + (await foreign.appendToHistory(workspaceId, createMuxMessage("new", "user", "new"))).success + ).toBe(true); + const chatPath = path.join(sessionDir, "chat.jsonl"); + const bytes = await fs.readFile(chatPath); + expect(await state.retry()).toBe("applied"); + expect(await fs.readFile(chatPath)).toEqual(bytes); + expect(onCommitted).toHaveBeenCalledTimes(1); + }); + + it.skipIf(process.platform === "win32").each([false, true])( + "uncertain deletion durability never authorizes retry over a foreign append (archive=%s)", + async (archive) => { + expect( + ( + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old", "user", "old") + ) + ).success + ).toBe(true); + const chatPath = path.join(sessionDir, "chat.jsonl"); + if (archive) await fs.copyFile(chatPath, path.join(sessionDir, "chat-archive.jsonl")); + const onCommitted = mock(() => undefined); + const sync = nodeFs.fsyncSync; + const failure = spyOn(nodeFs, "fsyncSync").mockImplementation((fd) => { + if (!nodeFs.existsSync(chatPath)) throw new Error("delete sync failed"); + sync(fd); + }); + await assert.rejects( + state.cancel({ fullHistoryDeletion: { percentage: 1, onCommitted } }), + /delete sync failed/ + ); + expect(onCommitted).not.toHaveBeenCalled(); + expect(await storage.read()).toBeNull(); + failure.mockRestore(); + expect( + (await foreign.appendToHistory(workspaceId, createMuxMessage("new", "user", "new"))).success + ).toBe(true); + const bytes = await fs.readFile(chatPath); + await assert.rejects(state.retry(), /new explicit clear/); + expect(await fs.readFile(chatPath)).toEqual(bytes); + expect(state.blocksRecovery).toBe(true); } ); + it("a full-deletion scope that became partial cannot erase history", async () => { + for (let i = 0; i < 5; i++) + expect( + ( + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage(String(i), "user", "message ".repeat(50)) + ) + ).success + ).toBe(true); + const chatPath = path.join(sessionDir, "chat.jsonl"); + const before = await fs.readFile(chatPath); + const receipt = mock(() => undefined); + await assert.rejects( + state.cancel({ fullHistoryDeletion: { percentage: 0.1, onCommitted: receipt } }), + /leave messages/ + ); + expect(await fs.readFile(chatPath)).toEqual(before); + expect(receipt).not.toHaveBeenCalled(); + expect(await storage.read()).toBeNull(); + }); + it("never resurrects a removed session through publication or repair", async () => { await state.cancel(); await removeSessionDirUnderMemoryLocks({ diff --git a/src/node/services/compactionCancellation.test.ts b/src/node/services/compactionCancellation.test.ts index a9435c7e230..828e93621e0 100644 --- a/src/node/services/compactionCancellation.test.ts +++ b/src/node/services/compactionCancellation.test.ts @@ -179,6 +179,104 @@ describe("inactive cancellation state core", () => { expect(state.needsPersistence).toBe(false); }); + it.each([false, true])( + "explicit recovery fences an oversized narrow debt (joined in flight=%s)", + async (inFlight) => { + const { state, storage } = harness(); + await state.cancel(); + const original = (await state.read())!; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + storage.mutate.mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + throw new CompactionCancellationReadRefusedError("oversized narrow"); + }); + const narrowing = state.narrow(original.nonce, summary); + const failed = assert.rejects(narrowing, CompactionCancellationReadRefusedError); + await entered.promise; + if (!inFlight) { + release.resolve(); + await failed; + } + expect(state.blocksRecovery).toBe(true); + expect(await state.read()).toEqual(original); + const replacement = state.readForReplacement(); + release.resolve(); + await failed; + const retained = await replacement; + expect(retained).toMatchObject({ + retainUntilReplacement: true, + scope: { kind: "unresolved" }, + }); + expect(retained?.nonce).not.toBe(original.nonce); + expect(state.needsPersistence).toBe(false); + expect(storage.mutate.mock.calls.map(([mutation]) => mutation.kind)).toEqual([ + "publish", + "narrow", + "publish", + ]); + } + ); + + it("manual recovery cannot replace a newer Stop while oversized narrowing fails", async () => { + const { state, storage } = harness(); + await state.cancel(); + const original = (await state.read())!; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + storage.mutate.mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + throw new CompactionCancellationReadRefusedError("oversized narrow"); + }); + const failed = assert.rejects(state.narrow(original.nonce, summary)); + await entered.promise; + const replacement = state.readForReplacement(); + const stopping = state.cancel(); + const successor = await state.read(); + release.resolve(); + await failed; + await stopping; + expect(await replacement).toEqual(successor); + expect(storage.mutate.mock.calls.map(([mutation]) => mutation.kind)).toEqual([ + "publish", + "narrow", + "publish", + ]); + }); + + it("manual recovery joins a newer retry of the same narrow mutation", async () => { + const { state, storage } = harness(); + await state.cancel(); + const original = (await state.read())!; + storage.mutate.mockRejectedValueOnce( + new CompactionCancellationReadRefusedError("oversized narrow") + ); + await assert.rejects(state.narrow(original.nonce, summary)); + const replacement = state.readForReplacement(); + await state.retry(); + expect(await replacement).toMatchObject({ nonce: original.nonce, scope: { kind: "summary" } }); + expect(storage.mutate.mock.calls.map(([mutation]) => mutation.kind)).toEqual([ + "publish", + "narrow", + "narrow", + ]); + }); + + it("ordinary failed narrowing keeps its exact debt during explicit recovery", async () => { + const { state, storage } = harness(); + await state.cancel(); + const original = (await state.read())!; + storage.mutate + .mockRejectedValueOnce(new Error("disk unavailable")) + .mockRejectedValueOnce(new Error("disk unavailable")); + await assert.rejects(state.narrow(original.nonce, summary)); + await assert.rejects(state.readForReplacement(), /disk unavailable/); + expect(await state.read()).toEqual(original); + expect(state.blocksRecovery).toBe(true); + }); + it("failed narrowing retains unresolved exclusion until the exact retry commits", async () => { const { state, storage } = harness(); await state.cancel(); @@ -775,13 +873,17 @@ describe("inactive cancellation state core", () => { expect(state.repairRevision).toBe(0); }); - it("replacement propagates a read refusal without repair or fallback publication", async () => { + it("automatic refusal preserves bytes while explicit replacement installs a retained fence", async () => { const { state, storage } = harness(); const refusal = new CompactionCancellationReadRefusedError("Read refused"); storage.read.mockRejectedValueOnce(refusal); - await assert.rejects(state.readForReplacement(), (error) => error === refusal); + await assert.rejects(state.read(), (error) => error === refusal); expect(storage.repair).not.toHaveBeenCalled(); expect(storage.mutate).not.toHaveBeenCalled(); + storage.read.mockRejectedValueOnce(refusal); + expect(await state.readForReplacement()).toMatchObject({ retainUntilReplacement: true }); + expect(storage.repair).not.toHaveBeenCalled(); + expect(storage.mutate).toHaveBeenCalledTimes(1); expect(state.needsPersistence).toBe(false); }); diff --git a/src/node/services/compactionCancellation.ts b/src/node/services/compactionCancellation.ts index 58cfc84e579..2aa8987f0da 100644 --- a/src/node/services/compactionCancellation.ts +++ b/src/node/services/compactionCancellation.ts @@ -43,6 +43,11 @@ export interface CompactionCancellationRecord { scope: { kind: "unresolved" } | ({ kind: "summary" } & CompactionCancellationSummary); } +export interface CompactionHistoryDeletion { + readonly percentage: number; + onCommitted?: (deletedSequences: number[]) => undefined; +} + export interface CompactionCancellationPublication { attempts: number; // The adapter records each admitted/advanced frontier BEFORE any subsequent failing await. @@ -60,6 +65,11 @@ export type CompactionCancellationMutation = kind: "publish"; record: CompactionCancellationRecord; publication: CompactionCancellationPublication; + /** Explicit full deletion owns row removal; never serialized or used by ordinary Stop. */ + fullHistoryDeletion?: CompactionHistoryDeletion; + /** In-memory completion of the captured engine and terminal policy, never serialized. */ + settled?: Promise; + onCaptured?: (capture: CompactionReplacementCapture) => void; } | { kind: "narrow"; record: CompactionCancellationRecord } | { @@ -77,7 +87,7 @@ export type CompactionCancellationMutationOutcome = "applied" | "superseded"; /** Only successfully read bytes with invalid JSON/schema authorize automatic repair. */ export class MalformedCompactionCancellationError extends Error {} -/** Unsupported or oversized records must be preserved instead of repaired or overwritten. */ +/** Automatic readers must preserve unsupported or oversized records instead of repairing them. */ export class CompactionCancellationReadRefusedError extends Error {} export interface CompactionCancellationStorage { @@ -140,8 +150,10 @@ function serializeCancellation(record: CompactionCancellationRecord) { return { contents, record: CancellationRecordSchema.parse(JSON.parse(contents)) }; } -/** Inactive real adapter. H2b supplies accepted-row verification; H2c wires runtime consumers. */ +/** Durable cancellation adapter; replacement retirement verifies accepted history rows. */ export class FileCompactionCancellationStorage implements CompactionCancellationStorage { + private readonly deletedHistories = new WeakSet(); + private readonly attemptedHistoryDeletions = new WeakSet(); readonly path: string; constructor( @@ -225,7 +237,7 @@ export class FileCompactionCancellationStorage implements CompactionCancellation throw error; } if (!isCurrent()) return "superseded"; - return this.history.withCompactionStorageLock(this.workspaceId, async (_dir, checkLock) => { + const write = this.history.withCompactionStorageLock(this.workspaceId, async (_, checkLock) => { if (!isCurrent()) return "superseded"; if ( mutation.kind === "publish" && @@ -234,10 +246,9 @@ export class FileCompactionCancellationStorage implements CompactionCancellation ) throw new Error("Cancellation frontier was not captured; a new Stop is required"); const current = await this.read().catch((error: unknown) => { - if (mutation.kind !== "publish" || error instanceof CompactionCancellationReadRefusedError) - throw error; - // Explicit Stop may overwrite unreadable state, inheriting its unknown - // full-clear obligation. Reads and automatic repair never gain this authority. + if (mutation.kind !== "publish") throw error; + // Publication is explicit Stop/manual intervention, including downgrade recovery. + // Unknown bytes inherit retention; automatic reads/repair never gain this authority. return undefined; }); if (mutation.kind === "publish") { @@ -267,19 +278,53 @@ export class FileCompactionCancellationStorage implements CompactionCancellation isCurrent ); if (!advanced) return "superseded"; - return (await publishCompactionFile( - this.path, - contents, - isCurrent, - () => { - frontier.nonce = committed.nonce; - // Install inherited retention before cleanup can admit a newer read. - onCommitted(committed); - }, - checkLock - )) - ? "applied" - : "superseded"; + // A durable Stop must already be safe for older readers, which only inspect + // summary/partial metadata. Failed cleanup leaves exact retry debt, not a receipt. + if (mutation.fullHistoryDeletion) { + if (!this.deletedHistories.has(publication)) { + // A failed transaction may already have removed bytes. Only a NEW explicit + // Clear may retry deletion; nonce/generation alone cannot fence ordinary appends. + if (this.attemptedHistoryDeletions.has(publication)) + throw new Error("History deletion was not confirmed; start a new explicit clear."); + this.attemptedHistoryDeletions.add(publication); + if ( + !(await this.history.clearCompactionHistoryUnderHistoryLock( + this.workspaceId, + mutation.fullHistoryDeletion.percentage, + isCurrent, + checkLock, + (sequences) => { + this.deletedHistories.add(publication); + mutation.fullHistoryDeletion?.onCommitted?.(sequences); + return undefined; + } + )) + ) + return "superseded"; + } + } else if ( + !(await this.history.neutralizeCompactionRecoveryUnderHistoryLock( + this.workspaceId, + isCurrent, + checkLock + )) + ) + return "superseded"; + if ( + !(await publishCompactionFile( + this.path, + contents, + isCurrent, + () => { + frontier.nonce = committed.nonce; + // Install inherited retention before cleanup can admit a newer read. + onCommitted(committed); + }, + checkLock + )) + ) + return "superseded"; + return "applied"; } const nonce = mutation.kind === "retire" ? mutation.nonce : mutation.record.nonce; if (current?.nonce !== nonce) return "superseded"; @@ -332,6 +377,28 @@ export class FileCompactionCancellationStorage implements CompactionCancellation onCommitted(null, retiredPredecessor); return "applied"; }); + const outcome = await write; + if (mutation.kind !== "publish" || !mutation.settled || outcome !== "applied") return outcome; + // Settlement may write a final partial/legacy summary after the first cleanup. + // Never hold history locks while joining those writers, or let an old Stop clear a successor. + await mutation.settled; + return this.history.withCompactionStorageLock(this.workspaceId, async (_dir, checkLock) => { + if (!isCurrent()) return "superseded"; + const current = await this.read(); + const generation = await this.history + .getContinuousCompactionJournal(this.workspaceId) + .captureGenerationUnderHistoryLock(); + const frontier = mutation.publication.predecessor; + if (current?.nonce !== frontier?.nonce || generation !== frontier?.generation) + return "superseded"; + return (await this.history.neutralizeCompactionRecoveryUnderHistoryLock( + this.workspaceId, + isCurrent, + checkLock + )) + ? "applied" + : "superseded"; + }); } repair( @@ -404,8 +471,8 @@ export class FileCompactionCancellationStorage implements CompactionCancellation } /** - * Inactive cancellation state core. Stop's retry identity outlives failed turn preparation; - * a later delivery must supply the storage adapter and activate every recovery consumer. + * Stop's retry identity outlives failed turn preparation. AgentSession shares this authority + * across manual replacement acceptance and every automatic recovery consumer. * Injected-adapter tests establish state invariants, not filesystem or cross-process CAS. */ export class CompactionCancellation { @@ -487,21 +554,34 @@ export class CompactionCancellation { cancel(options?: { retainUntilReplacement?: boolean; + settled?: Promise; + onCaptured?: (capture: CompactionReplacementCapture) => void; + fullHistoryDeletion?: CompactionHistoryDeletion; }): Promise { this.current = { version: 1, nonce: randomUUID(), scope: { kind: "unresolved" }, - ...(options?.retainUntilReplacement || this.current?.retainUntilReplacement + ...(options?.fullHistoryDeletion || + options?.retainUntilReplacement || + this.current?.retainUntilReplacement ? { retainUntilReplacement: true } : {}), }; - return this.persist({ kind: "publish", record: this.current, publication: { attempts: 0 } }); + return this.persist({ + kind: "publish", + record: this.current, + publication: { attempts: 0 }, + fullHistoryDeletion: options?.fullHistoryDeletion && { ...options.fullHistoryDeletion }, + onCaptured: options?.onCaptured, + ...(options?.settled ? { settled: options.settled } : {}), + }); } async readForReplacement(): Promise { for (;;) { if (this.blocksRecovery) { + const mutation = this.mutation; const pending = this.pending; const retryFailed = !this.inFlight; try { @@ -509,7 +589,17 @@ export class CompactionCancellation { } catch (error) { // Retry already-failed debt; readers joining an in-flight attempt share its // outcome instead of turning one failure into a chain of additional retries. - if (pending !== this.pending) continue; + if (pending !== this.pending || mutation !== this.mutation) continue; + if ( + mutation?.kind === "narrow" && + error instanceof CompactionCancellationReadRefusedError + ) { + // A legitimate follow-up can exceed the sidecar cap. Only explicit + // replacement may supersede that exact failed refinement, preserving + // Stop until a replacement commits; automatic readers keep the debt. + await this.cancel({ retainUntilReplacement: true }); + continue; + } if (!retryFailed) throw error; const retried = this.retry(); try { @@ -530,8 +620,7 @@ export class CompactionCancellation { if (this.current === undefined) return this.refreshForReplacement(); // A Stop or newer read can commit after reading resolves but before we resume. if (!this.blocksRecovery) return this.effectiveRecord(); - } catch (error) { - if (error instanceof CompactionCancellationReadRefusedError) throw error; + } catch { // Refresh unknown state once; propagate that read's failure instead of repeatedly // publishing fallback Stops that a foreign cancellation keeps superseding. if (this.current === undefined && (this.mutation !== mutation || this.pending !== pending)) @@ -662,6 +751,17 @@ export class CompactionCancellation { (record, retired) => { if (!isCurrent()) return; this.current = structuredClone(record); + if ( + mutation.kind === "publish" && + record && + mutation.publication.predecessor?.nonce === record.nonce + ) { + // This receipt belongs to the initiating Stop, even if cleanup later yields to a peer. + mutation.onCaptured?.({ + nonce: record.nonce, + generation: mutation.publication.predecessor.generation, + }); + } // Commit invalidates pre-deletion reads before lock release. A later foreign // read must survive acknowledgment delayed by adapter cleanup. this.acceptedReadGeneration = ++this.readGeneration; diff --git a/src/node/services/compactionHandler.preparation.test.ts b/src/node/services/compactionHandler.preparation.test.ts index 606f304b6db..56ba8d39de1 100644 --- a/src/node/services/compactionHandler.preparation.test.ts +++ b/src/node/services/compactionHandler.preparation.test.ts @@ -108,8 +108,9 @@ describe("runtime compaction preparation admission", () => { "request" ) .catch((error: unknown) => error); - assert(failure instanceof Error && "code" in failure); - expect(failure.code).toBe("EISDIR"); + assert(failure instanceof Error); + expect(failure.message).toContain("Failed to capture replacement"); + expect(failure.message).toContain("EISDIR"); const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); assert(history.success); expect(history.data.map((row) => row.id)).toEqual(["request"]); @@ -154,13 +155,26 @@ describe("runtime compaction preparation admission", () => { emitter: new EventEmitter(), }); const journal = h.historyService.getContinuousCompactionJournal(workspaceId); - const capture = journal.captureGeneration.bind(journal); - spyOn(journal, "captureGeneration").mockImplementationOnce(async () => { - const generation = await capture(); - entered.resolve(); - await release.promise; - return generation; - }); + if (route === "heartbeat") { + const capture = journal.captureGeneration.bind(journal); + spyOn(journal, "captureGeneration").mockImplementationOnce(async () => { + const generation = await capture(); + entered.resolve(); + await release.promise; + return generation; + }); + } else { + // Manual/idle completion captures the Stop frontier through HistoryService. + const capture = h.historyService.captureCompactionReplacement.bind(h.historyService); + spyOn(h.historyService, "captureCompactionReplacement").mockImplementationOnce( + async (...args) => { + const captured = await capture(...args); + entered.resolve(); + await release.promise; + return captured; + } + ); + } const event: StreamEndEvent = { type: "stream-end", workspaceId, diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 9b8e74877da..31cb52d5f13 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -365,6 +365,7 @@ export class CompactionHandler { async appendHeartbeatContextResetBoundary(params: { boundaryText: string; pendingFollowUp: CompactionFollowUpRequest; + publication?: ContinuousCompactionPublication; isCurrent?: () => boolean; }): Promise> { assert( @@ -375,11 +376,15 @@ export class CompactionHandler { const preparation = this.beginPreparation(params.isCurrent ?? (() => true)); const { boundaryText } = params; const pendingFollowUp = structuredClone(params.pendingFollowUp); - const publication = { - generation: await this.historyService - .getContinuousCompactionJournal(this.workspaceId) - .captureGeneration(), - }; + // Session callers capture before cancellation admission; never adopt a Stop that lands + // while the heartbeat gate or boundary preparation is awaiting disk I/O. + const publication = params.publication + ? structuredClone(params.publication) + : { + generation: await this.historyService + .getContinuousCompactionJournal(this.workspaceId) + .captureGeneration(), + }; await this.retirePartial(preparation); const historyResult = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); @@ -505,19 +510,18 @@ export class CompactionHandler { async handleCompletion( event: StreamEndEvent, compactionRequestMessageId?: string, - isCurrent: () => boolean = () => true + isCurrent: () => boolean = () => true, + publication?: ContinuousCompactionPublication ): Promise { const preparation = this.beginPreparation(isCurrent); event = structuredClone(event); - // Capture before reading history so a reset during classification cannot adopt old rows. - // Defer storage errors until classification: ordinary completion must still run its policy. - const generation = await this.historyService - .getContinuousCompactionJournal(this.workspaceId) - .captureGeneration() - .then( - (value) => Ok(value), - (error: unknown) => Err(error) - ); + publication = publication && structuredClone(publication); + // Live producers retain their admission generation: completion after Stop must not + // adopt its new frontier. Legacy/unowned callers may capture only without Stop debt. + // Defer errors until classification so ordinary completion still runs its policy. + const capture = publication + ? Ok({ nonce: null, generation: publication.generation }) + : await this.historyService.captureCompactionReplacement(this.workspaceId); // The current stream identifies its request when available. Synthetic prompt snapshots can // follow that request in history, so the last user row is not always the compaction request. const historyResult = compactionRequestMessageId @@ -539,8 +543,9 @@ export class CompactionHandler { return false; } - if (!generation.success) throw generation.error; - const publication = { generation: generation.data }; + if (!capture.success) throw new Error(capture.error); + if (capture.data.nonce !== null) return false; + publication ??= { generation: capture.data.generation }; // Determine idle-compaction (auto-triggered due to inactivity) up-front so the // post-stream failure paths below can report a terminal outcome to the idle loop. diff --git a/src/node/services/historyService.compactionFollowUpCleanup.test.ts b/src/node/services/historyService.compactionFollowUpCleanup.test.ts index d01dace8d7f..274dc58d7cb 100644 --- a/src/node/services/historyService.compactionFollowUpCleanup.test.ts +++ b/src/node/services/historyService.compactionFollowUpCleanup.test.ts @@ -49,6 +49,81 @@ describe("conditional compaction follow-up cleanup", () => { await store.cleanup(); }); + test.each([ + "cleared", + "missing", + "sequence", + "duplicate", + "role", + "metadata", + "request", + "null request", + "malformed", + "retired owner", + ] as const)("confirmation requires an exact cleared summary (%s)", async (kind) => { + const expected = summary(); + await store.historyService.appendToHistory(workspaceId, expected); + const sequence = expected.metadata?.historySequence; + assert(sequence != null, "Expected persisted identity"); + const cleared = { + ...expected, + workspaceId, + role: kind === "role" ? "user" : expected.role, + metadata: { + ...expected.metadata, + historySequence: kind === "sequence" ? sequence + 1 : sequence, + muxMetadata: + kind === "metadata" + ? { type: "unrelated" } + : { + type: "compaction-summary", + pendingFollowUp: + kind === "request" + ? { ...request, text: "new work" } + : kind === "null request" + ? null + : undefined, + }, + }, + }; + const row = JSON.stringify(cleared) + "\n"; + const bytes = + kind === "missing" + ? "" + : kind === "malformed" + ? "{broken\n" + : row.repeat(kind === "duplicate" ? 2 : 1); + const historyPath = path.join(store.config.sessionsDir, workspaceId, "chat.jsonl"); + await fs.writeFile(historyPath, bytes); + for (let attempt = 0; attempt < 2; attempt++) { + expect( + await store.historyService.cleanupCompactionFollowUp( + workspaceId, + expected, + "confirm-cleared", + () => kind !== "retired owner" + ) + ).toEqual(Ok(kind === "cleared" ? "applied" : "skipped")); + expect(await fs.readFile(historyPath, "utf8")).toBe(bytes); + } + }); + + test("unreadable history cannot confirm a cleared handoff", async () => { + const expected = summary(); + await store.historyService.appendToHistory(workspaceId, expected); + const historyPath = path.join(store.config.sessionsDir, workspaceId, "chat.jsonl"); + await fs.rm(historyPath); + await fs.mkdir(historyPath); + const result = await store.historyService.cleanupCompactionFollowUp( + workspaceId, + expected, + "confirm-cleared", + () => true + ); + expect(result.success).toBe(false); + expect((await fs.stat(historyPath)).isDirectory()).toBe(true); + }); + test("clearing a handoff preserves late summary finalization and unrelated rows", async () => { const expected = summary(); expect(await store.historyService.appendToHistory(workspaceId, expected)).toEqual( diff --git a/src/node/services/historyService.replacement.test.ts b/src/node/services/historyService.replacement.test.ts index ce5e5a1b4ca..0d1b4f360cc 100644 --- a/src/node/services/historyService.replacement.test.ts +++ b/src/node/services/historyService.replacement.test.ts @@ -43,6 +43,63 @@ describe("compaction replacement acceptance", () => { await fixture.cleanup(); }); + it("keeps a competing Stop outside provider construction and copies the entry capture", async () => { + const captured = await capture(); + const journal = history.getContinuousCompactionJournal(workspaceId); + const read = journal.captureGenerationUnderHistoryLock.bind(journal); + const compared = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(journal, "captureGenerationUnderHistoryLock").mockImplementationOnce(async () => { + const generation = await read(); + compared.resolve(); + await release.promise; + return generation; + }); + const order: string[] = []; + const construction = history.runWithCompactionAdmission(workspaceId, captured, () => { + expect(nodeFs.existsSync(historyWriteLockPath(fixture.config.rootDir, workspaceId))).toBe( + true + ); + order.push("registered"); + }); + await compared.promise; + // Caller mutation cannot replace the original frontier while the lock is awaited. + captured.nonce = "later-unowned-stop"; + const foreign = new CompactionCancellation( + new HistoryService(fixture.config).getCompactionCancellationStorage(workspaceId) + ); + const stopped = foreign.cancel().then((result) => { + order.push("stopped"); + return result; + }); + release.resolve(); + expect(await construction).toEqual(Ok(undefined)); + expect(await stopped).toBe("applied"); + expect(order).toEqual(["registered", "stopped"]); + }); + + it("refuses an older provider capture and releases the lock when construction throws", async () => { + const captured = await capture(); + await stop.cancel(); + const construct = mock(() => undefined); + expect( + (await history.runWithCompactionAdmission(workspaceId, captured, construct)).success + ).toBe(false); + expect(construct).not.toHaveBeenCalled(); + const current = await capture(); + const thrown = await history.runWithCompactionAdmission(workspaceId, current, () => { + throw new Error("provider construction failed"); + }); + expect(thrown.success).toBe(false); + expect( + await history.appendToHistory( + workspaceId, + createMuxMessage("after-factory-error", "user", "retry") + ) + ).toEqual(Ok(undefined)); + expect((await rows()).some((row) => row.id === "after-factory-error")).toBe(true); + }); + async function capture() { const result = await history.captureCompactionReplacement(workspaceId); assert(result.success); diff --git a/src/node/services/historyService.truncation.test.ts b/src/node/services/historyService.truncation.test.ts index d0ede7902f7..d98b235b82f 100644 --- a/src/node/services/historyService.truncation.test.ts +++ b/src/node/services/historyService.truncation.test.ts @@ -1,10 +1,14 @@ -import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test"; import { createHash } from "node:crypto"; +import assert from "node:assert/strict"; +import * as nodeFs from "node:fs"; import * as fs from "node:fs/promises"; import * as path from "node:path"; +import * as atomicWrite from "write-file-atomic"; import { createMuxMessage } from "@/common/types/message"; import { HistoryService } from "./historyService"; import { createTestHistoryService } from "./testHistoryService"; +import { historyWriteLockPath } from "./workspaceRemoval"; const workspaceId = "truncation-compatibility"; const hash = (contents: string | Buffer) => createHash("sha256").update(contents).digest("hex"); @@ -51,9 +55,192 @@ describe("HistoryService truncation marker compatibility", () => { ).toBe(true); }); afterEach(async () => { + mock.restore(); await h.cleanup(); }); + async function publishOwnedArchive( + finalArchive: Buffer | null, + isCurrent: () => boolean, + onCommitted: () => undefined, + finalChat: Buffer | null = active + ) { + const history = h.historyService as unknown as { + rewriteHistoryFilesUnlocked( + workspace: string, + archive: Buffer | null, + chat: Buffer | null, + publication: { + assertStillOwned: () => Promise; + isCurrent: () => boolean; + onCommitted: () => undefined; + } + ): Promise; + }; + return h.historyService.withCompactionStorageLock(workspaceId, (_dir, assertStillOwned) => + history.rewriteHistoryFilesUnlocked(workspaceId, finalArchive, finalChat, { + assertStillOwned, + isCurrent, + onCommitted, + }) + ); + } + + test.skipIf(process.platform === "win32").each([ + ["retained archive", false], + ["retained archive", true], + ["removed archive", false], + ["removed archive", true], + ["removed chat", false], + ["removed chat", true], + ] as const)( + "owned %s receipt waits for directory durability (flush fails=%s)", + async (kind, fails) => { + if (kind !== "removed chat") await fs.writeFile(archivePath, backup); + const finalArchive = kind === "retained archive" ? reset : null; + const finalChat = kind === "removed chat" ? null : active; + let flushed = false; + let flushedAtReceipt = false; + const sync = nodeFs.fsyncSync; + spyOn(nodeFs, "fsyncSync").mockImplementation((fd) => { + if (nodeFs.fstatSync(fd).isDirectory()) { + if (fails) throw new Error("truncation directory unavailable"); + flushed = true; + } + return sync(fd); + }); + const committed = mock(() => { + flushedAtReceipt = flushed; + return undefined; + }); + if (fails) { + await assert.rejects( + publishOwnedArchive(finalArchive, () => true, committed, finalChat), + /directory unavailable/ + ); + expect(committed).not.toHaveBeenCalled(); + } else { + await publishOwnedArchive(finalArchive, () => true, committed, finalChat); + expect(committed).toHaveBeenCalledTimes(1); + expect(flushedAtReceipt).toBe(true); + if (finalChat) expect(await fs.readFile(chatPath)).toEqual(finalChat); + else expect(nodeFs.existsSync(chatPath)).toBe(false); + if (finalArchive) expect(await fs.readFile(archivePath)).toEqual(finalArchive); + else expect(nodeFs.existsSync(archivePath)).toBe(false); + } + } + ); + + test("owned archive publication commits retained raw bytes and chat before notification", async () => { + await fs.writeFile(archivePath, backup); + let current = true; + const committed = mock(() => { + expect(nodeFs.readFileSync(archivePath)).toEqual(reset); + expect(nodeFs.readFileSync(chatPath)).toEqual(active); + current = false; + return undefined; + }); + await publishOwnedArchive(reset, () => current, committed); + expect(committed).toHaveBeenCalledTimes(1); + expect(nodeFs.existsSync(markerPath)).toBe(false); + expect(nodeFs.existsSync(tombstonePath)).toBe(false); + }); + + test.each(["archive", "chat"] as const)( + "Stop during owned %s staging leaves all history unchanged", + async (stage) => { + await fs.writeFile(archivePath, backup); + const beforeChat = await fs.readFile(chatPath); + let current = true; + const atomic = atomicWrite.default; + spyOn(atomicWrite, "default").mockImplementation( + new Proxy(atomic, { + async apply(target, receiver, args: Parameters) { + const result = await Reflect.apply(target, receiver, args); + if ( + String(args[0]).startsWith( + `${stage === "archive" ? archivePath : chatPath}.publication-` + ) + ) + current = false; + return result; + }, + }) + ); + const committed = mock(() => undefined); + await assert.rejects( + publishOwnedArchive(reset, () => current, committed), + /no longer owned/ + ); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(beforeChat); + expect(await fs.readFile(archivePath)).toEqual(backup); + expect(nodeFs.existsSync(markerPath)).toBe(false); + expect(nodeFs.existsSync(tombstonePath)).toBe(false); + } + ); + + test.each([false, true])( + "failed owned archive publication restores history only while its lease remains current (foreign=%s)", + async (foreign) => { + await fs.writeFile(archivePath, backup); + const beforeChat = await fs.readFile(chatPath); + const lockPath = historyWriteLockPath(h.config.rootDir, workspaceId); + const successor = new Map([ + [chatPath, "foreign chat"], + [archivePath, "foreign archive"], + [markerPath, "foreign marker"], + [tombstonePath, "foreign tombstone"], + ]); + const rename = nodeFs.renameSync; + spyOn(nodeFs, "renameSync").mockImplementation((source, destination) => { + if (destination === chatPath) { + if (foreign) { + nodeFs.writeFileSync(lockPath, "foreign-owner"); + for (const [file, bytes] of successor) nodeFs.writeFileSync(file, bytes); + } + throw new Error("chat publication failed"); + } + return rename(source, destination); + }); + const committed = mock(() => undefined); + try { + await assert.rejects(publishOwnedArchive(reset, () => true, committed)); + expect(committed).not.toHaveBeenCalled(); + if (foreign) { + for (const [file, bytes] of successor) + expect(await fs.readFile(file, "utf8")).toBe(bytes); + } else { + expect(await fs.readFile(chatPath)).toEqual(beforeChat); + expect(await fs.readFile(archivePath)).toEqual(backup); + expect(nodeFs.existsSync(markerPath)).toBe(false); + expect(nodeFs.existsSync(tombstonePath)).toBe(false); + } + } finally { + if (foreign) await fs.rm(lockPath, { force: true }); + } + } + ); + + test("owned archive cleanup failure preserves a committed transaction for restart", async () => { + await fs.writeFile(archivePath, backup); + const rm = fs.rm; + spyOn(fs, "rm").mockImplementation((file, options) => + file === tombstonePath ? Promise.reject(new Error("cleanup failed")) : rm(file, options) + ); + const committed = mock(() => undefined); + await publishOwnedArchive(reset, () => true, committed); + expect(committed).toHaveBeenCalledTimes(1); + expect(await fs.readFile(chatPath)).toEqual(active); + expect(await fs.readFile(archivePath)).toEqual(reset); + mock.restore(); + await new HistoryService(h.config).getLastMessages(workspaceId, 1); + expect(await fs.readFile(chatPath)).toEqual(active); + expect(await fs.readFile(archivePath)).toEqual(reset); + expect(nodeFs.existsSync(markerPath)).toBe(false); + expect(nodeFs.existsSync(tombstonePath)).toBe(false); + }); + async function seedTransaction(marker: unknown, archive = reset): Promise { await fs.writeFile(archivePath, archive); await fs.writeFile(chatPath, active); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 18143266337..24b04481501 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -32,7 +32,7 @@ import { isManualHistoryReset } from "@/common/utils/messages/contextWindows"; import { createContextBudgetRejectedMessage } from "@/common/utils/messages/contextBudgetRejection"; import * as path from "path"; import { createHash, randomUUID } from "node:crypto"; -import { fsyncSync, renameSync, rmdirSync, unlinkSync, writeSync } from "node:fs"; +import { fsyncSync, renameSync, rmdirSync, rmSync, unlinkSync, writeSync } from "node:fs"; import { isDeepStrictEqual } from "node:util"; import * as fs from "fs/promises"; import type { @@ -117,6 +117,12 @@ interface HistoryTruncateTransaction extends HistoryTruncateHashes { rawHashes?: HistoryTruncateHashes & { version: 1 }; } +interface CompactionReplacementEdit { + capture: CompactionReplacementCapture; + isCurrent: () => boolean; + onGenerationAdvanced: (generation: string) => undefined; +} + interface HistoryPublicationObserver { assertStillOwned: () => Promise; isCurrent: () => boolean; @@ -667,7 +673,7 @@ export class HistoryService { : path.join(this.config.sessionsDir, workspaceId); } - /** Inactive sidecar seam: Stop must remain publishable when transcript recovery fails. */ + /** Stop must remain publishable even when transcript recovery fails. */ withCompactionStorageLock( workspaceId: string, operation: (sessionDir: string, assertStillOwned: () => Promise) => Promise @@ -719,30 +725,34 @@ export class HistoryService { const partialBytes = await this.readExistingFileBytes(partialPath); if (partialBytes !== null) { const text = partialBytes.toString("utf8"); - let partial: MuxMessage | null; + let partial: MuxMessage | null = null; try { partial = this.normalizeTranscriptMessage(JSON.parse(text)); } catch { - throw new Error("Cannot safely neutralize malformed partial summary"); + // A torn partial cannot be recovered; Stop must not leave every manual send retrying it. } - if ( - !isReadableHistoryMessage(partial) || - !Buffer.from(text).equals(partialBytes) || - (hasRawResetMarker(text) && hasAmbiguousResetKeys(text)) - ) + if (hasRawResetMarker(text) && hasAmbiguousResetKeys(text)) throw new Error("Cannot safely neutralize malformed partial summary"); - const cleared = clearFollowUp(partial); - if ( - cleared !== partial && - !(await publishCompactionFile( - partialPath, - JSON.stringify(cleared), - isCurrent, - undefined, - assertStillOwned - )) - ) - return false; + if (!isReadableHistoryMessage(partial) || !Buffer.from(text).equals(partialBytes)) { + // Match guarded cancellation cleanup: no await between the final ownership check and + // deletion, so a displaced Stop cannot erase a valid successor's partial. + await assertStillOwned(); + if (!isCurrent()) return false; + rmSync(partialPath, { force: true }); + } else { + const cleared = clearFollowUp(partial); + if ( + cleared !== partial && + !(await publishCompactionFile( + partialPath, + JSON.stringify(cleared), + isCurrent, + undefined, + assertStillOwned + )) + ) + return false; + } } // Repair every recoverable epoch, including summaries restored by truncate // recovery. Raw rewrite helpers retain malformed/ambiguous privacy floors. @@ -782,6 +792,49 @@ export class HistoryService { }, assertStillOwned); } + /** Full Clear has already retired external wakes and fenced the journal under these locks. */ + async clearCompactionHistoryUnderHistoryLock( + workspaceId: string, + percentage: number, + isCurrent: () => boolean, + assertStillOwned: () => Promise, + onCommitted: (deletedSequences: number[]) => undefined + ): Promise { + return this.getAppendProvenance(workspaceId).runMutation(async () => { + if (!isCurrent()) return false; + await this.recoverTruncateTransactionUnlocked(workspaceId, assertStillOwned); + const archive = await this.readHistoryForRewrite(this.getChatArchivePath(workspaceId)); + const chat = await this.readHistoryForRewrite(this.getChatHistoryPath(workspaceId)); + const messages = [...archive.messages, ...chat.messages]; + if ( + percentage < 1 && + messages.length > 0 && + (await this.computeTruncationRemoveCount(messages, percentage)) < messages.length + ) + throw new Error( + "Truncation classified as a full clear would leave messages; retry to re-run it." + ); + const sequences = messages + .map((row) => row.metadata?.historySequence) + .filter((sequence): sequence is number => isNonNegativeInteger(sequence)); + await assertStillOwned(); + if (!isCurrent()) return false; + // A foreign partial may arrive after workspace preflight; it must not restore the + // deleted transcript on downgrade. Full deletion also authorizes malformed bytes. + rmSync(this.getPartialPath(workspaceId), { force: true }); + await this.rewriteHistoryFilesUnlocked(workspaceId, null, null, { + isCurrent, + assertStillOwned, + onCommitted: () => { + this.sequenceCounters.set(workspaceId, 0); + onCommitted(sequences); + return undefined; + }, + }); + return isCurrent(); + }, assertStillOwned); + } + async getSubagentTranscript( input: { taskId: string; requestingWorkspaceId?: string | null }, dependencies: SubagentTranscriptDependencies @@ -1290,7 +1343,8 @@ export class HistoryService { private async rewriteHistoryFilesUnlocked( workspaceId: string, finalArchiveContents: Buffer | null, - finalChatContents: Buffer | null + finalChatContents: Buffer | null, + publication?: HistoryPublicationObserver ): Promise { invalidateHistoryAppendProvenance(); const archivePath = this.getChatArchivePath(workspaceId); @@ -1308,35 +1362,57 @@ export class HistoryService { if (!archiveExists) { assert(finalArchiveContents === null, "cannot replace a missing history archive"); if (finalChatContents === null) { + if (publication) { + await using directory = await this.openHistoryPublicationDirectory( + this.getChatHistoryPath(workspaceId) + ); + await publication.assertStillOwned(); + if (!publication.isCurrent()) throw new Error("History publication no longer owned"); + rmSync(this.getChatHistoryPath(workspaceId), { force: true }); + if (directory) fsyncSync(directory.fd); + publication.onCommitted(); + return; + } await fs.rm(this.getChatHistoryPath(workspaceId), { force: true }); } else { - await writeFileAtomic(this.getChatHistoryPath(workspaceId), finalChatContents); + await this.publishHistoryUnderWriteLock( + this.getChatHistoryPath(workspaceId), + finalChatContents, + publication + ); } return; } - await writeFileAtomic( - markerPath, - JSON.stringify({ - // Older builds hash decoded UTF-8. Keep these fields compatible so a - // downgrade cannot roll back a committed byte-preserving truncation. + const markerContents = JSON.stringify({ + // Older builds hash decoded UTF-8. Keep these fields compatible so a + // downgrade cannot roll back a committed byte-preserving truncation. + finalArchiveHash: + finalArchiveContents === null + ? null + : this.historyContentsHash(finalArchiveContents.toString("utf8")), + finalChatHash: + finalChatContents === null + ? null + : this.historyContentsHash(finalChatContents.toString("utf8")), + rawHashes: { + version: 1, finalArchiveHash: - finalArchiveContents === null - ? null - : this.historyContentsHash(finalArchiveContents.toString("utf8")), + finalArchiveContents === null ? null : this.historyContentsHash(finalArchiveContents), finalChatHash: - finalChatContents === null - ? null - : this.historyContentsHash(finalChatContents.toString("utf8")), - rawHashes: { - version: 1, - finalArchiveHash: - finalArchiveContents === null ? null : this.historyContentsHash(finalArchiveContents), - finalChatHash: - finalChatContents === null ? null : this.historyContentsHash(finalChatContents), - }, - }) - ); + finalChatContents === null ? null : this.historyContentsHash(finalChatContents), + }, + }); + if (publication) { + return this.publishTruncationUnderWriteLock( + workspaceId, + finalArchiveContents, + finalChatContents, + markerContents, + publication + ); + } + await writeFileAtomic(markerPath, markerContents); try { await fs.rename(archivePath, archiveTombstonePath); } catch (error) { @@ -1381,6 +1457,71 @@ export class HistoryService { } } + private async publishTruncationUnderWriteLock( + workspaceId: string, + archive: Buffer | null, + chat: Buffer | null, + marker: string, + publication: HistoryPublicationObserver + ): Promise { + const archivePath = this.getChatArchivePath(workspaceId); + const chatPath = this.getChatHistoryPath(workspaceId); + const markerPath = this.getTruncateTransactionPath(workspaceId); + const outputs: Array<[string, Buffer | null]> = [ + [markerPath, Buffer.from(marker)], + [archivePath, archive], + [chatPath, chat], + ]; + const staged = new Map(); + let committed = false; + try { + // Stop may win throughout staging. No live marker, archive, or chat changes yet. + for (const [target, bytes] of outputs) { + if (bytes === null) continue; + const stagedPath = `${target}.publication-${randomUUID()}`; + staged.set(target, stagedPath); + await writeFileAtomic(stagedPath, bytes, { mode: 0o600 }); + } + await using directory = await this.openHistoryPublicationDirectory(chatPath); + await publication.assertStillOwned(); + if (!publication.isCurrent()) throw new Error("History publication no longer owned"); + try { + // Preserve the existing recovery hashes and tombstone protocol, but leave no await + // between the final ownership check and the complete destructive transaction. + renameSync(staged.get(markerPath)!, markerPath); + renameSync(archivePath, `${archivePath}.truncate`); + if (archive !== null) renameSync(staged.get(archivePath)!, archivePath); + if (chat === null) rmSync(chatPath, { force: true }); + else renameSync(staged.get(chatPath)!, chatPath); + committed = true; + } catch (error) { + // Lease loss forbids recovery over a successor. The next owner can reconcile + // an interrupted transaction using the same marker format as earlier builds. + committed = await this.recoverTruncateTransactionUnlocked( + workspaceId, + publication.assertStillOwned + ); + if (!committed) throw error; + } + // Rename/remove durability is part of the edit receipt. A recovered transaction must + // pass the same barrier; observing its new contents alone cannot accept the edit. + if (directory) fsyncSync(directory.fd); + publication.onCommitted(); + try { + await this.recoverTruncateTransactionUnlocked(workspaceId, publication.assertStillOwned); + } catch (error) { + log.warn("History truncation cleanup deferred to the next owner", { workspaceId, error }); + } + } finally { + for (const stagedPath of staged.values()) { + await fs.rm(stagedPath, { force: true }).catch((error: unknown) => { + if (!committed) throw error; + log.warn("History truncated but staging cleanup failed", { error }); + }); + } + } + } + private getPartialPath(workspaceId: string): string { return path.join(this.getSessionDir(workspaceId), this.PARTIAL_FILE); } @@ -3281,7 +3422,7 @@ export class HistoryService { ); } - /** Inactive until all Stop/send/resume/recovery consumers adopt the same authority. */ + /** Shared history-lock authority for Stop, replacement acceptance, and recovery. */ getCompactionCancellationStorage(workspaceId: string): FileCompactionCancellationStorage { return new FileCompactionCancellationStorage(this, workspaceId, async (witness, signal) => { const evidence = await this.prepareCompactionReplacementWitness( @@ -3329,6 +3470,28 @@ export class HistoryService { ); } + /** Keep the persisted frontier stable through synchronous provider construction/registration. */ + runWithCompactionAdmission( + workspaceId: string, + captured: CompactionReplacementCapture, + construct: () => void + ): Promise> { + const expected = { ...captured }; + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to validate stream admission", + async (assertStillOwned) => { + const current = await this.captureCompactionReplacementUnderHistoryLock(workspaceId); + if (current.nonce !== expected.nonce || current.generation !== expected.generation) + return Err("Compaction admission was superseded"); + await assertStillOwned(); + // Do not await playback, envelopes, or cleanup here: they may acquire this lock. + construct(); + return Ok(undefined); + } + ); + } + /** The caller's capture fences preparation, including a Stop/retirement back to absence. */ async acceptCompactionReplacement( workspaceId: string, @@ -3898,7 +4061,7 @@ export class HistoryService { async cleanupCompactionFollowUp( workspaceId: string, summary: MuxMessage, - action: "clear" | "rollback-heartbeat", + action: "clear" | "rollback-heartbeat" | "confirm-cleared", isCurrent: () => boolean, // Unlike void, undefined rejects async observers that would outlive the held locks. onCommitted?: () => undefined, @@ -3923,7 +4086,8 @@ export class HistoryService { workspaceId, "Failed to clean up compaction follow-up", async (assertStillOwned) => { - if (!isCurrent() || !expected.pendingFollowUp) return Ok("skipped"); + if (!isCurrent() || (!expected.pendingFollowUp && action !== "confirm-cleared")) + return Ok("skipped"); const historyPath = this.getChatHistoryPath(workspaceId); const { rows, messages } = await this.readHistoryForRewrite(historyPath); // Archived summaries no longer own an active continuation or reset rollback. @@ -3940,12 +4104,18 @@ export class HistoryService { // Reused IDs/sequences do not transfer a handoff to a different publication. current.metadata?.compactionPublicationId !== summary.metadata?.compactionPublicationId || !isCompactionSummaryMetadata(metadata) || - !isDeepStrictEqual(metadata.pendingFollowUp, expected.pendingFollowUp) || + (action !== "confirm-cleared" && + !isDeepStrictEqual(metadata.pendingFollowUp, expected.pendingFollowUp)) || (action === "rollback-heartbeat" && current.metadata?.compacted !== "heartbeat") || !isCurrent() ) return Ok("skipped"); + // A crash can leave cancellation debt after the clear committed. Only this exact, + // valid active summary proves absence; missing rows and changed handoffs do not. + if (action === "confirm-cleared") + return Ok(metadata.pendingFollowUp === undefined ? "applied" : "skipped"); + const { pendingFollowUp: _pending, ...remainingMetadata } = metadata; const replacement = action === "rollback-heartbeat" @@ -4692,14 +4862,29 @@ export class HistoryService { async truncateAfterMessage( workspaceId: string, messageId: string, - options?: { keepTargetMessage?: boolean } + options?: { keepTargetMessage?: boolean; replacement?: CompactionReplacementEdit } ): Promise> { return this.withRecoveredHistoryWriteResultLock( workspaceId, "Failed to truncate history", - async () => { + async (assertStillOwned) => { invalidateHistoryAppendProvenance(); try { + const replacement = options?.replacement; + const publication: HistoryPublicationObserver | undefined = replacement && { + assertStillOwned, + isCurrent: replacement.isCurrent, + onCommitted: () => undefined, + }; + if ( + replacement && + (!replacement.isCurrent() || + !isDeepStrictEqual( + await this.captureCompactionReplacementUnderHistoryLock(workspaceId), + replacement.capture + )) + ) + return Err("Compaction replacement changed before edit truncation"); // Structural rewrite requires full file content const { rows, messages } = await this.readHistoryForRewrite( this.getChatHistoryPath(workspaceId) @@ -4724,7 +4909,9 @@ export class HistoryService { messageId, keepTargetMessage, messages, - rows + rows, + replacement, + publication ); } @@ -4749,10 +4936,13 @@ export class HistoryService { if (tailCutChangesProviderContext(removedMessages)) { await this.getContinuousCompactionJournal( workspaceId - ).advanceGenerationUnderHistoryLock(); + ).advanceGenerationUnderHistoryLock( + replacement?.onGenerationAdvanced, + assertStillOwned, + replacement?.isCurrent + ); } - // Atomic write prevents corruption if app crashes mid-write - await writeFileAtomic(historyPath, historyEntries); + await this.publishHistoryUnderWriteLock(historyPath, historyEntries, publication); // Update sequence counter to continue from where we truncated. // Self-healing read path: skip malformed persisted historySequence values. @@ -4808,7 +4998,9 @@ export class HistoryService { keepTargetMessage: boolean, /** Active-epoch messages already read by the caller; all of them are discarded on this branch. */ activeEpochMessages: MuxMessage[], - activeEpochRows: HistoryRewriteRow[] + activeEpochRows: HistoryRewriteRow[], + replacement?: CompactionReplacementEdit, + publication?: HistoryPublicationObserver ): Promise> { try { const { rows: archiveRows, messages: archiveMessages } = await this.readHistoryForRewrite( @@ -4832,7 +5024,11 @@ export class HistoryService { archiveRows.push({ raw: Buffer.from("\n"), message: undefined }); } if (tailCutChangesProviderContext(removedMessages)) { - await this.getContinuousCompactionJournal(workspaceId).advanceGenerationUnderHistoryLock(); + await this.getContinuousCompactionJournal(workspaceId).advanceGenerationUnderHistoryLock( + replacement?.onGenerationAdvanced, + publication?.assertStillOwned, + replacement?.isCurrent + ); } await this.rewriteHistoryFilesUnlocked( workspaceId, @@ -4841,7 +5037,8 @@ export class HistoryService { [...archiveRows, ...activeEpochRows], workspaceId, truncatedMessages - ) + ), + publication ); // chat.jsonl may contain sealed epochs again — allow the lazy check to re-run. this.sealedRotationChecked.delete(workspaceId); diff --git a/src/node/services/messageQueue.test.ts b/src/node/services/messageQueue.test.ts index 3e1474ce55a..162b8a6f423 100644 --- a/src/node/services/messageQueue.test.ts +++ b/src/node/services/messageQueue.test.ts @@ -1,3 +1,4 @@ +import { Err, Ok } from "@/common/types/result"; import { describe, it, expect, beforeEach } from "bun:test"; import { MessageQueue } from "./messageQueue"; import { createMuxMessage, type MuxMessageMetadata } from "@/common/types/message"; @@ -11,6 +12,74 @@ describe("MessageQueue", () => { }); describe("acceptance origin", () => { + it.each([0, 1])("keeps every compaction probe on a batch (stale add=%s)", (staleAdd) => { + const stale = [false, false]; + for (const index of [0, 1]) { + queue.add(`addition ${index}`, undefined, { + compactionAdmissionStale: () => stale[index], + refreshCompactionAdmission: () => { + stale[index] = false; + }, + }); + } + const dispatched = queue.dequeueNext(); + expect(dispatched.message).toBe("addition 0\naddition 1"); + expect(queue.isEmpty()).toBe(true); + expect(dispatched.internal?.admissionStale?.()).toBe(false); + stale[staleAdd] = true; + expect(dispatched.internal?.admissionStale?.()).toBe(true); + stale.fill(true); + dispatched.internal?.refreshCompactionAdmission?.(() => false); + expect(dispatched.internal?.admissionStale?.()).toBe(false); + stale[staleAdd] = true; + expect(dispatched.internal?.admissionStale?.()).toBe(true); + }); + + it("does not refresh automatic authority in a mixed batch", () => { + let manualStale = true; + let automaticStale = true; + queue.add("automatic", undefined, { + acceptanceOrigin: "automatic", + compactionAdmissionStale: () => automaticStale, + refreshCompactionAdmission: () => { + automaticStale = false; + }, + }); + queue.add("manual", undefined, { + compactionAdmissionStale: () => manualStale, + refreshCompactionAdmission: () => { + manualStale = false; + }, + }); + const dispatched = queue.dequeueNext(); + expect(dispatched.message).toBe("automatic\nmanual"); + dispatched.internal?.refreshCompactionAdmission?.(() => false); + expect(manualStale).toBe(false); + expect(automaticStale).toBe(true); + expect(dispatched.internal?.admissionStale?.()).toBe(true); + }); + + it("removes only a withdrawn add's compaction authority", () => { + queue.addOnce("automatic", undefined, "auto:1", { + acceptanceOrigin: "automatic", + compactionAdmissionStale: () => false, + }); + let refreshed = false; + queue.addOnce("manual", undefined, "manual:1", { + compactionAdmissionStale: () => true, + refreshCompactionAdmission: () => { + refreshed = true; + }, + }); + expect(queue.removeByDedupeKeyPrefix("manual:").removedCount).toBe(1); + const dispatched = queue.dequeueNext(); + expect(dispatched.message).toBe("automatic"); + expect(dispatched.internal?.acceptanceOrigin).toBe("automatic"); + expect(dispatched.internal?.admissionStale?.()).toBe(false); + dispatched.internal?.refreshCompactionAdmission?.(() => false); + expect(refreshed).toBe(false); + }); + it("preserves automatic origin across batching without changing visibility or billing", () => { const internal = { acceptanceOrigin: "automatic" as const }; queue.add("first", undefined, internal); @@ -45,6 +114,67 @@ describe("MessageQueue", () => { }); }); + describe("durable admission frontier", () => { + const older = { nonce: null, generation: undefined }; + const newer = { nonce: "new-stop", generation: "new-generation" }; + + it.each(["match", "nonce", "generation", "error"] as const)( + "owned reset receipts preserve %s queued authority through consecutive resets", + async (kind) => { + const source = + kind === "nonce" + ? { ...older, nonce: "foreign" } + : kind === "generation" + ? { ...older, generation: "foreign" } + : older; + const result = kind === "error" ? Err("capture failed") : Ok(source); + queue.add("queued", undefined, { readCompactionAdmission: () => Promise.resolve(result) }); + const first = { ...older, generation: "first-reset" }; + const second = { ...older, generation: "second-reset" }; + queue.advanceCompactionAdmission(older, first); + queue.advanceCompactionAdmission(first, second); + expect(await queue.dequeueNext().internal?.readCompactionAdmission?.()).toEqual( + kind === "match" ? Ok(second) : result + ); + } + ); + + it("does not let a newer batched add authorize an older frontier", async () => { + queue.add("old", undefined, { readCompactionAdmission: () => Promise.resolve(Ok(older)) }); + queue.add("fresh", undefined, { readCompactionAdmission: () => Promise.resolve(Ok(newer)) }); + expect((await queue.dequeueNext().internal?.readCompactionAdmission?.())?.success).toBe( + false + ); + }); + + it("removes only the withdrawn add's durable frontier", async () => { + queue.addOnce("old", undefined, "withdraw:old", { + readCompactionAdmission: () => Promise.resolve(Ok(older)), + }); + queue.add("fresh", undefined, { readCompactionAdmission: () => Promise.resolve(Ok(newer)) }); + expect(queue.removeByDedupeKeyPrefix("withdraw:").removedCount).toBe(1); + expect(await queue.dequeueNext().internal?.readCompactionAdmission?.()).toEqual(Ok(newer)); + }); + + it.each(["manual", "automatic"] as const)( + "Send Now refreshes manual additions without reauthorizing %s siblings", + async (origin) => { + queue.add("manual", undefined, { + readCompactionAdmission: () => Promise.resolve(Ok(older)), + }); + queue.add("sibling", undefined, { + acceptanceOrigin: origin, + readCompactionAdmission: () => Promise.resolve(Ok(older)), + }); + const { internal } = queue.dequeueNext(); + internal?.refreshCompactionAdmission?.(() => false, newer); + const acquired = await internal?.readCompactionAdmission?.(); + if (origin === "manual") expect(acquired).toEqual(Ok(newer)); + else expect(acquired?.success).toBe(false); + } + ); + }); + describe("authoredAtMs", () => { it("returns the request-entry authoring time from dequeueNext when provided", () => { // Codex P2 (PRRT_kwDOPxxmWM6b-orA): the sender captures authoring time diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index 1ef64a2e194..db00bc4a8aa 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -1,7 +1,9 @@ +import type { CompactionReplacementCapture } from "./compactionCancellation"; +import { Err, type Result } from "@/common/types/result"; import { randomUUID } from "node:crypto"; import type { GoalSyntheticMessageKind } from "@/constants/goals"; import assert from "@/common/utils/assert"; -import type { FilePart, SendMessageOptions } from "@/common/orpc/types"; +import type { FilePart, SendMessageOptions, WorkspaceChatMessage } from "@/common/orpc/types"; import { AGENT_PEER_MESSAGE_DEDUPE_PREFIX } from "@/constants/agentMessaging"; import { getValidAgentPeerTriggerMeta } from "@/common/utils/agentMessageEnvelope"; import type { SendMessageError } from "@/common/types/errors"; @@ -97,6 +99,11 @@ type GoalInterventionPolicy = NonNullable; +export type QueuedInput = Pick< + Extract, + "text" | "fileParts" | "reviews" +>; + /** onCanceled text for a send whose cancel signal fired before the turn was accepted. */ export function cancelReasonBeforeAcceptance(signal: AbortSignal): string { return typeof signal.reason === "string" @@ -166,6 +173,15 @@ interface QueuedMessageInternalOptions { * dequeue — where queue clearing can no longer see the entry — still refuses the turn. */ admissionStale?: () => boolean; + /** Stop capture shared by otherwise batchable additions; caller probes remain isolated. */ + compactionAdmissionStale?: () => boolean; + /** The original acquired storage frontier survives preflight, batching, and queue waits. */ + readCompactionAdmission?: () => Promise>; + /** Reauthorize this manual entry only when the user explicitly selects Send now. */ + refreshCompactionAdmission?: ( + isStale: () => boolean, + capture?: CompactionReplacementCapture + ) => void; } type QueueClearCallbacks = Pick< @@ -208,7 +224,12 @@ interface QueueEntry { agentInitiatedCount: number; // Keep per-add origins so removing a keyed manual add cannot promote remaining // automatic work into replacement authority. This does not change queue grouping. - acceptanceOrigins: Array<{ origin: TurnAcceptanceOrigin; dedupeKey?: string }>; + acceptanceOrigins: Array< + { origin: TurnAcceptanceOrigin; dedupeKey?: string } & Pick< + QueuedMessageInternalOptions, + "compactionAdmissionStale" | "refreshCompactionAdmission" | "readCompactionAdmission" + > + >; /** * Timestamp of the latest add batched into this entry. Dispatch exposes it so * goal safety can tell messages typed before a goal existed (queued while the @@ -255,6 +276,26 @@ interface QueueEntry { export class MessageQueue { private entries: QueueEntry[] = []; + /** Carry queued work across only its own durably committed context reset. */ + advanceCompactionAdmission( + predecessor: CompactionReplacementCapture, + successor: CompactionReplacementCapture + ): void { + for (const entry of this.entries) + for (const add of entry.acceptanceOrigins) { + const read = add.readCompactionAdmission; + if (read) + add.readCompactionAdmission = async () => { + const captured = await read(); + return captured.success && + captured.data.nonce === predecessor.nonce && + captured.data.generation === predecessor.generation + ? { success: true, data: { ...successor } } + : captured; + }; + } + } + /** * Check if the queue currently contains a compaction request. */ @@ -713,9 +754,13 @@ export class MessageQueue { if (internal?.admissionStale != null) { entry.admissionStale = internal.admissionStale; } - entry.addCount += 1; - entry.acceptanceOrigins.push({ origin: internal?.acceptanceOrigin ?? "manual" }); + entry.acceptanceOrigins.push({ + origin: internal?.acceptanceOrigin ?? "manual", + compactionAdmissionStale: internal?.compactionAdmissionStale, + readCompactionAdmission: internal?.readCompactionAdmission, + refreshCompactionAdmission: internal?.refreshCompactionAdmission, + }); // Codex security P2 (PRRT_kwDOPxxmWM6b_OS9): batched sends can finish // preflight out of authoring order. Keep the NEWEST authoring time for // the entry — a plain overwrite would let an older pre-goal message mask @@ -844,6 +889,28 @@ export class MessageQueue { return this.getReviewsForEntries(this.getVisibleEntries()); } + /** Stop restores authored input, including an entry already dequeued into preparation. */ + getInputForRestore(): QueuedInput | undefined { + return this.inputForRestore(this.entries); + } + + private inputForRestore(entries: readonly QueueEntry[]): QueuedInput | undefined { + const restorable = entries.filter( + (entry) => + entry.userAuthored && + this.getAcceptanceOrigin(entry) === "manual" && + !entry.cancelSignal?.aborted && + entry.admissionStale?.() !== true + ); + return restorable.length > 0 + ? { + text: this.getDisplayTextForEntries(restorable), + fileParts: this.getFilePartsForEntries(restorable), + reviews: this.getReviewsForEntries(restorable), + } + : undefined; + } + /** Whether a user-visible queued entry is a compaction request. */ hasVisibleCompactionRequest(): boolean { return this.getVisibleEntries().some((entry) => isCompactionMetadata(entry.muxMetadata)); @@ -967,7 +1034,12 @@ export class MessageQueue { /** Capture before admission publication; observers may remove or reorder the head. */ peekNext(): - | { identity: object; muxMetadata: unknown; acceptanceOrigin: TurnAcceptanceOrigin } + | { + identity: object; + muxMetadata: unknown; + acceptanceOrigin: TurnAcceptanceOrigin; + inputForRestore: () => QueuedInput | undefined; + } | undefined { const entry = this.entries[0]; return entry @@ -975,6 +1047,7 @@ export class MessageQueue { identity: entry, muxMetadata: entry.muxMetadata, acceptanceOrigin: this.getAcceptanceOrigin(entry), + inputForRestore: () => this.inputForRestore([entry]), } : undefined; } @@ -1017,6 +1090,53 @@ export class MessageQueue { const allAddsAreAgentInitiated = entry.addCount > 0 && entry.agentInitiatedCount === entry.addCount; const automaticAcceptance = this.getAcceptanceOrigin(entry) === "automatic"; + // Stop fences every add without sealing ordinary follow-ups. Keep the probes + // with their origins so keyed removal also removes only that add's authority. + const admissionStale = entry.acceptanceOrigins.some((add) => add.compactionAdmissionStale) + ? () => + entry.admissionStale?.() === true || + entry.acceptanceOrigins.some((add) => add.compactionAdmissionStale?.() === true) + : entry.admissionStale; + const readCompactionAdmission = entry.acceptanceOrigins.some( + (add) => add.readCompactionAdmission + ) + ? async (): Promise> => { + const captures = await Promise.all( + entry.acceptanceOrigins.map( + (add) => add.readCompactionAdmission?.() ?? Promise.resolve(undefined) + ) + ); + const first = captures.find((capture) => capture !== undefined); + if (!first?.success) return first ?? Err("Queued admission has no captured frontier."); + // A batch cannot promote an older add into a newer add's replacement authority. + if ( + captures.some( + (capture) => + capture && + (!capture.success || + capture.data.nonce !== first.data.nonce || + capture.data.generation !== first.data.generation) + ) + ) + return Err("Queued admission spans different Stop frontiers."); + return first; + } + : undefined; + const refreshCompactionAdmission = + readCompactionAdmission || + entry.acceptanceOrigins.some( + (add) => add.origin === "manual" && add.refreshCompactionAdmission + ) + ? (isStale: () => boolean, capture?: CompactionReplacementCapture) => { + for (const add of entry.acceptanceOrigins) { + if (add.origin !== "manual") continue; + if (capture) + add.readCompactionAdmission = () => + Promise.resolve({ success: true, data: capture }); + add.refreshCompactionAdmission?.(isStale, capture); + } + } + : undefined; const hasInternalOptions = automaticAcceptance || allAddsAreSynthetic || @@ -1025,7 +1145,9 @@ export class MessageQueue { entry.onAcceptedPreStreamFailure != null || entry.onCanceled != null || entry.cancelSignal != null || - entry.admissionStale != null || + admissionStale != null || + refreshCompactionAdmission != null || + readCompactionAdmission != null || (entry.preTurnMessages?.length ?? 0) > 0; const internal = hasInternalOptions ? { @@ -1046,7 +1168,9 @@ export class MessageQueue { ...(entry.onPreTurnRowsPersisted != null ? { onPreTurnRowsPersisted: entry.onPreTurnRowsPersisted } : {}), - ...(entry.admissionStale != null ? { admissionStale: entry.admissionStale } : {}), + ...(admissionStale != null ? { admissionStale } : {}), + ...(readCompactionAdmission != null ? { readCompactionAdmission } : {}), + ...(refreshCompactionAdmission != null ? { refreshCompactionAdmission } : {}), } : undefined; diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index de9492fc057..7711d0d0f4c 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -19,7 +19,7 @@ import type { ToolCallStartEvent, WorkflowRunAttachedEvent, } from "@/common/types/stream"; -import type { MuxMessage } from "@/common/types/message"; +import { createMuxMessage, type MuxMessage } from "@/common/types/message"; import type { WorkflowRunRecord } from "@/common/types/workflow"; import { Ok, Err } from "@/common/types/result"; import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; @@ -49,7 +49,8 @@ import { import { z } from "zod"; import * as modelStatsModule from "@/common/utils/tokens/modelStats"; import { SessionUsageService } from "./sessionUsageService"; -import type { HistoryService } from "./historyService"; +import { HistoryService } from "./historyService"; +import { CompactionCancellation } from "./compactionCancellation"; import { createTestHistoryService } from "./testHistoryService"; import { makeTestEffectRunner } from "./di/testEffectRunner"; import { closeScopeBounded } from "./di/appRuntime"; @@ -1717,58 +1718,75 @@ describe("StreamManager - engine supervision (AppFiberScope occupant)", () => { expect(getWorkspaceStreamsForTests(streamManager).size).toBe(0); }); - test("closing the engine scope while the turn-envelope write is pending cancels the STARTING stream inside the close", async () => { - // startStream registers the stream, then awaits onStreamConstructed (the - // durable turn-envelope write) before launching processing. Supervision - // starts at registration, so a shutdown landing inside that await cancels - // the STARTING stream through the same hard-interrupt path a user stop - // takes there — before closeScopeBounded resolves, not after teardown has - // moved on and the envelope write finally returns. - const workspaceId = "supervised-starting-window-workspace"; - const { streamManager, events, engineScope } = - createSupervisedStreamManagerForTests(flowingThenBlockedStream); - const messageId = `${workspaceId}-msg`; - await appendPartialAssistantForTests(workspaceId, messageId, 1); - let releaseEnvelope!: () => void; - const envelopeWritten = new Promise((resolve) => { - releaseEnvelope = resolve; - }); - const startPromise = streamManager.startStream( - testStartOptions({ - workspaceId, - messageId, - model: createTestLanguageModel(), - tools: {}, - onStreamConstructed: () => envelopeWritten, - }) - ); - const workspaceStreams = getWorkspaceStreamsForTests(streamManager); - const deadline = Date.now() + 5_000; - while (!workspaceStreams.has(workspaceId)) { - if (Date.now() > deadline) throw new Error("stream never registered"); - await new Promise((resolve) => setTimeout(resolve, 5)); - } + test.each(["envelope", "construction fence"] as const)( + "closing the engine scope while %s is pending cancels the STARTING stream inside the close", + async (held) => { + // startStream registers the stream, then awaits onStreamConstructed (the + // durable turn-envelope write) before launching processing. Supervision + // starts at registration, so a shutdown landing inside that await cancels + // the STARTING stream through the same hard-interrupt path a user stop + // takes there — before closeScopeBounded resolves, not after teardown has + // moved on and the envelope write finally returns. + const workspaceId = "supervised-starting-window-workspace"; + const { streamManager, events, engineScope } = + createSupervisedStreamManagerForTests(flowingThenBlockedStream); + const messageId = `${workspaceId}-msg`; + await appendPartialAssistantForTests(workspaceId, messageId, 1); + let releaseEnvelope!: () => void; + const envelopeWritten = new Promise((resolve) => { + releaseEnvelope = resolve; + }); + const captured = await historyService.captureCompactionReplacement(workspaceId); + if (!captured.success) throw new Error(captured.error); + const startPromise = streamManager.startStream( + testStartOptions({ + workspaceId, + messageId, + model: createTestLanguageModel(), + tools: {}, + onStreamConstructed: held === "envelope" ? () => envelopeWritten : undefined, + withAdmissionCurrent: + held === "construction fence" + ? async (construct) => { + const result = await historyService.runWithCompactionAdmission( + workspaceId, + captured.data, + construct + ); + if (!result.success) throw new Error(result.error); + await envelopeWritten; + } + : undefined, + }) + ); + const workspaceStreams = getWorkspaceStreamsForTests(streamManager); + const deadline = Date.now() + 5_000; + while (!workspaceStreams.has(workspaceId)) { + if (Date.now() > deadline) throw new Error("stream never registered"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } - await closeScopeBounded(engineScope); + await closeScopeBounded(engineScope); - // Cancelled inside the close: abort delivered, registry cleared, no - // stream-start ever emitted for it. - expect(terminalEvents(events).map((event) => event.type)).toEqual(["stream-abort"]); - expect((terminalEvents(events)[0] as StreamAbortEvent).abortReason).toBe("system"); - expect(workspaceStreams.size).toBe(0); + // Cancelled inside the close: abort delivered, registry cleared, no + // stream-start ever emitted for it. + expect(terminalEvents(events).map((event) => event.type)).toEqual(["stream-abort"]); + expect((terminalEvents(events)[0] as StreamAbortEvent).abortReason).toBe("system"); + expect(workspaceStreams.size).toBe(0); - releaseEnvelope(); - const result = await startPromise; - expect(result.success).toBe(true); - if (!result.success) throw new Error("expected Ok"); - expect(await result.data.completion).toMatchObject({ - status: "aborted", - abortReason: "system", - }); - expect(events.filter((event) => event.type === "stream-start")).toHaveLength(0); - expect(terminalEvents(events)).toHaveLength(1); - expect(streamManager.isStreaming(workspaceId)).toBe(false); - }); + releaseEnvelope(); + const result = await startPromise; + expect(result.success).toBe(true); + if (!result.success) throw new Error("expected Ok"); + expect(await result.data.completion).toMatchObject({ + status: "aborted", + abortReason: "system", + }); + expect(events.filter((event) => event.type === "stream-start")).toHaveLength(0); + expect(terminalEvents(events)).toHaveLength(1); + expect(streamManager.isStreaming(workspaceId)).toBe(false); + } + ); test("a provider whose iterator rejects on abort is still recorded as an abort, not a failure", async () => { // Some transports surface a cancellation as an iterator rejection rather @@ -2982,6 +3000,8 @@ describe("StreamManager - call settings overrides", () => { }); describe("StreamManager - language model cleanup", () => { + afterEach(() => mock.restore()); + const runtime = LOCAL_TEST_RUNTIME; function createCleanupModel(modelId: string): { @@ -3151,6 +3171,104 @@ describe("StreamManager - language model cleanup", () => { expect(getCleanupCalls()).toBe(1); }); + test.each([ + "before provider", + "after check", + "during envelope", + "local abort", + "local envelope abort", + ] as const)("recorded admission gates startup and releases resources: %s", async (timing) => { + using tempDir = new DisposableTempDir("admission-stream"); + const runtime = createRuntime({ type: "local", srcBaseDir: "/tmp" }); + spyOn(runtime, "resolvePath").mockResolvedValue(tempDir.path); + const workspaceId = `admission-${timing}`; + const streamManager = new StreamManager(historyService); + const { model, getCleanupCalls } = createCleanupModel(workspaceId); + if (typeof model === "string" || !("doStream" in model)) + throw new Error("Expected provider model"); + const providerEntered = Promise.withResolvers(); + let providerSignal: AbortSignal | undefined; + const provider = spyOn(model, "doStream").mockImplementation( + ({ abortSignal }: { abortSignal?: AbortSignal }) => { + providerSignal = abortSignal; + providerEntered.resolve(); + return new Promise((_, reject) => { + if (!abortSignal) return reject(new Error("Expected provider abort signal")); + const aborted = () => + reject(new Error("Provider aborted", { cause: abortSignal.reason })); + if (abortSignal.aborted) aborted(); + else abortSignal.addEventListener("abort", aborted, { once: true }); + }); + } + ); + const constructed = spyOn(aiSdk, "streamText"); + const captured = await historyService.captureCompactionReplacement(workspaceId); + if (!captured.success) throw new Error(captured.error); + const foreign = new CompactionCancellation( + new HistoryService(historyConfig).getCompactionCancellationStorage(workspaceId) + ); + const abort = new AbortController(); + const events: unknown[] = []; + onTurnEngineEvent(streamManager, "stream-start", (event) => events.push(event)); + const acquire = streamManager.createTempDirForStream.bind(streamManager); + spyOn(streamManager, "createTempDirForStream").mockImplementationOnce(async (...args) => { + const dir = await acquire(...args); + if (timing === "before provider") await foreign.cancel(); + return dir; + }); + let checks = 0; + try { + const result = await streamManager.startStream( + testStartOptions({ + workspaceId, + messageId: "gated-start", + runtime, + model, + abortSignal: abort.signal, + assertAdmissionCurrent: async () => { + checks += 1; + const current = await historyService.captureCompactionReplacement(workspaceId); + if (!current.success) throw new Error(current.error); + if (timing === "local abort") abort.abort(); + if (timing === "after check" && checks === 1) await foreign.cancel(); + if ( + current.data.nonce !== captured.data.nonce || + current.data.generation !== captured.data.generation + ) + throw new Error("Recorded admission was superseded"); + }, + withAdmissionCurrent: async (construct) => { + const result = await historyService.runWithCompactionAdmission( + workspaceId, + captured.data, + construct + ); + if (!result.success) throw new Error(result.error); + }, + onStreamConstructed: async () => { + await providerEntered.promise; + if (timing === "local envelope abort") abort.abort(); + await foreign.cancel(); + }, + }) + ); + expect(result.success).toBe(timing === "local abort" || timing === "local envelope abort"); + expect(checks).toBe(timing === "during envelope" ? 2 : 1); + // The first gate prevents streamText itself and the underlying provider call. + // A Stop during the envelope can only prevent processing of the constructed stream. + const reachedProvider = timing === "during envelope" || timing === "local envelope abort"; + expect(constructed).toHaveBeenCalledTimes(reachedProvider ? 1 : 0); + expect(provider).toHaveBeenCalledTimes(reachedProvider ? 1 : 0); + if (reachedProvider) expect(providerSignal?.aborted).toBe(true); + expect(events).toHaveLength(0); + expect(getWorkspaceStreamsForTests(streamManager).has(workspaceId)).toBe(false); + expect(getCleanupCalls()).toBe(1); + if (result.success) expect((await result.data.completion).status).toBe("aborted"); + } finally { + abort.abort(); + } + }); + test("interrupt during onStreamConstructed skips processing and preserves a replacement registration", async () => { const streamManager = new StreamManager(historyService); const { model, getCleanupCalls } = createCleanupModel("constructed-abort-model"); @@ -3275,25 +3393,52 @@ describe("StreamManager - language model cleanup", () => { expect(getCleanupCalls()).toBe(1); }); - test("runs model cleanup when stream creation throws before processing", async () => { - const streamManager = new StreamManager(historyService); - const { model, getCleanupCalls } = createCleanupModel("cleanup-create-throw-model"); - const replaceCreateStreamResult = Reflect.set(streamManager, "createStreamResult", () => { - throw new Error("create stream failed"); - }); - expect(replaceCreateStreamResult).toBe(true); + test.each(["factory", "fence release"] as const)( + "runs model cleanup when %s throws before processing", + async (failure) => { + const streamManager = new StreamManager(historyService); + const { model, getCleanupCalls } = createCleanupModel("cleanup-create-throw-model"); + if (failure === "factory") { + const replaceCreateStreamResult = Reflect.set(streamManager, "createStreamResult", () => { + throw new Error("create stream failed"); + }); + expect(replaceCreateStreamResult).toBe(true); + } - const result = await streamManager.startStream( - testStartOptions({ - workspaceId: "cleanup-create-throw-workspace", - messageId: "cleanup-create-throw-message", - model, - }) - ); + const workspaceId = "cleanup-create-throw-workspace"; + const captured = await historyService.captureCompactionReplacement(workspaceId); + if (!captured.success) throw new Error(captured.error); + const result = await streamManager.startStream( + testStartOptions({ + workspaceId, + withAdmissionCurrent: async (construct) => { + const result = await historyService.runWithCompactionAdmission( + workspaceId, + captured.data, + construct + ); + if (!result.success) throw new Error(result.error); + expect(getWorkspaceStreamsForTests(streamManager).has(workspaceId)).toBe(true); + throw new Error("fence release failed"); + }, + messageId: "cleanup-create-throw-message", + model, + }) + ); - expect(result.success).toBe(false); - expect(getCleanupCalls()).toBe(1); - }); + expect(result.success).toBe(false); + expect(getCleanupCalls()).toBe(1); + expect(getWorkspaceStreamsForTests(streamManager).has(workspaceId)).toBe(false); + expect( + ( + await historyService.appendToHistory( + workspaceId, + createMuxMessage("after-error", "user", "retry") + ) + ).success + ).toBe(true); + } + ); }); describe("StreamManager - turn completion", () => { diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 07a54e78cc9..196cb0228ef 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -311,6 +311,8 @@ export interface TurnExecutionOptions extends StreamRequestOptions { providedRuntimeTempDir?: string; modelFallback?: ModelFallbackOptions; onStreamConstructed?: () => Promise; + assertAdmissionCurrent?: () => Promise; + withAdmissionCurrent?: (construct: () => void) => Promise; } type StreamRequestInput = StreamRequestOptions & { @@ -5360,6 +5362,10 @@ export class StreamManager { const resourceScope = Scope.makeUnsafe(); let processingStarted = false; const cleanupStartup = async (): Promise => { + // streamText may already have invoked the provider while the envelope awaited I/O. + // A refused startup must cancel that request before releasing its owned resources. + if (!streamAbortController.signal.aborted) + streamAbortController.abort(new Error("Stream startup did not complete")); if (registeredStream) return this.closeStreamResources(registeredStream); runLanguageModelCleanup(model); unlinkAbortSignal(); @@ -5422,17 +5428,29 @@ export class StreamManager { return settleStartupAbort(); } - // Step 4: Atomic stream creation and registration - const streamInfo = this.createStreamAtomically(options, { - streamToken, - runtimeTempDir, - resourceScope, - abortController: streamAbortController, - completionController, - }); - - registeredStream = streamInfo; - streamInfo.unlinkAbortSignal = unlinkAbortSignal; + // Construction invokes the provider: validate after every startup resource await. + await options.assertAdmissionCurrent?.(); + if (streamAbortController.signal.aborted) return settleStartupAbort(); + + // The persisted comparison and synchronous provider registration share one lock. + // Record cleanup ownership inside the callback, even if releasing the lock fails. + const construct = () => { + if (streamAbortController.signal.aborted) return; + registeredStream = this.createStreamAtomically(options, { + streamToken, + runtimeTempDir, + resourceScope, + abortController: streamAbortController, + completionController, + }); + registeredStream.unlinkAbortSignal = unlinkAbortSignal; + // Scope close must own STARTING streams before the fence's release can await. + this.superviseEngine(typedWorkspaceId, registeredStream); + }; + if (options.withAdmissionCurrent) await options.withAdmissionCurrent(construct); + else construct(); + const streamInfo = registeredStream; + if (!streamInfo) return settleStartupAbort(); // Guard against a narrow race: // - stopStream() may abort while we're between the last aborted-check and stream registration. @@ -5446,15 +5464,16 @@ export class StreamManager { return settleStartupAbort(); } - // Supervise from registration on: a shutdown landing during the envelope - // write below must find this STARTING stream and cancel it inside the - // scope close (the hard-interrupt path documented after the await), - // not after teardown has moved past the bridges. - this.superviseEngine(typedWorkspaceId, streamInfo); - // Stream constructed + registered: durable request-describing side // effects (turn envelope) may be recorded now. await onStreamConstructed?.(); + // The envelope may wait on storage after construction; do not begin processing a + // superseded request. Existing failure cleanup owns its registered stream and handle. + if ( + !streamAbortController.signal.aborted && + this.workspaceStreams.get(typedWorkspaceId) === streamInfo + ) + await options.assertAdmissionCurrent?.(); // A hard interrupt during the awaited envelope write finds the // registered STARTING stream, aborts it, awaits its placeholder diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index ec5de7d6513..2d19f4b03f0 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -3417,7 +3417,17 @@ describe("TaskService", () => { } ); - test.each(["running", "awaiting_report", "stopped", "opted-out"] as const)( + const startupGuidanceStates = [ + "running", + "awaiting_report", + "stopped", + "opted-out", + "compaction-stopped", + "compaction-unrecorded", + "compaction-unsupported", + "compaction-during-probe", + ] as const; + test.each([...startupGuidanceStates])( "startup restores question guidance only as a queue (%s)", async (state) => { const config = await createTestConfig(rootDir); @@ -3484,11 +3494,50 @@ describe("TaskService", () => { config, historyService, }); - workspaceService.getStartupRecoveryState = () => session.getStartupRecoveryState(); + if (state === "compaction-stopped") { + expect(await session.cancelCompaction(true)).toEqual(Ok(undefined)); + } + if (state === "compaction-unrecorded") { + const publication = spyOn( + historyService, + "withCompactionStorageLock" + ).mockRejectedValueOnce(new Error("Stop publication unavailable")); + expect((await session.cancelCompaction(true)).success).toBe(false); + publication.mockRestore(); + } + if (state === "compaction-during-probe") { + const readTail = historyService.getLastMessages.bind(historyService); + spyOn(historyService, "getLastMessages").mockImplementationOnce(async (...args) => { + const result = await readTail(...args); + expect(await session.cancelCompaction(true)).toEqual(Ok(undefined)); + return result; + }); + } + if (state === "compaction-unsupported") { + await fsPromises.writeFile( + historyService.getCompactionCancellationStorage(childId).path, + JSON.stringify({ version: 99, futureIntent: "Stop" }) + ); + } + workspaceService.getStartupRecoveryState = () => + session.getStartupRecoveryState( + state === "compaction-unrecorded" || state === "compaction-unsupported" ? 25 : undefined + ); try { await taskService.initialize(); expect(compaction).not.toHaveBeenCalled(); - if (state === "stopped" || state === "opted-out") { + if (state === "compaction-unrecorded" || state === "compaction-unsupported") { + expect(sends).toHaveLength(0); + expect(findWorkspaceInConfig(config, childId)?.taskPendingGuidance).toEqual(guidance); + expect(findWorkspaceInConfig(config, childId)?.taskStatus).toBe("running"); + return; + } + if ( + state === "stopped" || + state === "opted-out" || + state === "compaction-stopped" || + state === "compaction-during-probe" + ) { expect(findWorkspaceInConfig(config, childId)?.taskPendingGuidance).toBeUndefined(); expect(sends).toHaveLength(0); return; diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 587039013e3..f1b123420a0 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -291,6 +291,9 @@ export interface StreamMessageOptions { onPreStartError?: (event: ErrorEvent) => void; /** Synchronous registration of the facade's handleless startup notification identity. */ onStreamStarting?: (messageId: string) => void; + /** Revalidate recorded admission after asynchronous startup, without acquiring new authority. */ + assertAdmissionCurrent?: () => Promise; + withAdmissionCurrent?: (construct: () => void) => Promise; /** Tool names that should be delegated back to ACP clients for this request. */ delegatedToolNames?: string[]; recordFileState?: (filePath: string, state: FileState) => Promise; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index c6567ed2857..5d0eec02a76 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1,4 +1,5 @@ import type { TurnCompletion } from "./streamManager"; +import { FileCompactionCancellationStorage } from "./compactionCancellation"; import { CompactionPendingState } from "./compactionPendingState"; import * as historyScanner from "./historyScanner"; import type { TurnCoordinator } from "./turnCoordinator"; @@ -105,6 +106,17 @@ import type { BashMonitorWakeDispatch, } from "./bashMonitorWakeReconciler"; +// Policy fixtures do not run a session; runtime cancellation races use real session fixtures. +function createCompactionAdmissionMocks() { + return { + captureCompactionAdmission: mock(() => () => false), + beginResumeIntent: mock(() => ({ + signal: new AbortController().signal, + [Symbol.dispose]: () => undefined, + })), + }; +} + // Helper to access private renamingWorkspaces set function addToRenamingWorkspaces(service: WorkspaceService, workspaceId: string): void { // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call @@ -752,7 +764,29 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); - test("hard Stop retires owed attention without disarming future idle wakes", async () => { + async function expectMonitorDeferredUntilManual( + h: Awaited>, + requestsBeforeReplacement: number + ): Promise { + // V1 cannot prove that an unresolved Stop's producers settled. Preserve attention until + // manual replacement; the V2 layer separately permits fresh automatic replacement. + expect(h.requests).toHaveLength(requestsBeforeReplacement); + expect((await h.reconciler.snapshot(h.workspaceId)).pendingWakeKinds.size).toBe(2); + expect(await h.session.isAutomaticSendBlocked()).toBe(true); + expect(h.session.isBusy()).toBe(false); + await h.reconciler.reconcile(h.workspaceId); + expect(h.requests).toHaveLength(requestsBeforeReplacement); + expect(h.internal.pendingBashMonitorWakeIdleWaitsByOwner.has(h.workspaceId)).toBe(false); + expect( + await h.session.sendMessage("manual replacement", { model: h.model, agentId: "exec" }) + ).toEqual(Ok(undefined)); + await h.complete(); + await h.reconciler.reconcile(h.workspaceId); + expect(h.requests).toHaveLength(requestsBeforeReplacement + 2); + expect((await h.reconciler.snapshot(h.workspaceId)).pendingWakeKinds.size).toBe(0); + } + + test("hard Stop retires old attention and defers fresh attention until manual replacement", async () => { const h = await createActiveWakeHarness(); try { await h.session.sendMessage("original", { model: h.model, agentId: "exec" }); @@ -772,12 +806,107 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { expect(h.requests).toHaveLength(1); expect((await h.reconciler.snapshot(h.workspaceId)).pendingWakeKinds.size).toBe(0); await h.addAttention(20); + await expectMonitorDeferredUntilManual(h, 1); + } finally { + await h.finish(); + } + }); + + test("retained Stop leaves fresh monitor attention owed without spinning and manual replacement re-arms it", async () => { + const h = await createActiveWakeHarness(); + const internal = h.internal as typeof h.internal & { + scheduleBashMonitorWakeReconcileAfterIdle(ownerWorkspaceId: string): void; + }; + // Suppress a broken immediate retry so the refusal is a bounded assertion, not a timeout. + const idleRetry = spyOn( + internal, + "scheduleBashMonitorWakeReconcileAfterIdle" + ).mockImplementation(() => undefined); + try { + await h.session.cancelCompaction(true); + const storage = h.historyService.getCompactionCancellationStorage(h.workspaceId); + const stop = await storage.read(); + await h.addAttention(20); + expect(idleRetry).not.toHaveBeenCalled(); + expect(h.requests).toHaveLength(0); + expect((await h.reconciler.snapshot(h.workspaceId)).pendingWakeKinds.size).toBe(2); + expect((await storage.read())?.nonce).toBe(stop?.nonce); + const history = await h.historyService.getHistoryFromLatestBoundary(h.workspaceId); + expect(history).toEqual(Ok([])); + idleRetry.mockRestore(); + await h.session.sendMessage("manual replacement", { model: h.model, agentId: "exec" }); + await h.complete(); + await h.reconciler.reconcile(h.workspaceId); expect(h.requests).toHaveLength(2); + expect((await h.reconciler.snapshot(h.workspaceId)).pendingWakeKinds.size).toBe(0); } finally { await h.finish(); } }); + test("a cancellation write failure still completes successful hard-Stop cleanup", async () => { + const h = await createActiveWakeHarness(); + const descendants = mock(() => Promise.resolve(["child"])); + h.service.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ terminateAllDescendantAgentTasks: descendants }) + ); + const partialDeleted = spyOn(h.historyService, "deletePartial"); + const queueRestored = spyOn(h.session, "restoreQueueToInput"); + const accountingEntered = Promise.withResolvers(); + const releaseAccounting = Promise.withResolvers(); + const policy = h.session as unknown as { + recordGoalAccountingFromUsage(input: unknown): Promise; + }; + spyOn(policy, "recordGoalAccountingFromUsage").mockImplementation(async () => { + accountingEntered.resolve(); + await releaseAccounting.promise; + }); + const stopSession = h.session.interruptStream.bind(h.session); + let sessionResult: Awaited> | undefined; + spyOn(h.session, "interruptStream").mockImplementation(async (...args) => { + sessionResult = await stopSession(...args); + return sessionResult; + }); + let interrupt: Promise> | undefined; + try { + await h.session.sendMessage("original", { model: h.model, agentId: "exec" }); + await h.addAttention(10); + spyOn(h.aiService, "stopStream").mockImplementation(() => { + h.abort("user"); + return Promise.resolve(Ok(undefined)); + }); + spyOn(h.aiService, "isStreaming").mockReturnValue(false); + spyOn(FileCompactionCancellationStorage.prototype, "mutate").mockImplementationOnce(() => + Promise.reject(new Error("cancellation write failed")) + ); + let returned = false; + interrupt = h.service.interruptStream(h.workspaceId, { + abandonPartial: true, + retireBashMonitorAttention: true, + }); + const settled = interrupt.then(() => { + returned = true; + }); + await accountingEntered.promise; + await new Promise((resolve) => setImmediate(resolve)); + expect(returned).toBe(false); + releaseAccounting.resolve(); + expect(await interrupt).toEqual(Err(STOP_UNRECORDED_MESSAGE)); + await settled; + expect(sessionResult).toMatchObject({ success: false, error: "cancellation write failed" }); + expect(partialDeleted).toHaveBeenCalledWith(h.workspaceId); + expect(descendants).toHaveBeenCalledWith(h.workspaceId); + expect(queueRestored).toHaveBeenCalledTimes(1); + expect((await h.reconciler.snapshot(h.workspaceId)).pendingWakeKinds.size).toBe(0); + expect(h.session.isBusy()).toBe(false); + } finally { + releaseAccounting.resolve(); + await interrupt; + await h.session.cancelCompaction(); + await h.finish(); + } + }); + test("hard Stop does not wait behind a wake admission holding the history lock", async () => { const h = await createActiveWakeHarness(); try { @@ -841,13 +970,13 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { expect(h.requests).toHaveLength(1); expect((await h.reconciler.snapshot(h.workspaceId)).pendingWakeKinds.size).toBe(0); await h.addAttention(20); - expect(h.requests).toHaveLength(2); + await expectMonitorDeferredUntilManual(h, 1); } finally { await h.finish(); } }); - test("a failed hard Stop keeps owed attention for the idle wake", async () => { + test("a failed hard Stop keeps owed attention until manual replacement", async () => { const h = await createActiveWakeHarness(); try { await h.session.sendMessage("original", { model: h.model, agentId: "exec" }); @@ -861,13 +990,13 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { await h.complete(); await h.internal.pendingBashMonitorWakeIdleWaitsByOwner.get(h.workspaceId); await h.reconciler.reconcile(h.workspaceId); - expect(h.requests).toHaveLength(2); + await expectMonitorDeferredUntilManual(h, 1); } finally { await h.finish(); } }); - test("an interrupt without retireBashMonitorAttention keeps owed attention for the idle wake", async () => { + test("an interrupt without retireBashMonitorAttention preserves owed attention until manual replacement", async () => { const h = await createActiveWakeHarness(); try { await h.session.sendMessage("original", { model: h.model, agentId: "exec" }); @@ -881,7 +1010,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { expect((await h.service.interruptStream(h.workspaceId)).success).toBe(true); await h.internal.pendingBashMonitorWakeIdleWaitsByOwner.get(h.workspaceId); await h.reconciler.reconcile(h.workspaceId); - expect(h.requests).toHaveLength(2); + await expectMonitorDeferredUntilManual(h, 1); } finally { await h.finish(); } @@ -929,7 +1058,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { expect(h.session.isBusy()).toBe(false); expect((await h.reconciler.snapshot(h.workspaceId)).pendingWakeKinds.size).toBe(0); await h.addAttention(20); - expect(h.requests).toHaveLength(1); + await expectMonitorDeferredUntilManual(h, 0); } finally { await h.finish(); } @@ -1109,8 +1238,8 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { release.resolve(); expect((await stop).success).toBe(true); await Promise.all([attention, later]); - // Only the frontier the Stop saw was retired; the newer output woke the idle agent. - expect(h.requests).toHaveLength(1); + // Only the frontier the Stop saw was retired; the newer output remains owed. + await expectMonitorDeferredUntilManual(h, 0); } finally { release.resolve(); await h.finish(); @@ -7383,6 +7512,148 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { return { aiService, config, historyService, workspaceService, goalService, cleanup }; } + test.each(["send", "resume", "resume-replaced"] as const)( + "service %s pricing cannot adopt a later Stop or replacement intent", + async (kind) => { + const { config, historyService, workspaceService, goalService, cleanup } = + await createServices(); + const workspaceId = `pricing-cancellation-${kind}`; + await config.addWorkspace("/tmp/pricing-cancellation-project", { + id: workspaceId, + name: workspaceId, + projectName: "pricing-cancellation-project", + projectPath: "/tmp/pricing-cancellation-project", + runtimeConfig: { type: "local" }, + }); + const h = await createAgentSessionHarness({ + workspaceId, + config, + historyService, + workspaceGoalService: goalService, + }); + workspaceService.registerSession(workspaceId, h.session); + const stream = spyOn(h.aiService, "streamMessage"); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("prior", "user", "old request") + ); + await h.session.cancelCompaction(true); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const pricing = goalService.assertPricedModelForBudgetedGoal.bind(goalService); + spyOn(goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + entered.resolve(); + await release.promise; + return pricing(...args); + } + ); + const dispatch = + kind === "send" + ? workspaceService.sendMessage(workspaceId, "stale input", { + model: "openai:gpt-4o", + agentId: "exec", + }) + : workspaceService.resumeStream(workspaceId, { model: "openai:gpt-4o", agentId: "exec" }); + try { + await entered.promise; + if (kind === "resume-replaced") + h.session.queueMessage("new input", { model: "openai:gpt-4o", agentId: "exec" }); + else expect(await h.session.interruptStream()).toEqual(Ok(undefined)); + release.resolve(); + const result = await dispatch; + if (kind === "send") expect(result.success).toBe(false); + else expect(result).toEqual(Ok({ started: false })); + const persisted = await historyService.getLastMessages(workspaceId, 10); + expect(persisted.success && persisted.data.map((row) => row.id)).toEqual(["prior"]); + expect( + await historyService.getCompactionCancellationStorage(workspaceId).read() + ).not.toBeNull(); + expect(stream).not.toHaveBeenCalled(); + } finally { + release.resolve(); + await dispatch; + await workspaceService.disposeSession(workspaceId); + await cleanup(); + } + } + ); + + test.each(["send", "resume"] as const)( + "service %s pricing preserves the frontier against a foreign backend Stop", + async (kind) => { + const { config, historyService, workspaceService, goalService, cleanup } = + await createServices(); + const workspaceId = `foreign-pricing-cancellation-${kind}`; + await config.addWorkspace("/tmp/pricing-cancellation-project", { + id: workspaceId, + name: workspaceId, + projectName: "pricing-cancellation-project", + projectPath: "/tmp/pricing-cancellation-project", + runtimeConfig: { type: "local" }, + }); + const h = await createAgentSessionHarness({ + workspaceId, + config, + historyService, + workspaceGoalService: goalService, + }); + workspaceService.registerSession(workspaceId, h.session); + const stream = spyOn(h.aiService, "streamMessage"); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("prior", "user", "old request") + ); + const foreign = await createAgentSessionHarness({ + workspaceId, + config, + historyService: new HistoryService(config), + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const pricing = goalService.assertPricedModelForBudgetedGoal.bind(goalService); + spyOn(goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + entered.resolve(); + await release.promise; + return pricing(...args); + } + ); + const dispatch = + kind === "send" + ? workspaceService.sendMessage(workspaceId, "stale input", { + model: "openai:gpt-4o", + agentId: "exec", + }) + : workspaceService.resumeStream(workspaceId, { model: "openai:gpt-4o", agentId: "exec" }); + try { + await entered.promise; + expect(await foreign.session.interruptStream()).toEqual(Ok(undefined)); + const stopped = await historyService.getCompactionCancellationStorage(workspaceId).read(); + expect(stopped).not.toBeNull(); + release.resolve(); + const result = await dispatch; + if (kind === "send") expect(result.success).toBe(false); + else expect(result.success && result.data?.started).toBe(false); + const persisted = await historyService.getLastMessages(workspaceId, 10); + expect(persisted.success && persisted.data.map((row) => row.id)).toEqual(["prior"]); + expect( + await historyService.getCompactionCancellationStorage(workspaceId).read() + ).not.toBeNull(); + expect(stream).not.toHaveBeenCalled(); + expect(await historyService.getCompactionCancellationStorage(workspaceId).read()).toEqual( + stopped + ); + } finally { + release.resolve(); + await dispatch; + await workspaceService.disposeSession(workspaceId); + await foreign.session.dispose(); + await cleanup(); + } + } + ); + test("requireIdle sends carry a live idle-admission probe re-evaluated at session gates", async () => { // Codex P1 (PRRT_kwDOPxxmWM6cJ6NI): the preflight count check at // sendMessage entry is a one-shot snapshot — a manual send can enter @@ -7407,6 +7678,7 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { }); let capturedProbe: (() => boolean) | undefined; const fakeSession = { + ...createCompactionAdmissionMocks(), isBusy: mock(() => false), emitMetadata: mock(() => undefined), drainQueuedMessagesIfIdle: mock(() => undefined), @@ -7539,7 +7811,7 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { bashMonitorRecoveryPromise: Promise; }; internal.bashMonitorRecoveryPromise = recovery.promise; - const truncateSpy = spyOn(historyService, "truncateHistory").mockResolvedValue(Ok([])); + const truncateSpy = spyOn(historyService, "clearCompactionHistoryUnderHistoryLock"); try { const clearPromise = workspaceService.truncateHistory(workspaceId, 1.0); @@ -7556,6 +7828,170 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { } }); + test.each( + (["clear", "replace"] as const).flatMap((kind) => + (["barrier", "post-deletion"] as const).map((failure) => ({ kind, failure })) + ) + )("full $kind accounts for actual deletion when $failure fails", async ({ kind, failure }) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const id = `clear-receipt-${kind}-${failure}`; + const h = await createAgentSessionHarness({ workspaceId: id, config, historyService }); + workspaceService.registerSession(id, h.session); + try { + await config.addWorkspace("/tmp/clear-receipt-project", { + id, + name: id, + projectName: "clear-receipt-project", + projectPath: "/tmp/clear-receipt-project", + runtimeConfig: { type: "local" }, + }); + expect( + (await historyService.appendToHistory(id, createMuxMessage("old", "user", "old"))).success + ).toBe(true); + const storage = historyService.getCompactionCancellationStorage(id); + const internal = workspaceService as unknown as { + bashMonitorRecoveryPromise: Promise; + bashMonitorWakeReconciler: BashMonitorWakeReconciler; + contextMutationEpochs: Map; + }; + await internal.bashMonitorRecoveryPromise; + const priorEpoch = internal.contextMutationEpochs.get(id) ?? 0; + const finish = spyOn(internal.bashMonitorWakeReconciler, "finishFullHistoryClear"); + const emit = spyOn(h.session, "emitChatEvent"); + if (failure === "barrier") { + spyOn(internal.bashMonitorWakeReconciler, "beginFullHistoryClear").mockRejectedValueOnce( + new Error("barrier unavailable") + ); + } else { + const clear = historyService.clearCompactionHistoryUnderHistoryLock.bind(historyService); + spyOn(historyService, "clearCompactionHistoryUnderHistoryLock").mockImplementationOnce( + async (...args) => { + await clear(...args); + throw new Error("post-deletion unavailable"); + } + ); + } + const result = await ( + kind === "clear" + ? workspaceService.truncateHistory(id) + : workspaceService.replaceHistory(id, createMuxMessage("new", "user", "new")) + ).catch((error: unknown) => Err(String(error))); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain(`${failure} unavailable`); + const history = await historyService.getHistoryFromLatestBoundary(id); + expect(history.success && history.data.map((row) => row.id)).toEqual( + failure === "barrier" ? ["old"] : [] + ); + expect(await storage.read()).toBeNull(); + expect(internal.contextMutationEpochs.get(id) ?? 0).toBe( + priorEpoch + (failure === "post-deletion" ? 1 : 0) + ); + if (failure === "post-deletion") { + expect(finish).toHaveBeenCalledTimes(1); + expect(emit).toHaveBeenCalledWith({ type: "delete", historySequences: [0] }); + } else { + expect(finish).not.toHaveBeenCalled(); + expect(emit.mock.calls.some(([event]) => event.type === "delete")).toBe(false); + } + } finally { + mock.restore(); + await h.session.dispose(); + await h.cleanup(); + await cleanup(); + } + }); + + test.each([ + ["clear", false], + ["clear", true], + ["replace", false], + ["replace", true], + ] as const)( + "full %s deletes malformed summaries and recovers failed cancellation (delete failure=%s)", + async (kind, failDeletion) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = `clear-malformed-summary-${kind}-${failDeletion}`; + const h = await createAgentSessionHarness({ workspaceId, config, historyService }); + workspaceService.registerSession(workspaceId, h.session); + try { + await config.addWorkspace("/tmp/clear-malformed-summary-project", { + id: workspaceId, + name: workspaceId, + projectName: "clear-malformed-summary-project", + projectPath: "/tmp/clear-malformed-summary-project", + runtimeConfig: { type: "local" }, + }); + const summary = createMuxMessage("damaged-summary", "assistant", "summary", { + compactionBoundary: true, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "obsolete input", model: "openai:gpt-4o", agentId: "exec" }, + }, + }); + expect((await historyService.appendToHistory(workspaceId, summary)).success).toBe(true); + const chatPath = path.join(config.sessionsDir, workspaceId, "chat.jsonl"); + const damaged = JSON.stringify({ ...summary, parts: null }) + "\n"; + await fsPromises.writeFile(chatPath, damaged); + // Ordinary Stop still refuses unsafe row-wise repair; explicit full deletion can + // recover a workspace already left with that failed cancellation's blocking debt. + expect((await h.session.cancelCompaction(true)).success).toBe(false); + const foreign = new HistoryService(config); + const generation = await foreign + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(); + const storage = foreign.getCompactionCancellationStorage(workspaceId); + const clear = () => + kind === "clear" + ? workspaceService.truncateHistory(workspaceId) + : workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "new context") + ); + if (failDeletion) { + const failing = spyOn( + historyService, + "clearCompactionHistoryUnderHistoryLock" + ).mockRejectedValueOnce(new Error("deletion unavailable")); + expect((await clear()).success).toBe(false); + failing.mockRestore(); + expect(await fsPromises.readFile(chatPath, "utf8")).toBe(damaged); + expect(await storage.read()).toBeNull(); + } + expect(await clear()).toEqual(Ok(undefined)); + const remaining = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(remaining.success && remaining.data.map((row) => row.id)).toEqual( + kind === "clear" ? [] : ["replacement"] + ); + expect(await storage.read()).toMatchObject({ retainUntilReplacement: true }); + // A producer captured by another HistoryService before deletion cannot re-publish + // its old boundary into the new epoch, even though the malformed row is now gone. + const committed = mock(() => undefined); + expect( + await foreign.persistBoundaryWithTailCopies(workspaceId, summary, [], false, undefined, { + publication: { generation }, + onCommitted: committed, + }) + ).toEqual(Err("Compaction publication changed")); + expect(committed).not.toHaveBeenCalled(); + expect( + ( + await h.session.sendMessage("manual input after clear", { + model: "openai:gpt-4o", + agentId: "exec", + }) + ).success + ).toBe(true); + expect(await storage.read()).toBeNull(); + const sent = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(sent.success && sent.data.filter((row) => row.role === "user")).toHaveLength(1); + } finally { + await h.session.dispose(); + await h.cleanup(); + await cleanup(); + } + } + ); + test("full chat clear preserves the goal and requires user acknowledgment", async () => { const { config, historyService, workspaceService, goalService, cleanup } = await createServices(); @@ -7597,7 +8033,8 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { }); test("full chat clear without a goal does not create goal state", async () => { - const { config, workspaceService, goalService, cleanup } = await createServices(); + const { config, historyService, workspaceService, goalService, cleanup } = + await createServices(); const workspaceId = "clear-without-goal-workspace"; try { await config.addWorkspace("/tmp/clear-without-goal-project", { @@ -7612,6 +8049,9 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { expect(result.success).toBe(true); expect(await goalService.getGoal(workspaceId)).toBeNull(); + expect( + await historyService.getCompactionCancellationStorage(workspaceId).read() + ).toMatchObject({ retainUntilReplacement: true }); } finally { await cleanup(); } @@ -8586,9 +9026,9 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { const appendReached = createDeferred(); const releaseAppend = createDeferred(); - const originalAppend = historyService.appendToHistory.bind(historyService); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementationOnce( - async (...args: Parameters) => { + const originalAppend = historyService.acceptCompactionReplacement.bind(historyService); + const appendSpy = spyOn(historyService, "acceptCompactionReplacement").mockImplementationOnce( + async (...args: Parameters) => { appendReached.resolve(); await releaseAppend.promise; return originalAppend(...args); @@ -8654,9 +9094,9 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { // pre-persist gate, strictly before its rows land. const appendReached = createDeferred(); const releaseAppend = createDeferred(); - const originalAppend = historyService.appendToHistory.bind(historyService); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementationOnce( - async (...args: Parameters) => { + const originalAppend = historyService.acceptCompactionReplacement.bind(historyService); + const appendSpy = spyOn(historyService, "acceptCompactionReplacement").mockImplementationOnce( + async (...args: Parameters) => { appendReached.resolve(); await releaseAppend.promise; return originalAppend(...args); @@ -8875,9 +9315,10 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { }); await seedSettledBranchSummaryRegistration(historyService, workspaceId); - const truncateSpy = spyOn(historyService, "truncateHistory").mockImplementationOnce(() => - Promise.resolve(Err("disk full")) - ); + const truncateSpy = spyOn( + historyService, + "clearCompactionHistoryUnderHistoryLock" + ).mockRejectedValueOnce(new Error("disk full")); try { expect(await workspaceService.truncateHistory(workspaceId)).toEqual({ success: false, @@ -9947,6 +10388,7 @@ describe("WorkspaceService initialize", () => { test("disposes transient startup-recovery sessions that go idle", async () => { const dispose = mock(() => undefined); const fakeSession = { + ...createCompactionAdmissionMocks(), runStartupRecovery: mock(() => Promise.resolve()), shouldRetainAfterStartupRecovery: mock(() => false), scheduleStartupRecovery: mock(() => undefined), @@ -9976,6 +10418,7 @@ describe("WorkspaceService initialize", () => { const onChatEvent = mock(() => () => undefined); const onMetadataEvent = mock(() => () => undefined); const fakeSession = { + ...createCompactionAdmissionMocks(), runStartupRecovery: mock(() => Promise.resolve()), shouldRetainAfterStartupRecovery: mock(() => true), scheduleStartupRecovery: mock(() => undefined), @@ -10003,6 +10446,7 @@ describe("WorkspaceService initialize", () => { const onChatEvent = mock(() => () => undefined); const onMetadataEvent = mock(() => () => undefined); const fakeSession = { + ...createCompactionAdmissionMocks(), onChatEvent, onMetadataEvent, } as unknown as AgentSession; @@ -10504,6 +10948,7 @@ describe("WorkspaceService sendMessage status clearing", () => { }); fakeSession = { + ...createCompactionAdmissionMocks(), isBusy: mock(() => true), hasQueuedMessages: mock(() => false), hasQueuedOrDispatchingEntry: mock(() => false), @@ -11709,6 +12154,7 @@ describe("WorkspaceService pending auto-title", () => { ); fakeSession = { + ...createCompactionAdmissionMocks(), isBusy: mock(() => false), hasQueuedMessages: mock(() => false), hasQueuedOrDispatchingEntry: mock(() => false), @@ -11758,6 +12204,16 @@ describe("WorkspaceService pending auto-title", () => { test("concurrent sends only claim one pending auto-title generation", async () => { const releaseSend = createDeferred>(); fakeSession.sendMessage.mockImplementation(() => releaseSend.promise); + const capturesEntered = createDeferred(); + const releaseCaptures = createDeferred(); + const capture = historyService.captureCompactionReplacement.bind(historyService); + let captures = 0; + spyOn(historyService, "captureCompactionReplacement").mockImplementation(async (...args) => { + const result = await capture(...args); + if (++captures === 2) capturesEntered.resolve(); + await releaseCaptures.promise; + return result; + }); const autoTitleSpy = spyOn( workspaceService as unknown as { maybeRunPendingAutoTitleFromMessage: ( @@ -11778,6 +12234,8 @@ describe("WorkspaceService pending auto-title", () => { agentId: "exec", }); + await capturesEntered.promise; + releaseCaptures.resolve(); releaseSend.resolve(Ok(undefined)); const [firstResult, secondResult] = await Promise.all([firstSend, secondSend]); @@ -19011,6 +19469,7 @@ describe("WorkspaceService init cancellation", () => { const sessionEmitter = new EventEmitter(); const fakeSession = { + ...createCompactionAdmissionMocks(), onChatEvent: (listener: (event: unknown) => void) => { sessionEmitter.on("chat-event", listener); return () => sessionEmitter.off("chat-event", listener); @@ -20502,6 +20961,39 @@ describe("WorkspaceService interruptStream", () => { await cleanupHistory(); }); + test("soft Send Now dispatches without requiring a hard Stop receipt", async () => { + const workspaceId = "soft-send-now-receipt"; + const h = await createAgentSessionHarness({ workspaceId }); + const service = createWorkspaceServiceForTest({ + config: h.config, + historyService: h.historyService, + aiService: h.aiService as AIService, + initStateManager: h.initStateManager, + backgroundProcessManager: h.backgroundProcessManager, + }); + spyOn(service, "getOrCreateSession").mockReturnValue(h.session); + const accepted = Promise.withResolvers(); + try { + h.session.queueMessage( + "soft queued input", + { model: "openai:gpt-4o", agentId: "exec" }, + { onAccepted: () => accepted.resolve() } + ); + expect( + await service.interruptStream(workspaceId, { soft: true, sendQueuedImmediately: true }) + ).toEqual(Ok(undefined)); + await accepted.promise; + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.some((row) => row.role === "user")).toBe(true); + expect( + await h.historyService.getCompactionCancellationStorage(workspaceId).read() + ).toBeNull(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + }); + test("sendQueuedImmediately waits for interrupted accounting and terminal publication", async () => { const workspaceId = "ws-interrupt-policy-barrier"; const completion = Promise.withResolvers(); @@ -20638,6 +21130,7 @@ describe("WorkspaceService interruptStream", () => { const restoreQueueToInput = mock(() => undefined); const interruptStream = mock(() => Promise.resolve(Ok(undefined))); const fakeSession = { + ...createCompactionAdmissionMocks(), interruptStream, sendNextUserQueuedMessage, restoreQueueToInput, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 0a6dc14ed30..3540323d91a 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -1,3 +1,4 @@ +import type { CompactionReplacementCapture } from "./compactionCancellation"; import type { RestartBlocker } from "@/common/orpc/types"; import { CompactionPendingState } from "./compactionPendingState"; import { POST_COMPACTION_STATE_FILENAME } from "@/constants/compaction"; @@ -2670,6 +2671,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (hasAiServiceStream) { return "deferred"; } + // Retained Stop waits for manual replacement; an already-idle retry would spin. + if (await this.sessions.get(ownerWorkspaceId)?.isAutomaticSendBlocked()) return "deferred"; const sendOptions = (await this.getDelegatedTurnContinuationSendOptions(ownerWorkspaceId)) ?? (await this.getWorkflowContinuationSendOptions(ownerWorkspaceId)); @@ -2724,6 +2727,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } if (!sendResult.success && !accepted) { + if (await this.sessions.get(ownerWorkspaceId)?.isAutomaticSendBlocked()) return "deferred"; this.scheduleBashMonitorWakeReconcileAfterIdle(ownerWorkspaceId); return "deferred"; } @@ -11092,7 +11096,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // pre-admission work refuses the send instead of letting it append and // stream stale content into the fresh context. let admissionEpoch = this.contextMutationEpochs.get(workspaceId) ?? 0; + let compactionAdmissionStale = () => false; const admissionEpochStale = () => + compactionAdmissionStale() || (this.contextMutationEpochs.get(workspaceId) ?? 0) !== admissionEpoch; // r41: count this send as in-preflight until it settles so refine // publication refuses to interleave with its pre-admission window @@ -11170,6 +11176,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const session = this.getOrCreateSession(workspaceId); // Skip recency update for idle compaction - preserve original "last used" time + compactionAdmissionStale = session.captureCompactionAdmission( + internal?.acceptanceOrigin ?? "manual" + ); + // Storage acquisition is admission's cross-backend linearization point. Complete it + // before pricing/settings can suspend; dispatch must never adopt a later Stop. + const admission = await this.historyService.captureCompactionReplacement(workspaceId, { + onRepaired: () => session.clearUsageState(), + replaceUnreadable: (internal?.acceptanceOrigin ?? "manual") === "manual", + }); + if (!admission.success) return Err({ type: "unknown", raw: admission.error }); + const readCompactionAdmission = () => Promise.resolve(admission); + const muxMeta = options?.muxMetadata as { type?: string; source?: string } | undefined; const isIdleCompaction = muxMeta?.type === "compaction-request" && muxMeta?.source === "idle-compaction"; @@ -11253,6 +11271,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // rejected row and applies goal safety. return await session.sendMessage(message, normalizedOptions, { acceptanceOrigin: internal?.acceptanceOrigin ?? "manual", + readCompactionAdmission, synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, goalKind: internal?.goalKind, @@ -11434,6 +11453,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { continuationSendState.options, { acceptanceOrigin: internal?.acceptanceOrigin ?? "manual", + readCompactionAdmission, synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, authoredAtMs, @@ -11452,6 +11472,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // invisible to queue clearing, so the session's turn-admission gates must // re-check it at dispatch. admissionStale: internal?.admissionStale, + compactionAdmissionStale: () => compactionAdmissionStale(), + refreshCompactionAdmission: + (internal?.acceptanceOrigin ?? "manual") === "manual" + ? (isStale) => { + compactionAdmissionStale = isStale; + } + : undefined, } ); @@ -11552,6 +11579,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // paths never fire the callback; the scoped disposal releases on return. const result = await session.sendMessage(message, continuationSendState.options, { acceptanceOrigin: internal?.acceptanceOrigin ?? "manual", + readCompactionAdmission, onTurnAdmissionCommitted: () => sessionInvisiblePreflight.release(), onContextWindowRollover: () => { this.advanceContextMutationEpoch(workspaceId); @@ -11799,6 +11827,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // Reject before persistence/dispatch when the chosen model would silently // bypass budget enforcement on a budgeted resumable goal. + const resumeStale = session.captureCompactionAdmission( + internal?.acceptanceOrigin ?? "manual" + ); + using resumeIntent = + (internal?.acceptanceOrigin ?? "manual") === "manual" + ? session.beginResumeIntent() + : undefined; + const admission = await this.historyService.captureCompactionReplacement(workspaceId, { + onRepaired: () => session.clearUsageState(), + replaceUnreadable: (internal?.acceptanceOrigin ?? "manual") === "manual", + }); + if (!admission.success) return Err({ type: "unknown", raw: admission.error }); const pricingGate = await this.assertPricedModelForBudgetedGoal( workspaceId, normalizedOptions @@ -11806,6 +11846,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (!pricingGate.success) { return Err(pricingGate.error); } + if (resumeStale() || resumeIntent?.signal.aborted) return Ok({ started: false }); // Non-destructive interrupt cascades preserve descendant task workspaces with // taskStatus=interrupted. Transition before stream start so task orchestration stream-end @@ -11832,6 +11873,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // resumed turn itself can observe the reservation and self-veto. const result = await session.resumeStream(normalizedOptions, { acceptanceOrigin: internal?.acceptanceOrigin ?? "manual", + preparationSignal: resumeIntent?.signal, + readCompactionAdmission: () => Promise.resolve(admission), agentInitiated: internal?.agentInitiated, }); sessionInvisiblePreflight.release(); @@ -12038,11 +12081,21 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { log.warn("Failed to disable auto-retry during Stop", { workspaceId, error }); }) : undefined; - let stopResult: Result | undefined; + let stopResult: Awaited> | undefined; + let stopCapture: CompactionReplacementCapture | undefined; + let stopAdmission: ReturnType = () => true; try { - stopResult = await session.interruptStream(options); + const stopping = session.interruptStream({ + ...options, + onCompactionCanceled: (capture) => { + stopCapture = capture; + }, + }); + // cancelCompaction advances synchronously; later local or foreign Stops cannot be adopted. + stopAdmission = session.captureCompactionAdmission("automatic"); + stopResult = await stopping; } finally { - settleStop(stopResult?.success === true); + settleStop(stopResult?.success === true || stopResult?.streamStopped === true); } await retirement; await optOut; @@ -12058,7 +12111,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const stopRecorded = !(retiring || disabling) || ((await session.recordPendingAutoRetryState()) && retirementRecorded); - if (!stopResult.success) { + if (!stopResult.success && !stopResult.streamStopped) { // Interrupt failed, so clear hard-interrupt suppression we set above. if (!options?.soft) { this.agentTaskIntegration?.resetAutoResumeCount(workspaceId); @@ -12101,13 +12154,20 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.agentTaskIntegration?.resetAutoResumeCount(workspaceId); // The card represents only user-authored queue content. Prioritize that // entry over hidden synthetic/background work before dispatching. - session.sendNextUserQueuedMessage(); + session.sendNextUserQueuedMessage( + options?.soft + ? undefined + : { + isStale: stopAdmission, + readCapture: () => stopCapture, + } + ); } else { // Restore queued messages to input box for user-initiated interrupts session.restoreQueueToInput(); } - if (!stopRecorded) { + if (!stopRecorded || !stopResult.success) { log.error("Stop left stopped work eligible to resume on restart", { workspaceId }); return Err(STOP_UNRECORDED_MESSAGE); } @@ -12611,6 +12671,29 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ]); } + private async clearHistoryThroughCompactionCancellation( + workspaceId: string, + percentage: number, + onCancellationFailure: (error: string) => void + ): Promise> { + let deleted: number[] | undefined; + const canceled = await this.getOrCreateSession(workspaceId).cancelCompaction(true, undefined, { + fullHistoryDeletion: { + percentage, + onCommitted: (sequences) => { + deleted = sequences; + return undefined; + }, + }, + }); + if (deleted === undefined) + return canceled.success ? Err("History deletion was superseded; retry the clear.") : canceled; + // The transcript is already gone even if publishing Stop failed. Finish monitor and + // deletion accounting, then report that failure without appending replacement input. + if (!canceled.success) onCancellationFailure(canceled.error); + return Ok(deleted); + } + private clearHistoryWithRetiredBashMonitorWakes( workspaceId: string, clear: () => Promise>, @@ -12773,13 +12856,20 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // direction (a partial cut becoming a full delete skips the full-clear guards; a no-op // becoming a real cut skips reference retirement; a full clear leaving survivors would // apply full-clear-only discards while rows remain). + let cancellationError: string | undefined; const truncate = () => - this.historyService.truncateHistory(workspaceId, effectivePercentage, { - refuseFullDelete: truncationScope === "partial", - refuseRowRemoval: truncationScope === "none", - requireFullDelete: truncationScope === "all", - fenceEmptyHistory: isFullClear, - }); + isFullClear + ? this.clearHistoryThroughCompactionCancellation( + workspaceId, + effectivePercentage, + (error) => { + cancellationError = error; + } + ) + : this.historyService.truncateHistory(workspaceId, effectivePercentage, { + refuseFullDelete: truncationScope === "partial", + refuseRowRemoval: truncationScope === "none", + }); const truncateResult = effectivePercentage > 0 ? await this.clearHistoryWithRetiredBashMonitorWakes(workspaceId, truncate, { @@ -12876,7 +12966,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } - return Ok(undefined); + return cancellationError ? Err(cancellationError) : Ok(undefined); } async resetContext(workspaceId: string): Promise> { @@ -13188,9 +13278,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } this.sessions.get(workspaceId)?.clearUsageState(); + let cancellationError: string | undefined; const clearResult = await this.clearHistoryWithRetiredBashMonitorWakes( workspaceId, - () => this.historyService.clearHistory(workspaceId, { fenceEmptyHistory: !isCompaction }), + () => + isCompaction + ? this.historyService.clearHistory(workspaceId, { fenceEmptyHistory: false }) + : this.clearHistoryThroughCompactionCancellation(workspaceId, 1, (error) => { + cancellationError = error; + }), { discardUnacceptedOnSuccess: true } ); if (!clearResult.success) { @@ -13249,6 +13345,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { status: "completed", }); deletedSequences = clearResult.data; + if (cancellationError) { + if (deletedSequences.length > 0) { + const deleted: DeleteMessage = { type: "delete", historySequences: deletedSequences }; + const session = this.sessions.get(workspaceId); + if (session) session.emitChatEvent(deleted); + else this.emit("chat", { workspaceId, message: deleted }); + } + return Err(cancellationError); + } } const appendResult = await this.historyService.appendToHistory(workspaceId, messageToAppend); diff --git a/tests/ipc/streamCollector.ts b/tests/ipc/streamCollector.ts index f8da0b50f0e..523371eb6c3 100644 --- a/tests/ipc/streamCollector.ts +++ b/tests/ipc/streamCollector.ts @@ -489,6 +489,53 @@ export function createStreamCollector( return new StreamCollector(client, workspaceId); } +/** + * Helper: Resume stream and wait for successful completion + * Uses StreamCollector for ORPC-native event handling + */ +export async function resumeAndWaitForSuccess( + workspaceId: string, + client: OrpcTestClient, + model: string, + timeoutMs = 15000, + options?: { + toolPolicy?: Array<{ regex_match: string; action: "enable" | "disable" | "require" }>; + } +): Promise { + const collector = createStreamCollector(client, workspaceId); + collector.start(); + + try { + // Finish replay before resuming so the previous stream's error cannot be + // mistaken for a failure of the new attempt. Live errors remain checked below. + await collector.waitForSubscription(5000); + collector.clear(); + const resumeResult = await client.workspace.resumeStream({ + workspaceId, + options: { model, agentId: "exec", toolPolicy: options?.toolPolicy }, + }); + + if (!resumeResult.success) { + throw new Error(`Resume failed: ${resumeResult.error}`); + } + + // Wait for stream-end event after resume + const streamEnd = await collector.waitForEvent("stream-end", timeoutMs); + + if (!streamEnd) { + throw new Error("Stream did not complete after resume"); + } + + // Check for errors + const hasError = collector.hasError(); + if (hasError) { + throw new Error("Resumed stream encountered an error"); + } + } finally { + collector.stop(); + } +} + /** * Assert that a stream completed successfully. * Provides helpful error messages when assertions fail. diff --git a/tests/ipc/streaming/stopAdmission.mock.test.ts b/tests/ipc/streaming/stopAdmission.mock.test.ts new file mode 100644 index 00000000000..cc0d02d2cf6 --- /dev/null +++ b/tests/ipc/streaming/stopAdmission.mock.test.ts @@ -0,0 +1,228 @@ +import { createTestEnvironment, cleanupTestEnvironment } from "../setup"; +import { + createTempGitRepo, + cleanupTempGitRepo, + createWorkspace, + generateBranchName, + createStreamCollector, + sendMessageWithModel, + HAIKU_MODEL, +} from "../helpers"; +import { HistoryService } from "@/node/services/historyService"; +import { resumeAndWaitForSuccess } from "../streamCollector"; +import { MockAiRouter } from "@/node/services/mock/mockAiRouter"; + +async function createMockWorkspace() { + const env = await createTestEnvironment(); + env.services.aiService.enableMockMode(); + const repo = await createTempGitRepo(); + const workspace = await createWorkspace(env, repo, generateBranchName("stop-admission")); + if (!workspace.success) throw new Error(String(workspace.error)); + return { + env, + workspaceId: workspace.metadata.id, + async [Symbol.asyncDispose]() { + await env.orpc.workspace.remove({ + workspaceId: workspace.metadata.id, + options: { force: true }, + }); + await cleanupTestEnvironment(env); + await cleanupTempGitRepo(repo); + }, + }; +} + +describe("manual admission after mock stream interruption", () => { + test.each([ + "manual", + "manual-batch", + "automatic", + "caller-canceled", + "manual-with-automatic", + "stopped-again", + ] as const)( + "send now preserves %s admission", + async (kind) => { + await using fixture = await createMockWorkspace(); + const { env, workspaceId } = fixture; + const collector = createStreamCollector(env.orpc, workspaceId); + const session = env.services.workspaceService.getOrCreateSession(workspaceId); + let stopAgain = false; + let stopping: ReturnType | undefined; + const unsubscribe = session.onChatEvent(({ message }) => { + if (stopAgain && message.type === "queued-message-changed" && !message.hasQueuedMessages) { + stopAgain = false; + stopping = session.cancelCompaction(); + } + }); + collector.start(); + try { + await collector.waitForSubscription(5000); + expect( + ( + await sendMessageWithModel( + env, + workspaceId, + `source${" keep-streaming".repeat(600)}`, + HAIKU_MODEL + ) + ).success + ).toBe(true); + expect(await collector.waitForEvent("stream-start", 5000)).not.toBeNull(); + expect(await collector.waitForEvent("stream-delta", 5000)).not.toBeNull(); + let callerCanceled = false; + if (kind === "manual-with-automatic") { + expect( + ( + await env.services.workspaceService.sendMessage( + workspaceId, + "hidden automatic", + { model: HAIKU_MODEL, agentId: "exec" }, + { acceptanceOrigin: "automatic", synthetic: true, agentInitiated: true } + ) + ).success + ).toBe(true); + } + expect( + ( + await env.services.workspaceService.sendMessage( + workspaceId, + "queued replacement", + { model: HAIKU_MODEL, agentId: "exec" }, + { + acceptanceOrigin: kind === "automatic" ? "automatic" : "manual", + ...(kind === "caller-canceled" ? { admissionStale: () => callerCanceled } : {}), + } + ) + ).success + ).toBe(true); + if (kind === "manual-batch") { + expect( + ( + await env.services.workspaceService.sendMessage(workspaceId, "second addition", { + model: HAIKU_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + } + callerCanceled = kind === "caller-canceled"; + expect(await collector.waitForEvent("queued-message-changed", 5000)).not.toBeNull(); + stopAgain = kind === "stopped-again"; + const interrupted = await env.orpc.workspace.interruptStream({ + workspaceId, + options: { sendQueuedImmediately: true }, + }); + expect(interrupted).toEqual({ success: true, data: undefined }); + await session.waitForIdle(); + await stopping; + if (kind === "stopped-again") expect(stopping).toBeDefined(); + const history = await new HistoryService(env.config).getLastMessages(workspaceId, 10); + expect(history.success).toBe(true); + expect(collector.getEvents().filter((event) => event.type === "stream-error")).toEqual([]); + expect( + history.success && history.data.filter((row) => row.role === "user").map((row) => row.id) + ).toHaveLength(kind.startsWith("manual") ? 2 : 1); + if (kind === "manual-batch" && history.success) { + expect(history.data.findLast((row) => row.role === "user")?.parts).toEqual([ + expect.objectContaining({ type: "text", text: "queued replacement\nsecond addition" }), + ]); + } + if (history.success) + expect( + history.data.some((row) => + row.parts.some((part) => part.type === "text" && part.text === "hidden automatic") + ) + ).toBe(false); + } finally { + unsubscribe(); + collector.stop(); + } + }, + 25000 + ); + + test.each([false, true])( + "resume checks only its own error window (fails=%s)", + async (fails) => { + await using fixture = await createMockWorkspace(); + const { env, workspaceId } = fixture; + const session = env.services.workspaceService.getOrCreateSession(workspaceId); + await session.setAutoRetryEnabled(false); + const original = createStreamCollector(env.orpc, workspaceId); + const reply = jest + .spyOn(MockAiRouter.prototype, "route") + .mockReturnValueOnce({ + assistantText: "stable original prefix", + error: { message: "original interruption", type: "server_error" }, + }) + .mockReturnValueOnce({ + assistantText: "continued response", + ...(fails + ? { error: { message: "resumed interruption", type: "server_error" as const } } + : {}), + }); + original.start(); + try { + await original.waitForSubscription(5000); + expect( + ( + await sendMessageWithModel( + env, + workspaceId, + `source${" keep-streaming".repeat(600)}`, + HAIKU_MODEL + ) + ).success + ).toBe(true); + expect(await original.waitForEvent("stream-delta", 5000)).not.toBeNull(); + expect(await original.waitForEvent("stream-error", 5000)).not.toBeNull(); + await session.waitForIdle(); + // Mock error playback clears partials. Seed the delivered prefix as real + // stream recovery does, so the resumed acceptance also preserves existing text. + const historyService = new HistoryService(env.config); + const before = await historyService.getLastMessages(workspaceId, 10); + if (!before.success) throw new Error(before.error); + const assistant = before.data.findLast((row) => row.role === "assistant"); + if (!assistant) throw new Error("Expected the interrupted assistant row"); + const prefix = original + .getDeltas() + .map((event) => ("delta" in event ? event.delta : "")) + .join(""); + expect(prefix.length).toBeGreaterThan(0); + expect( + ( + await historyService.updateHistory(workspaceId, { + ...assistant, + parts: [{ type: "text", text: prefix }], + }) + ).success + ).toBe(true); + original.clear(); + const resuming = resumeAndWaitForSuccess(workspaceId, env.orpc, HAIKU_MODEL, 5000); + if (fails) { + await expect(resuming).rejects.toThrow(); + expect(original.getEvents().filter((event) => event.type === "stream-error")).toEqual([ + expect.objectContaining({ error: "resumed interruption" }), + ]); + } else { + await resuming; + await session.waitForIdle(); + const history = await new HistoryService(env.config).getLastMessages(workspaceId, 10); + expect(history.success).toBe(true); + const texts = history.success + ? history.data + .flatMap((row) => row.parts) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + : []; + expect(texts).toContain(prefix); + expect(texts).toContain("continued response"); + } + } finally { + reply.mockRestore(); + original.stop(); + } + }, + 25000 + ); +}); diff --git a/tests/ipc/streaming/streamErrorRecovery.test.ts b/tests/ipc/streaming/streamErrorRecovery.test.ts index b3beccc3f20..9b371ebf48d 100644 --- a/tests/ipc/streaming/streamErrorRecovery.test.ts +++ b/tests/ipc/streaming/streamErrorRecovery.test.ts @@ -25,7 +25,7 @@ import { configureTestRetries, HAIKU_MODEL, } from "../helpers"; -import type { StreamCollector } from "../streamCollector"; +import { resumeAndWaitForSuccess, type StreamCollector } from "../streamCollector"; // Skip all tests if TEST_INTEGRATION is not set const describeIntegration = shouldRunIntegrationTests() ? describe : describe.skip; @@ -88,51 +88,6 @@ function truncateToLastCompleteMarker(text: string, nonce: string): string { return text.substring(0, endIndex); } -import type { OrpcTestClient } from "../orpcTestClient"; - -/** - * Helper: Resume stream and wait for successful completion - * Uses StreamCollector for ORPC-native event handling - */ -async function resumeAndWaitForSuccess( - workspaceId: string, - client: OrpcTestClient, - model: string, - timeoutMs = 15000, - options?: { - toolPolicy?: Array<{ regex_match: string; action: "enable" | "disable" | "require" }>; - } -): Promise { - const collector = createStreamCollector(client, workspaceId); - collector.start(); - - try { - const resumeResult = await client.workspace.resumeStream({ - workspaceId, - options: { model, agentId: "exec", toolPolicy: options?.toolPolicy }, - }); - - if (!resumeResult.success) { - throw new Error(`Resume failed: ${resumeResult.error}`); - } - - // Wait for stream-end event after resume - const streamEnd = await collector.waitForEvent("stream-end", timeoutMs); - - if (!streamEnd) { - throw new Error("Stream did not complete after resume"); - } - - // Check for errors - const hasError = collector.hasError(); - if (hasError) { - throw new Error("Resumed stream encountered an error"); - } - } finally { - collector.stop(); - } -} - /** * Collect stream deltas until predicate returns true * Returns the accumulated buffer