diff --git a/src/common/orpc/schemas/message.ts b/src/common/orpc/schemas/message.ts index 18658118e6..fa8f4ec8b5 100644 --- a/src/common/orpc/schemas/message.ts +++ b/src/common/orpc/schemas/message.ts @@ -162,6 +162,7 @@ export const MuxMessageSchema = z.object({ metadata: z .object({ historySequence: z.number().optional(), + compactionReplacementNonce: z.string().min(1).optional().catch(undefined), // Step cuts are an optimization; malformed legacy metadata must not block chat replay. stepStartPartIndices: z.array(z.number()).optional().catch(undefined), timestamp: z.number().optional(), diff --git a/src/common/types/message.ts b/src/common/types/message.ts index d96325aac8..3e729540df 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -961,6 +961,8 @@ export interface ContextBudgetRejectedMessage { // Our custom metadata type export interface MuxMetadata { + /** Stop replaced by this durably accepted row; its enclosing id/sequence identify the receipt. */ + compactionReplacementNonce?: string; /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) diff --git a/src/common/utils/messages/contextBudgetRejection.ts b/src/common/utils/messages/contextBudgetRejection.ts index 123e0a2cbe..8035215ddb 100644 --- a/src/common/utils/messages/contextBudgetRejection.ts +++ b/src/common/utils/messages/contextBudgetRejection.ts @@ -22,6 +22,8 @@ export function createContextBudgetRejectedMessage(message: MuxMessage): MuxMess parts: [], metadata: { historySequence: message.metadata?.historySequence, + // Quarantining this same occurrence must preserve its already-committed acceptance. + compactionReplacementNonce: message.metadata?.compactionReplacementNonce, timestamp: message.metadata?.timestamp, synthetic: true, uiVisible: false, diff --git a/src/node/services/compactionCancellation.storage.test.ts b/src/node/services/compactionCancellation.storage.test.ts index 7f79b82258..b4eed9d0c9 100644 --- a/src/node/services/compactionCancellation.storage.test.ts +++ b/src/node/services/compactionCancellation.storage.test.ts @@ -21,6 +21,7 @@ import { type CompactionCancellationMutation, type CompactionCancellationRecord, type CompactionCancellationStorage, + type CompactionReplacementCapture, } from "./compactionCancellation"; type RejectsAsync = (() => Promise) extends Observer ? false : true; @@ -103,6 +104,154 @@ describe("inactive real cancellation storage", () => { await h.cleanup(); }); + it("throwing retirement notification cannot skip unlink durability or recreate debt", async () => { + state = new CompactionCancellation( + h.historyService.getCompactionCancellationStorage(workspaceId) + ); + await state.cancel(); + const captured = await h.historyService.captureCompactionReplacement(workspaceId); + assert(captured.success && captured.data.nonce); + const accepted = await h.historyService.acceptCompactionReplacement( + workspaceId, + captured.data, + { kind: "append", messages: [createMuxMessage("replacement", "user", "Fresh input")] }, + { isCurrent: () => true, onCommitted: () => undefined } + ); + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + let notified = false; + let syncedAfterNotification = false; + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === sessionDir) { + const sync = handle.sync.bind(handle); + spyOn(handle, "sync").mockImplementation(async () => { + await sync(); + if (notified) syncedAfterNotification = true; + }); + } + return handle; + }); + const notifications: unknown[] = []; + expect( + await state.retireReplacement(accepted.data.witness, (before, after) => { + notifications.push([before, after]); + notified = true; + throw new Error("consumer notification failed"); + }) + ).toBe("applied"); + expect(notifications).toEqual([ + [captured.data, { nonce: null, generation: captured.data.generation }], + ]); + if (process.platform !== "win32") expect(syncedAfterNotification).toBe(true); + expect(await storage.read()).toBeNull(); + expect(state.needsPersistence).toBe(false); + expect(await state.retry()).toBe("applied"); + expect(notifications).toHaveLength(1); + }); + + it.each([ + "commit", + "foreign-before", + "unlink-retry", + "generic-retry", + "foreign-retry", + "cleanup-failure", + "later-foreign", + ] as const)("replacement retirement reports its exact locked frontier (%s)", async (phase) => { + state = new CompactionCancellation( + h.historyService.getCompactionCancellationStorage(workspaceId) + ); + await state.cancel(); + const captured = await h.historyService.captureCompactionReplacement(workspaceId); + assert(captured.success); + assert(captured.data.nonce); + const accepted = await h.historyService.acceptCompactionReplacement( + workspaceId, + captured.data, + { kind: "append", messages: [createMuxMessage("replacement", "user", "Fresh input")] }, + { isCurrent: () => true, onCommitted: () => undefined } + ); + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + const receipts: Array<[CompactionReplacementCapture, CompactionReplacementCapture]> = []; + const notify = (before: CompactionReplacementCapture, after: CompactionReplacementCapture) => { + expect(nodeFs.existsSync(storage.path)).toBe(false); + expect(nodeFs.existsSync(historyWriteLockPath(h.config.rootDir, workspaceId))).toBe(true); + expect(state.blocksRecovery).toBe(false); + receipts.push([before, after]); + return undefined; + }; + const remove = nodeFs.rmSync; + let failUnlink = + phase === "unlink-retry" || phase === "generic-retry" || phase === "foreign-retry"; + spyOn(nodeFs, "rmSync").mockImplementation((file, options) => { + if (file === storage.path && failUnlink) throw new Error("unlink unavailable"); + remove(file, options); + }); + if (phase === "cleanup-failure") { + const lock = h.historyService.withCompactionStorageLock.bind(h.historyService); + spyOn(h.historyService, "withCompactionStorageLock").mockImplementationOnce( + async (...args) => { + await lock(...args); + throw new Error("post-commit cleanup unavailable"); + } + ); + } + if (phase === "foreign-before") { + const other = new CompactionCancellation( + foreign.getCompactionCancellationStorage(workspaceId) + ); + await other.cancel({ retainUntilReplacement: true }); + } + const retiring = state.retireReplacement(accepted.data.witness, notify); + if (phase === "foreign-before") { + expect(await retiring).toBe("superseded"); + expect(receipts).toEqual([]); + expect((await storage.read())?.nonce).not.toBe(captured.data.nonce); + return; + } + if (failUnlink) { + await assert.rejects(retiring, /unlink unavailable/); + expect(receipts).toEqual([]); + expect((await storage.read())?.nonce).toBe(captured.data.nonce); + failUnlink = false; + if (phase === "foreign-retry") { + const other = new CompactionCancellation( + foreign.getCompactionCancellationStorage(workspaceId) + ); + await other.cancel({ retainUntilReplacement: true }); + const bytes = await fs.readFile(storage.path); + expect(await state.retry()).toBe("superseded"); + expect(receipts).toEqual([]); + expect(await fs.readFile(storage.path)).toEqual(bytes); + return; + } + expect( + await (phase === "generic-retry" ? state.retire(captured.data.nonce) : state.retry()) + ).toBe("applied"); + } else if (phase === "cleanup-failure") { + await assert.rejects(retiring, /post-commit cleanup unavailable/); + expect(state.needsPersistence).toBe(true); + expect(await state.retry()).toBe("superseded"); + } else expect(await retiring).toBe("applied"); + expect(receipts).toEqual([ + [captured.data, { nonce: null, generation: captured.data.generation }], + ]); + if (phase === "later-foreign") { + const other = new CompactionCancellation( + foreign.getCompactionCancellationStorage(workspaceId) + ); + await other.cancel({ retainUntilReplacement: true }); + const next = await h.historyService.acceptCompactionReplacement( + workspaceId, + receipts[0][1], + { kind: "append", messages: [createMuxMessage("queued", "user", "Queued input")] }, + { isCurrent: () => true, onCommitted: () => undefined } + ); + expect(next).toEqual({ success: true, data: { kind: "superseded" } }); + } + }); + it.each(["publish", "narrow", "confirm", "retire"] as const)( "reports the exact %s receipt before cleanup or lock release", async (phase) => { @@ -631,7 +780,7 @@ describe("inactive real cancellation storage", () => { /not configured/ ); const refusing = new FileCompactionCancellationStorage(foreign, workspaceId, () => - Promise.resolve(false) + Promise.resolve(() => Promise.resolve(false)) ); await assert.rejects( refusing.mutate(mutation, () => true, mutationCommitted), @@ -639,18 +788,17 @@ describe("inactive real cancellation storage", () => { ); expect(await storage.read()).toEqual(retained); let current = true; - const verifying = new FileCompactionCancellationStorage( - foreign, - workspaceId, - async (witness) => { - expect(witness.nonce).toBe(retained.nonce); + const verifying = new FileCompactionCancellationStorage(foreign, workspaceId, (witness) => { + expect(witness.nonce).toBe(retained.nonce); + expect(nodeFs.existsSync(historyWriteLockPath(h.config.rootDir, workspaceId))).toBe(false); + return Promise.resolve(async () => { expect( await fs.readFile(historyWriteLockPath(h.config.rootDir, workspaceId), "utf8") ).not.toBe(""); current = false; return true; - } - ); + }); + }); expect(await verifying.mutate(mutation, () => current, mutationCommitted)).toBe("superseded"); expect(await storage.read()).toEqual(retained); // This injected authority exercises the seam only; real accepted-row proof belongs to H2b. @@ -791,13 +939,26 @@ describe("inactive real cancellation storage", () => { expect(await storage.read()).toEqual(failed); }); + it("overlapping Stop during generation staging returns superseded and preserves its successor", async () => { + let successor: ReturnType | undefined; + afterCompactionStaging(path.join(sessionDir, CONTINUOUS_COMPACTION_GENERATION_FILE), () => { + successor ??= state.cancel(); + }); + const first = await state.cancel().catch((error: unknown) => error); + expect(first).toBe("superseded"); + expect(await successor).toBe("applied"); + expect(await storage.read()).toEqual(await state.read()); + expect((await storage.read())?.nonce).toBeDefined(); + }); + it("a locally superseded publication cannot rename its staged cancellation", async () => { let current = true; const journal = h.historyService.getContinuousCompactionJournal(workspaceId); const advance = journal.advanceGenerationUnderHistoryLock.bind(journal); spyOn(journal, "advanceGenerationUnderHistoryLock").mockImplementationOnce(async (...args) => { - await advance(...args); + const result = await advance(...args); current = false; + return result; }); expect(await storage.mutate(publication("stale"), () => current, mutationCommitted)).toBe( "superseded" @@ -1007,8 +1168,9 @@ describe("inactive real cancellation storage", () => { const journal = h.historyService.getContinuousCompactionJournal(workspaceId); const advance = journal.advanceGenerationUnderHistoryLock.bind(journal); spyOn(journal, "advanceGenerationUnderHistoryLock").mockImplementationOnce(async (...args) => { - await advance(...args); + const result = await advance(...args); current = false; + return result; }); expect(await storage.repair(() => current, committed)).toBeNull(); expect(await fs.readFile(storage.path, "utf8")).toBe("{bad again"); diff --git a/src/node/services/compactionCancellation.ts b/src/node/services/compactionCancellation.ts index 6f7abd3bd8..58cfc84e57 100644 --- a/src/node/services/compactionCancellation.ts +++ b/src/node/services/compactionCancellation.ts @@ -9,6 +9,26 @@ import { isPlainObject } from "@/common/utils/isPlainObject"; import type { HistoryService } from "./historyService"; import { publishCompactionFile } from "./continuousCompactionJournal"; import { hasAmbiguousResetKeys } from "./historyScanner"; +import type { MuxMessage } from "@/common/types/message"; +import { log } from "./log"; + +export interface CompactionReplacementCapture { + nonce: string | null; + generation: string | undefined; +} + +export type CompactionReplacementOperation = + | { + kind: "append"; + messages: MuxMessage[]; + /** Record ordinary input against this capture without replacing its Stop. */ + preserveCancellation?: true; + } + | { kind: "resume"; message: MuxMessage }; + +export type CompactionReplacementOutcome = + | { kind: "accepted"; witness: CompactionCancellationReplacementWitness | null } + | { kind: "superseded" | "skipped" }; export interface CompactionCancellationSummary { id: string; @@ -46,6 +66,10 @@ export type CompactionCancellationMutation = kind: "retire"; nonce: string; replacementWitness?: CompactionCancellationReplacementWitness; + onRetired?: ( + predecessor: CompactionReplacementCapture, + successor: CompactionReplacementCapture + ) => undefined; }; export type CompactionCancellationMutationOutcome = "applied" | "superseded"; @@ -71,7 +95,11 @@ export interface CompactionCancellationStorage { mutate( mutation: CompactionCancellationMutation, isCurrent: () => boolean, - onCommitted: (record: CompactionCancellationRecord | null) => undefined + onCommitted: ( + record: CompactionCancellationRecord | null, + retiredPredecessor?: CompactionReplacementCapture + ) => undefined, + signal?: AbortSignal ): Promise; /** * Re-read under the lock; preserve newer valid records. Neutralize obsolete recovery @@ -119,11 +147,12 @@ export class FileCompactionCancellationStorage implements CompactionCancellation constructor( private readonly history: HistoryService, private readonly workspaceId: string, - // This verifier runs under both history locks and must not re-enter them. - // No default authority: a caller-provided nonce alone cannot retire retention. - private readonly verifyReplacementUnderHistoryLock?: ( - witness: CompactionCancellationReplacementWitness - ) => Promise + // Prepare expensive history evidence outside both locks; the returned verifier only + // revalidates it under the lock. A caller-provided nonce alone grants no authority. + private readonly prepareVerification?: ( + witness: CompactionCancellationReplacementWitness, + signal?: AbortSignal + ) => Promise<() => Promise> ) { this.path = path.join( path.dirname(history.getContinuousCompactionJournal(workspaceId).path), @@ -172,11 +201,30 @@ export class FileCompactionCancellationStorage implements CompactionCancellation } } - mutate( + async mutate( mutation: CompactionCancellationMutation, isCurrent: () => boolean, - onCommitted: (record: CompactionCancellationRecord | null) => undefined + onCommitted: ( + record: CompactionCancellationRecord | null, + retiredPredecessor?: CompactionReplacementCapture + ) => undefined, + signal?: AbortSignal ): Promise { + if (!isCurrent()) return "superseded"; + let verifyReplacementUnderHistoryLock: (() => Promise) | undefined; + try { + if (mutation.kind === "retire" && mutation.replacementWitness) + verifyReplacementUnderHistoryLock = await this.prepareVerification?.( + mutation.replacementWitness, + signal + ); + } catch (error) { + // Superseded evidence must release its scanner before the queued Stop can publish. + // A current operation's I/O or cancellation failure is still a real failure. + if (!isCurrent()) return "superseded"; + throw error; + } + if (!isCurrent()) return "superseded"; return this.history.withCompactionStorageLock(this.workspaceId, async (_dir, checkLock) => { if (!isCurrent()) return "superseded"; if ( @@ -211,9 +259,14 @@ export class FileCompactionCancellationStorage implements CompactionCancellation const { contents, record: committed } = serializeCancellation(mutation.record); // Record admission before advancing, and advancement at its commit point. // Unobserved failures remain blocking until a new explicit Stop captures a frontier. - await journal.advanceGenerationUnderHistoryLock((advanced) => { - frontier.generation = advanced; - }, checkLock); + const advanced = await journal.advanceGenerationUnderHistoryLock( + (advanced) => { + frontier.generation = advanced; + }, + checkLock, + isCurrent + ); + if (!advanced) return "superseded"; return (await publishCompactionFile( this.path, contents, @@ -252,15 +305,31 @@ export class FileCompactionCancellationStorage implements CompactionCancellation } const witness = mutation.replacementWitness; if (witness) { - if (!this.verifyReplacementUnderHistoryLock) + if (!verifyReplacementUnderHistoryLock) throw new Error("Replacement witness verification is not configured"); - if (witness.nonce !== nonce || !(await this.verifyReplacementUnderHistoryLock(witness))) - throw new Error("Replacement witness was not verified"); + try { + if (witness.nonce !== nonce || !(await verifyReplacementUnderHistoryLock())) + throw new Error("Replacement witness was not verified"); + } catch (error) { + // Supersession can also abort stamp revalidation or flushes after taking the lock. + if (!isCurrent()) return "superseded"; + throw error; + } } else if (current.retainUntilReplacement) return "superseded"; + // Queued work may follow this exact replacement across unlink, but cannot adopt + // a later Stop. Capture the frontier under the same lock and report only deletion. + const retiredPredecessor = mutation.onRetired + ? { + nonce, + generation: await this.history + .getContinuousCompactionJournal(this.workspaceId) + .captureGenerationUnderHistoryLock(), + } + : undefined; await checkLock(); if (!isCurrent()) return "superseded"; rmSync(this.path, { force: true }); - onCommitted(null); + onCommitted(null, retiredPredecessor); return "applied"; }); } @@ -269,34 +338,68 @@ export class FileCompactionCancellationStorage implements CompactionCancellation isCurrent: () => boolean, onCommitted: () => undefined ): Promise { - return this.history.withCompactionStorageLock(this.workspaceId, async (_dir, checkLock) => { - if (!isCurrent()) return null; - try { - return await this.read(); - } catch (error) { - if (!(error instanceof MalformedCompactionCancellationError)) throw error; - } - if (!isCurrent()) return null; - await this.history - .getContinuousCompactionJournal(this.workspaceId) - .advanceGenerationUnderHistoryLock(undefined, checkLock); + return this.history.withCompactionStorageLock(this.workspaceId, (_dir, checkLock) => + this.repairUnderHistoryLock(isCurrent, onCommitted, checkLock) + ); + } + + /** Admission captures repair and its resulting frontier without releasing the history lock. */ + async repairUnderHistoryLock( + isCurrent: () => boolean, + onCommitted: () => void, + checkLock: () => Promise, + replaceUnreadable = false + ): Promise { + if (!isCurrent()) return null; + let replacement: CompactionCancellationRecord | undefined; + try { + return await this.read(); + } catch (error) { + if (error instanceof CompactionCancellationReadRefusedError && replaceUnreadable) { + // Explicit replacement preserves the same retention floor as readForReplacement's + // fallback Stop, but acquires its receipt without a foreign writer entering between. + replacement = { + version: 1, + nonce: randomUUID(), + scope: { kind: "unresolved" }, + retainUntilReplacement: true, + }; + } else if (!(error instanceof MalformedCompactionCancellationError)) throw error; + } + if (!isCurrent()) return null; + await this.history + .getContinuousCompactionJournal(this.workspaceId) + .advanceGenerationUnderHistoryLock(undefined, checkLock); + if ( + !(await this.history.neutralizeCompactionRecoveryUnderHistoryLock( + this.workspaceId, + isCurrent, + checkLock + )) || + !isCurrent() + ) + return null; + await checkLock(); + if (!isCurrent()) return null; + // Keep malformed bytes until all obsolete recovery has been neutralized. + // No await separates removal from the repair receipt or its final guard. + if (replacement) { + const serialized = serializeCancellation(replacement); if ( - !(await this.history.neutralizeCompactionRecoveryUnderHistoryLock( - this.workspaceId, + !(await publishCompactionFile( + this.path, + serialized.contents, isCurrent, + onCommitted, checkLock - )) || - !isCurrent() + )) ) - return null; - await checkLock(); - if (!isCurrent()) return null; - // Keep malformed bytes until all obsolete recovery has been neutralized. - // No await separates removal from the repair receipt or its final guard. - rmSync(this.path, { force: true }); - onCommitted(); - return null; - }); + throw new Error("Cancellation repair was superseded"); + return serialized.record; + } + rmSync(this.path, { force: true }); + onCommitted(); + return null; } } @@ -313,6 +416,7 @@ export class CompactionCancellation { Promise.resolve(undefined); private unsettled = false; private inFlight = false; + private mutationAbort?: AbortController; private readGeneration = 0; private acceptedReadGeneration = 0; private repairedHistoryRevision = 0; @@ -500,13 +604,20 @@ export class CompactionCancellation { return this.persist({ kind: "retire", nonce }); } - retireReplacement(witness: CompactionCancellationReplacementWitness) { + retireReplacement( + witness: CompactionCancellationReplacementWitness, + onRetired?: Extract["onRetired"] + ) { if (this.current?.nonce !== witness.nonce) return Promise.resolve(undefined); + // An ordinary cleanup retry must retain the original queued-work receipt obligation. + if (this.mutation?.kind === "retire" && this.mutation.nonce === witness.nonce) + onRetired ??= this.mutation.onRetired; this.replacementNonce = witness.nonce; return this.persist({ kind: "retire", nonce: witness.nonce, replacementWitness: { ...witness }, + onRetired, }); } @@ -533,6 +644,9 @@ export class CompactionCancellation { mutation: CompactionCancellationMutation ): Promise { this.mutation = mutation; + // Supersession cancels only this attempt's provisional scan, never its successor. + this.mutationAbort?.abort(new Error("Compaction mutation superseded")); + const abort = (this.mutationAbort = new AbortController()); this.unsettled = true; this.inFlight = true; const generation = this.acceptedReadGeneration; @@ -542,13 +656,29 @@ export class CompactionCancellation { .then(async (): Promise => { if (!isCurrent()) return "superseded"; if (mutation.kind === "publish") mutation.publication.attempts++; - const outcome = await this.storage.mutate(mutation, isCurrent, (record) => { - if (!isCurrent()) return; - this.current = structuredClone(record); - // Commit invalidates pre-deletion reads before lock release. A later foreign - // read must survive acknowledgment delayed by adapter cleanup. - this.acceptedReadGeneration = ++this.readGeneration; - }); + const outcome = await this.storage.mutate( + mutation, + isCurrent, + (record, retired) => { + if (!isCurrent()) return; + this.current = structuredClone(record); + // Commit invalidates pre-deletion reads before lock release. A later foreign + // read must survive acknowledgment delayed by adapter cleanup. + this.acceptedReadGeneration = ++this.readGeneration; + if (mutation.kind === "retire" && record === null && retired) { + try { + mutation.onRetired?.( + { ...retired }, + { nonce: null, generation: retired.generation } + ); + } catch (error) { + // Notification cannot undo the receipt or skip the unlink's directory flush. + log.warn("Compaction retirement observer failed", error); + } + } + }, + abort.signal + ); if (isCurrent()) { this.unsettled = false; if (outcome === "superseded" && this.acceptedReadGeneration === generation) diff --git a/src/node/services/continuousCompactionJournal.ts b/src/node/services/continuousCompactionJournal.ts index 4a415c0936..eb5577ea09 100644 --- a/src/node/services/continuousCompactionJournal.ts +++ b/src/node/services/continuousCompactionJournal.ts @@ -159,22 +159,26 @@ export class ContinuousCompactionJournalStore { /** Caller already holds the history lock; never re-enter the journal queue here. */ async advanceGenerationUnderHistoryLock( onCommitted?: (generation: string) => undefined, - assertStillOwned?: () => Promise - ): Promise { + assertStillOwned?: () => Promise, + isCurrent: () => boolean = () => true + ): Promise { const generation = randomUUID(); - await publishCompactionFile( + const committed = await publishCompactionFile( path.join(path.dirname(this.path), CONTINUOUS_COMPACTION_GENERATION_FILE), generation, - () => true, + isCurrent, // The cancellation retry must learn its exact frontier at rename, before // cleanup or lock release can fail or admit a foreign generation. () => onCommitted?.(createHash("sha256").update(generation).digest("hex")), assertStillOwned ); + return committed; } advanceGeneration(): Promise { - return this.enqueue(() => this.advanceGenerationUnderHistoryLock()); + return this.enqueue(async () => { + await this.advanceGenerationUnderHistoryLock(); + }); } private async readUnderHistoryLock(): Promise { diff --git a/src/node/services/historyAppendProvenance.test.ts b/src/node/services/historyAppendProvenance.test.ts index 63a8d9ee21..7083549afc 100644 --- a/src/node/services/historyAppendProvenance.test.ts +++ b/src/node/services/historyAppendProvenance.test.ts @@ -94,6 +94,24 @@ afterEach(async () => { }); describe("history append provenance", () => { + test("single replacement acceptance preserves the append cursor's fixed snapshot", async () => { + const cursor = await startCursor(); + const capture = await fixture.historyService.captureCompactionReplacement(ws); + assert(capture.success); + const accepted = await fixture.historyService.acceptCompactionReplacement( + ws, + capture.data, + { + kind: "append", + messages: [createMuxMessage("accepted", "user", "new input")], + }, + { isCurrent: () => true, onCommitted: () => undefined } + ); + expect(accepted).toEqual({ success: true, data: { kind: "accepted", witness: null } }); + expect((await resume(cursor)).map((row) => row.id)).toEqual(["row-1", "row-2"]); + expect((await store.read()).receipt?.epoch).toBe(cursor.provenanceEpoch); + }); + test("bootstraps privately without an existing receipt and uses exact bigint stamps", async () => { await fs.rm(store.receiptPath); const cursor = await startCursor(); diff --git a/src/node/services/historyAppendProvenance.ts b/src/node/services/historyAppendProvenance.ts index 308cf5484e..34a6750fd7 100644 --- a/src/node/services/historyAppendProvenance.ts +++ b/src/node/services/historyAppendProvenance.ts @@ -280,7 +280,8 @@ export class HistoryAppendProvenance { async appendChat( bytes: Buffer, atomic = false, - publishAtomic?: (filePath: string, bytes: Buffer) => Promise + publishAtomic?: (filePath: string, bytes: Buffer) => Promise, + publishAppend?: (filePath: string, bytes: Buffer, createsFile: boolean) => Promise ): Promise { const transaction = transactions.getStore(); assert( @@ -330,7 +331,8 @@ export class HistoryAppendProvenance { await tailHandle.close(); } } - await fs.appendFile(this.chatPath, bytes); + if (publishAppend) await publishAppend(this.chatPath, bytes, before.chat == null); + else await fs.appendFile(this.chatPath, bytes); published = true; const handle = await fs.open(this.chatPath, "r"); try { diff --git a/src/node/services/historyService.contextReset.test.ts b/src/node/services/historyService.contextReset.test.ts index 869f12f1c8..24b652178f 100644 --- a/src/node/services/historyService.contextReset.test.ts +++ b/src/node/services/historyService.contextReset.test.ts @@ -34,8 +34,9 @@ describe("empty context reset transactions", () => { async (...args) => { entered.resolve(); await release.promise; - await advance(...args); + const result = await advance(...args); order.push("fenced"); + return result; } ); const fencing = h.historyService.fenceEmptyContext(workspaceId); diff --git a/src/node/services/historyService.publication.test.ts b/src/node/services/historyService.publication.test.ts index 83ef5a4363..5e3d1305b4 100644 --- a/src/node/services/historyService.publication.test.ts +++ b/src/node/services/historyService.publication.test.ts @@ -15,7 +15,7 @@ import { historyWriteLockPath } from "./workspaceRemoval"; type PublicationObserver = NonNullable< Parameters[2] >; -// Compile-time assertion only; behavioral tests below check receipt timing at rename. +// Compile-time assertion only; behavioral tests below check receipt timing at publication. expectTypeOf<() => Promise>().not.toMatchTypeOf(); describe("HistoryService private publication seam", () => { @@ -86,7 +86,26 @@ describe("HistoryService private publication seam", () => { return result.success ? result.data : []; } - function afterPublicationStaging(action: () => void | Promise) { + function afterPublicationPreparation( + kind: "single" | "batch" | "update", + action: () => void | Promise + ) { + // Single rows prepare an append handle; batches and updates stage a replacement file. + if (kind === "single") { + const open = fs.open; + return spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && args[1] === "a") { + try { + await action(); + } catch (error) { + await handle.close(); + throw error; + } + } + return handle; + }); + } const atomic = atomicWrite.default; return spyOn(atomicWrite, "default").mockImplementation( new Proxy(atomic, { @@ -117,8 +136,8 @@ describe("HistoryService private publication seam", () => { let successorReceiptBytes = Buffer.alloc(0); let successor: Awaited> | undefined; let commits = 0; - const staging = afterPublicationStaging(async () => { - // Model an expired birth-less lease while staging is held, then let + const staging = afterPublicationPreparation(kind, async () => { + // Model an expired birth-less lease while publication is prepared, then let // the real lock protocol reclaim it and a successor publish new bytes. const token = await fs.readFile(lockPath, "utf8"); await fs.writeFile(lockPath, token.split(":").slice(0, 2).join(":")); @@ -170,7 +189,7 @@ describe("HistoryService private publication seam", () => { let staged = false; let current = true; let commits = 0; - const staging = afterPublicationStaging(() => { + const staging = afterPublicationPreparation(kind, () => { staged = true; }); const readFile = fs.readFile; @@ -251,34 +270,41 @@ describe("HistoryService private publication seam", () => { else expect(after?.epoch).toBe(before?.epoch); }); - it(`${kind}: rejects ownership after staging without publishing or notifying`, async () => { + it(`${kind}: rejects ownership after preparation without publishing or notifying`, async () => { const before = await fs.readFile(chatPath); - let staged = false; + let prepared = false; let commits = 0; - const result = await publish(kind, { - isCurrent: () => { - staged = nodeFs - .readdirSync(path.dirname(chatPath)) - .some((name) => name.startsWith("chat.jsonl.publication-")); - return false; - }, - onCommitted: () => { - commits++; - }, + const preparation = afterPublicationPreparation(kind, () => { + prepared = true; }); - expect(result.success).toBe(false); - expect(staged).toBe(true); - expect(commits).toBe(0); - expect(await fs.readFile(chatPath)).toEqual(before); - expect( - (await fs.readdir(path.dirname(chatPath))).filter((name) => name.includes(".publication-")) - ).toEqual([]); + try { + const result = await publish(kind, { + isCurrent: () => !prepared, + onCommitted: () => { + commits++; + }, + }); + expect(result.success).toBe(false); + expect(prepared).toBe(true); + expect(commits).toBe(0); + expect(await fs.readFile(chatPath)).toEqual(before); + expect( + (await fs.readdir(path.dirname(chatPath))).filter((name) => + name.includes(".publication-") + ) + ).toEqual([]); + } finally { + preparation.mockRestore(); + } }); - it(`${kind}: a failed rename leaves the old history and no commit receipt`, async () => { + it(`${kind}: a failed publication leaves the old history and no commit receipt`, async () => { const before = await fs.readFile(chatPath); let commits = 0; - const rename = spyOn(nodeFs, "renameSync").mockImplementationOnce(() => { + const failure = spyOn( + nodeFs, + kind === "single" ? "writeSync" : "renameSync" + ).mockImplementationOnce(() => { throw new Error("publication unavailable"); }); try { @@ -292,21 +318,36 @@ describe("HistoryService private publication seam", () => { expect(commits).toBe(0); expect(await fs.readFile(chatPath)).toEqual(before); } finally { - rename.mockRestore(); + failure.mockRestore(); } }); - it(`${kind}: observer and staging-cleanup failures cannot turn a commit into a retry`, async () => { + it(`${kind}: observer and cleanup failures cannot turn a commit into a retry`, async () => { let commits = 0; let cleanupFailures = 0; const remove = fs.rm; - const failure = spyOn(fs, "rm").mockImplementation((target, options) => { - if (String(target).startsWith(`${chatPath}.publication-`)) { - cleanupFailures++; - return Promise.reject(new Error("cleanup unavailable")); - } - return remove(target, options); - }); + const open = fs.open; + const failure = + kind === "single" + ? spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && args[1] === "a") { + const close = handle.close.bind(handle); + spyOn(handle, "close").mockImplementation(async () => { + await close(); + cleanupFailures++; + throw new Error("cleanup unavailable"); + }); + } + return handle; + }) + : spyOn(fs, "rm").mockImplementation((target, options) => { + if (String(target).startsWith(`${chatPath}.publication-`)) { + cleanupFailures++; + return Promise.reject(new Error("cleanup unavailable")); + } + return remove(target, options); + }); try { const result = await publish(kind, { isCurrent: () => true, diff --git a/src/node/services/historyService.replacement.test.ts b/src/node/services/historyService.replacement.test.ts new file mode 100644 index 0000000000..ce5e5a1b4c --- /dev/null +++ b/src/node/services/historyService.replacement.test.ts @@ -0,0 +1,3174 @@ +import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs/promises"; +import * as nodeFs from "node:fs"; +import * as path from "node:path"; +import * as atomicWrite from "write-file-atomic"; +import { createMuxMessage, type MuxMessage } from "@/common/types/message"; +import { Ok } from "@/common/types/result"; +import { createContextBudgetRejectedMessage } from "@/common/utils/messages/contextBudgetRejection"; +import { MuxMessageSchema } from "@/common/orpc/schemas/message"; +import { workspaceFileLocks } from "@/node/utils/concurrency/workspaceFileLocks"; +import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; +import { CONTINUOUS_COMPACTION_GENERATION_FILE } from "@/constants/continuousCompaction"; +import { SESSION_HISTORY_MAX_LINE_BYTES } from "@/common/constants/contextBudget"; +import type { ContinuousCompactionJournal } from "@/common/orpc/schemas/continuousCompaction"; +import { + CompactionCancellation, + type CompactionReplacementOperation, +} from "./compactionCancellation"; +import { HistoryService } from "./historyService"; +import { createTestHistoryService } from "./testHistoryService"; +import { historyWriteLockPath } from "./workspaceRemoval"; +import { HistoryAppendProvenance } from "./historyAppendProvenance"; + +const workspaceId = "replacement"; +const noObserver = { isCurrent: () => true, onCommitted: () => undefined }; + +describe("compaction replacement acceptance", () => { + let fixture: Awaited>; + let history: HistoryService; + let stop: CompactionCancellation; + let chatPath: string; + + beforeEach(async () => { + fixture = await createTestHistoryService(); + history = fixture.historyService; + stop = new CompactionCancellation(history.getCompactionCancellationStorage(workspaceId)); + chatPath = path.join(fixture.config.sessionsDir, workspaceId, "chat.jsonl"); + await history.appendToHistory(workspaceId, createMuxMessage("prior", "user", "question")); + }); + afterEach(async () => { + mock.restore(); + await fixture.cleanup(); + }); + + async function capture() { + const result = await history.captureCompactionReplacement(workspaceId); + assert(result.success); + return result.data; + } + async function rows() { + const result = await history.getLastMessages(workspaceId, 20); + assert(result.success); + return result.data; + } + it("admission captures repaired malformed state before a waiting foreign Stop", async () => { + const storage = history.getCompactionCancellationStorage(workspaceId); + await fs.writeFile(storage.path, "{malformed cancellation"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const neutralize = history.neutralizeCompactionRecoveryUnderHistoryLock.bind(history); + spyOn(history, "neutralizeCompactionRecoveryUnderHistoryLock").mockImplementationOnce( + async (...args) => { + entered.resolve(); + await release.promise; + return neutralize(...args); + } + ); + const repaired = mock(() => undefined); + const acquiring = history.captureCompactionReplacement(workspaceId, { onRepaired: repaired }); + await entered.promise; + const foreign = new CompactionCancellation( + new HistoryService(fixture.config).getCompactionCancellationStorage(workspaceId) + ); + const stopping = foreign.cancel(); + release.resolve(); + const acquired = await acquiring; + await stopping; + assert(acquired.success); + expect(acquired.data.nonce).toBeNull(); + expect(repaired).toHaveBeenCalledTimes(1); + expect(await capture()).not.toEqual(acquired.data); + expect( + await history.acceptCompactionReplacement( + workspaceId, + acquired.data, + { + kind: "append", + messages: [createMuxMessage("stale", "user", "older request")], + }, + noObserver + ) + ).toEqual(Ok({ kind: "superseded" })); + expect((await rows()).map((row) => row.id)).toEqual(["prior"]); + }); + + it.each([false, true])( + "admission capture retains unsupported bytes unless replacement is explicit (manual=%s)", + async (manual) => { + const storage = history.getCompactionCancellationStorage(workspaceId); + const unsupported = JSON.stringify({ + version: 999, + nonce: "future", + scope: { kind: "unresolved" }, + }); + await fs.writeFile(storage.path, unsupported); + const repaired = mock(() => undefined); + const acquired = await history.captureCompactionReplacement(workspaceId, { + onRepaired: repaired, + replaceUnreadable: manual, + }); + if (!manual) { + expect(acquired.success).toBe(false); + expect(await fs.readFile(storage.path, "utf8")).toBe(unsupported); + expect(repaired).not.toHaveBeenCalled(); + } else { + assert(acquired.success); + expect(await storage.read()).toMatchObject({ + nonce: acquired.data.nonce, + retainUntilReplacement: true, + }); + expect(acquired.data.generation).toBeDefined(); + expect(repaired).toHaveBeenCalledTimes(1); + } + } + ); + + it.each(["stamps", "flush"] as const)( + "superseding locked witness %s returns superseded after disposal", + async (phase) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const accepted = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + let locked = false; + let armed = true; + let opened = 0; + let closed = 0; + const hold = async () => { + if (!locked || !armed) return; + armed = false; + entered.resolve(); + await release.promise; + }; + const lock = history.withCompactionStorageLock.bind(history); + spyOn(history, "withCompactionStorageLock").mockImplementation((id, callback) => + lock(id, async (...args) => { + locked = true; + try { + return await callback(...args); + } finally { + locked = false; + } + }) + ); + const stamps = HistoryAppendProvenance.prototype.stamps; // eslint-disable-line @typescript-eslint/unbound-method -- original receiver retained + spyOn(HistoryAppendProvenance.prototype, "stamps").mockImplementation(async function ( + this: HistoryAppendProvenance + ) { + const result = await stamps.call(this); + if (phase === "stamps") await hold(); + return result; + }); + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && args[1] === "r+") { + opened++; + const sync = handle.sync.bind(handle); + const close = handle.close.bind(handle); + spyOn(handle, "sync").mockImplementation(async () => { + await sync(); + if (phase === "flush") await hold(); + }); + spyOn(handle, "close").mockImplementation(async () => { + await close(); + closed++; + }); + } + return handle; + }); + const retiring = stop.retireReplacement(accepted.data.witness).then( + (result) => result, + (error: unknown) => error + ); + await entered.promise; + const successor = stop.cancel({ retainUntilReplacement: true }); + release.resolve(); + const [retired, published] = await Promise.all([retiring, successor]); + expect(retired).toBe("superseded"); + expect(published).toBe("applied"); + expect(closed).toBe(opened); + if (phase === "flush") expect(opened).toBeGreaterThan(0); + expect((await capture()).nonce).not.toBe(expected.nonce); + expect((await rows()).some((row) => row.id === "accepted")).toBe(true); + } + ); + + it("a current locked witness flush error remains a failure with retry debt", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const accepted = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + const failure = new Error("injected current witness flush error"); + const open = fs.open; + const probe = spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && args[1] === "r+") + spyOn(handle, "sync").mockRejectedValueOnce(failure); + return handle; + }); + await assert.rejects( + stop.retireReplacement(accepted.data.witness), + (error) => error === failure + ); + expect(stop.needsPersistence).toBe(true); + expect((await capture()).nonce).toBe(expected.nonce); + probe.mockRestore(); + expect(await stop.retry()).toBe("applied"); + expect((await capture()).nonce).toBeNull(); + }); + + it.each(["chat", "archive", "giant archive"] as const)( + "same nonce on a different eligible identity in %s cannot retire Stop", + async (artifact) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + assert(expected.nonce); + const accepted = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + const second = createMuxMessage( + "second-identity", + "user", + artifact === "giant archive" + ? "x".repeat(SESSION_HISTORY_MAX_LINE_BYTES + 1) + : "other input", + { historySequence: 99, compactionReplacementNonce: expected.nonce } + ); + const file = + artifact === "chat" ? chatPath : path.join(path.dirname(chatPath), "chat-archive.jsonl"); + await fs.appendFile(file, JSON.stringify(second) + "\n"); + const bytes = await fs.readFile(file); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce)).toEqual( + Ok(null) + ); + await assert.rejects(stop.retireReplacement(accepted.data.witness), /not verified/); + expect((await capture()).nonce).toBe(expected.nonce); + expect(await fs.readFile(file)).toEqual(bytes); + } + ); + + it.each(["append", "resume"] as const)( + "an ineligible legacy system stamp does not consume a fresh %s replacement", + async (intent) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + assert(expected.nonce); + const invalid = + JSON.stringify({ + ...createMuxMessage("old-system-stamp", "user", "legacy context", { + historySequence: 99, + compactionReplacementNonce: expected.nonce, + }), + role: ["system"], + }) + "\n"; + await fs.appendFile(chatPath, invalid); + const replacement: CompactionReplacementOperation = + intent === "append" + ? await operation("single") + : { kind: "resume", message: (await rows()).find((row) => row.id === "prior")! }; + const accepted = await history.acceptCompactionReplacement( + workspaceId, + expected, + replacement, + noObserver + ); + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + expect((await fs.readFile(chatPath, "utf8")).includes(invalid.trimEnd())).toBe(true); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce)).toEqual( + Ok({ nonce: expected.nonce }) + ); + expect(await stop.retireReplacement(accepted.data.witness)).toBe("applied"); + expect((await capture()).nonce).toBeNull(); + } + ); + + it("an ineligible system stamp still occupies its identity", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + await fs.appendFile( + chatPath, + JSON.stringify({ + ...createMuxMessage("occupied", "user", "legacy context", { + historySequence: 99, + compactionReplacementNonce: expected.nonce!, + }), + role: ["system"], + }) + "\n" + ); + const before = await fs.readFile(chatPath); + expect( + await history.acceptCompactionReplacement( + workspaceId, + expected, + { + kind: "append", + messages: [createMuxMessage("occupied", "user", "new input")], + }, + noObserver + ) + ).toEqual(Ok({ kind: "skipped" })); + expect(await fs.readFile(chatPath)).toEqual(before); + expect((await capture()).nonce).toBe(expected.nonce); + }); + + it.each(["system", "user", "assistant"] as const)( + "legacy array role %s preserves its existing replacement authority", + async (role) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + assert(expected.nonce); + const target = { + ...createMuxMessage("legacy-role", "user", "legacy row", { historySequence: 1 }), + role: [role], + }; + await fs.appendFile(chatPath, JSON.stringify(target) + "\n"); + const before = await fs.readFile(chatPath); + const persisted = (await rows()).find((row) => row.id === target.id); + assert(persisted); + const accepted = await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "resume", message: persisted }, + noObserver + ); + if (role === "system") { + expect(accepted).toEqual(Ok({ kind: "skipped" })); + expect(await fs.readFile(chatPath)).toEqual(before); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce)).toEqual( + Ok(null) + ); + expect((await stop.read())?.nonce).toBe(expected.nonce); + } else { + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + expect(await stop.retireReplacement(accepted.data.witness)).toBe("applied"); + } + } + ); + + it("pre-stamped legacy system rows cannot authorize Stop retirement", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + assert(expected.nonce); + for (const size of [8, SESSION_HISTORY_MAX_LINE_BYTES + 1]) { + await fs.writeFile( + chatPath, + JSON.stringify({ + ...createMuxMessage("stamped-system", "user", "x".repeat(size), { + historySequence: 1, + compactionReplacementNonce: expected.nonce, + }), + role: ["system"], + }) + "\n" + ); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce)).toEqual( + Ok(null) + ); + await assert.rejects(stop.retireReplacement({ nonce: expected.nonce }), /not verified/); + expect((await history.getCompactionCancellationStorage(workspaceId).read())?.nonce).toBe( + expected.nonce + ); + } + }); + + it.each([1, 2])( + "superseding retirement %s times stops giant-row verification and disposes its handles", + async (count) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const accepted = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + const archivePath = path.join(path.dirname(chatPath), "chat-archive.jsonl"); + await fs.writeFile( + archivePath, + JSON.stringify( + createMuxMessage("giant", "assistant", "x".repeat(8 * 1024 * 1024), { + historySequence: 99, + }) + ) + "\n" + ); + const successors: Array> = []; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + let reads = 0; + let opens = 0; + let closes = 0; + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] !== archivePath || args[1] !== "r") return handle; + opens++; + const read = handle.read.bind(handle); + const close = handle.close.bind(handle); + spyOn(handle, "close").mockImplementation(async () => { + closes++; + await close(); + }); + spyOn(handle, "read").mockImplementation( + new Proxy(read, { + async apply(target, receiver, args: Parameters) { + const result = await (Reflect.apply(target, receiver, args) as ReturnType< + typeof read + >); + if (++reads === 1) { + entered.resolve(); + await release.promise; + } + return result; + }, + }) + ); + return handle; + }); + let successorSawDisposedScanner = false; + const lock = history.withCompactionStorageLock.bind(history); + spyOn(history, "withCompactionStorageLock").mockImplementation(async (...args) => { + if (successors.length > 0) successorSawDisposedScanner = closes === opens; + return lock(...args); + }); + const retiring = stop.retireReplacement(accepted.data.witness); + await entered.promise; + try { + for (let i = 0; i < count; i++) + successors.push(stop.cancel({ retainUntilReplacement: true })); + } finally { + release.resolve(); + } + expect(await retiring).toBe("superseded"); + expect(await Promise.all(successors)).toEqual( + Array.from({ length: count }, (_, i) => (i === count - 1 ? "applied" : "superseded")) + ); + expect(reads).toBe(1); + expect(opens).toBeGreaterThan(0); + expect(closes).toBe(opens); + expect(successorSawDisposedScanner).toBe(true); + expect((await stop.read())?.nonce).not.toBe(expected.nonce); + expect((await capture()).nonce).toEqual((await stop.read())?.nonce ?? null); + expect((await rows()).some((row) => row.id === "accepted")).toBe(true); + } + ); + + async function operation( + kind: "single" | "batch" | "resume" + ): Promise { + if (kind === "resume") return { kind, message: (await rows()).at(-1)! }; + const messages = [createMuxMessage("accepted", "user", "new input")]; + if (kind === "batch") messages.unshift(createMuxMessage("payload", "assistant", "context")); + return { kind: "append", messages }; + } + function afterStaging(action: () => void | Promise) { + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && args[1] === "a") { + try { + await action(); + } catch (error) { + await handle.close(); + throw error; + } + } + return handle; + }); + const atomic = atomicWrite.default; + spyOn(atomicWrite, "default").mockImplementation( + new Proxy(atomic, { + async apply(target, _thisArg, args: Parameters) { + const result = await target(...args); + if (String(args[0]).startsWith(`${chatPath}.publication-`)) await action(); + return result; + }, + }) + ); + } + + it.each(["chat", "archive", "oversized"] as const)( + "one Stop cannot accept a second replacement before retirement (%s)", + async (location) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const first = await history.acceptCompactionReplacement( + workspaceId, + expected, + { + kind: "append", + messages: [ + createMuxMessage( + "first-replacement", + "user", + location === "oversized" ? "x".repeat(SESSION_HISTORY_MAX_LINE_BYTES + 1) : "first" + ), + ], + }, + noObserver + ); + assert(first.success && first.data.kind === "accepted" && first.data.witness); + if (location === "archive") { + await fs.rename(chatPath, path.join(path.dirname(chatPath), "chat-archive.jsonl")); + } + const foreign = new HistoryService(fixture.config); + expect(await foreign.captureCompactionReplacement(workspaceId)).toEqual(Ok(expected)); + const observed = mock(() => undefined); + const second = await foreign.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "append", messages: [createMuxMessage("second-replacement", "user", "second")] }, + { isCurrent: () => true, onCommitted: observed } + ); + expect(second).toEqual(Ok({ kind: "skipped" })); + expect(observed).not.toHaveBeenCalled(); + expect((await rows()).some((row) => row.id === "second-replacement")).toBe(false); + expect(await stop.retireReplacement(first.data.witness)).toBe("applied"); + const successor = await capture(); + expect(successor).toEqual({ ...expected, nonce: null }); + const later = await foreign.acceptCompactionReplacement( + workspaceId, + successor, + { kind: "append", messages: [createMuxMessage("later", "user", "after retirement")] }, + noObserver + ); + expect(later).toEqual(Ok({ kind: "accepted", witness: null })); + } + ); + + it("a consumed nonce refuses another Resume but permits ordinary preserve-mode input", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const original = (await rows())[0]; + const first = await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "append", messages: [createMuxMessage("replacement", "user", "new input")] }, + noObserver + ); + assert(first.success && first.data.kind === "accepted" && first.data.witness); + const before = await fs.readFile(chatPath); + expect( + await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "resume", message: original }, + noObserver + ) + ).toEqual(Ok({ kind: "skipped" })); + expect(await fs.readFile(chatPath)).toEqual(before); + expect( + await history.acceptCompactionReplacement( + workspaceId, + expected, + { + kind: "append", + messages: [createMuxMessage("ordinary", "user", "ordinary input")], + preserveCancellation: true, + }, + noObserver + ) + ).toEqual(Ok({ kind: "accepted", witness: null })); + expect(await capture()).toEqual(expected); + expect((await rows()).at(-1)?.metadata?.compactionReplacementNonce).toBeUndefined(); + expect(await stop.retireReplacement(first.data.witness)).toBe("applied"); + }); + + it.each(["single", "batch"] as const)( + "publication bookkeeping uses the original %s array references after staging", + async (kind) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation(kind); + assert(input.kind === "append"); + const original = [...input.messages]; + const added = createMuxMessage("not-in-batch", "user", "late input"); + afterStaging(() => { + input.messages.unshift(added); + }); + const committed = mock(() => undefined); + const result = await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }); + expect(result).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + expect(committed).toHaveBeenCalledTimes(1); + expect(added.metadata?.historySequence).toBeUndefined(); + const saved = await rows(); + expect(saved.some((row) => row.id === added.id)).toBe(false); + for (const message of original) { + expect(message.metadata?.historySequence).toBe( + saved.find((row) => row.id === message.id)?.metadata?.historySequence + ); + } + expect(await stop.retireReplacement({ nonce: expected.nonce! })).toBe("applied"); + } + ); + + it.each([ + { kind: "single", rejected: false }, + { kind: "single", rejected: true }, + { kind: "single", rejected: "capsule" }, + { kind: "batch", rejected: false }, + { kind: "batch", rejected: true }, + { kind: "batch", rejected: "capsule" }, + ] as const)( + "$kind ordinary acceptance preserves scoped Stop bytes without a witness (rejected=$rejected)", + async ({ kind, rejected }) => { + await stop.cancel(); + const expected = await capture(); + await stop.narrow(expected.nonce!, { + id: "old-summary", + sequence: 0, + pendingFollowUp: { text: "old request" }, + }); + const stopPath = history.getCompactionCancellationStorage(workspaceId).path; + const before = await fs.readFile(stopPath); + const input = await operation(kind); + assert(input.kind === "append"); + input.preserveCancellation = true; + input.messages.at(-1)!.metadata = { + compactionReplacementNonce: expected.nonce!, + ...(rejected ? { contextBudgetRejected: true } : {}), + }; + const originalTrigger = input.messages.at(-1)!; + if (rejected === "capsule") + input.messages[input.messages.length - 1] = + createContextBudgetRejectedMessage(originalTrigger); + // Mutation after preparation must not turn ordinary input into replacement authority. + afterStaging(() => { + delete input.preserveCancellation; + }); + const committed = mock((receipt: { witness: unknown }) => { + expect(receipt.witness).toBeNull(); + expect(nodeFs.readFileSync(chatPath, "utf8")).toContain('"id":"accepted"'); + throw new Error("notification failed after commit"); + }); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }) + ).toEqual(Ok({ kind: "accepted", witness: null })); + expect(committed).toHaveBeenCalledTimes(1); + expect(await fs.readFile(stopPath)).toEqual(before); + const persisted = (await rows()).at(-1)!; + expect(persisted.metadata?.compactionReplacementNonce).toBeUndefined(); + expect(persisted.metadata?.contextBudgetRejected === true).toBe(Boolean(rejected)); + if (rejected === "capsule") { + expect(persisted.role).toBe("assistant"); + expect(persisted.parts).toEqual([]); + expect(persisted.metadata?.contextBudgetRejectedMessage?.parts).toEqual( + originalTrigger.parts + ); + } + expect( + await new HistoryService(fixture.config).findCompactionReplacementWitness( + workspaceId, + expected.nonce! + ) + ).toEqual(Ok(null)); + } + ); + + it.each([ + { kind: "single", capsule: false }, + { kind: "single", capsule: true }, + { kind: "resume", capsule: false }, + { kind: "resume", capsule: true }, + ] as const)( + "$kind replacement refuses budget-rejected input without changing Stop or history (capsule=$capsule)", + async ({ kind, capsule }) => { + let rejected = createMuxMessage("rejected", "user", "over budget"); + rejected.metadata = { contextBudgetRejected: true }; + if (capsule) rejected = createContextBudgetRejectedMessage(rejected); + if (kind === "resume") + expect((await history.appendToHistory(workspaceId, rejected)).success).toBe(true); + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input: CompactionReplacementOperation = + kind === "resume" + ? { kind, message: (await rows()).at(-1)! } + : { kind: "append", messages: [rejected] }; + const before = await fs.readFile(chatPath); + const stopPath = history.getCompactionCancellationStorage(workspaceId).path; + const stopBefore = await fs.readFile(stopPath); + const committed = mock(() => undefined); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }) + ).toEqual(Ok({ kind: "skipped" })); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + expect(await fs.readFile(stopPath)).toEqual(stopBefore); + } + ); + + it.each([ + { role: "assistant", rejected: false }, + { role: "system", rejected: true }, + ] as const)( + "ordinary acceptance refuses unrelated $role triggers", + async ({ role, rejected }) => { + await stop.cancel(); + const expected = await capture(); + const message = { ...createMuxMessage("unrelated", "assistant", "payload"), role }; + if (rejected) message.metadata = { contextBudgetRejected: true }; + const before = await fs.readFile(chatPath); + const committed = mock(() => undefined); + expect( + await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "append", messages: [message], preserveCancellation: true }, + { isCurrent: () => true, onCommitted: committed } + ) + ).toEqual(Ok({ kind: "skipped" })); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + expect(await capture()).toEqual(expected); + } + ); + + it.each(["single", "batch"] as const)( + "%s ordinary acceptance issues no receipt on staging failure", + async (kind) => { + await stop.cancel(); + const expected = await capture(); + const stopPath = history.getCompactionCancellationStorage(workspaceId).path; + const before = await fs.readFile(stopPath); + const chatBefore = await fs.readFile(chatPath); + const input = await operation(kind); + assert(input.kind === "append"); + input.preserveCancellation = true; + afterStaging(() => { + throw new Error("ordinary staging failed"); + }); + const committed = mock(() => undefined); + const result = await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }); + expect(result.success).toBe(false); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(chatBefore); + expect(await fs.readFile(stopPath)).toEqual(before); + } + ); + + it.each([ + { change: "nonce", rejected: false }, + { change: "nonce", rejected: true }, + { change: "nonce", rejected: "capsule" }, + { change: "generation", rejected: false }, + { change: "generation", rejected: true }, + { change: "generation", rejected: "capsule" }, + ] as const)( + "ordinary acceptance refuses a held foreign $change change (rejected=$rejected)", + async ({ change, rejected }) => { + if (change === "nonce") await stop.cancel(); + const expected = await capture(); + const before = await fs.readFile(chatPath); + const input = await operation("single"); + assert(input.kind === "append"); + input.preserveCancellation = true; + if (rejected) input.messages.at(-1)!.metadata = { contextBudgetRejected: true }; + if (rejected === "capsule") + input.messages[0] = createContextBudgetRejectedMessage(input.messages[0]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const withLock = workspaceFileLocks.withLock.bind(workspaceFileLocks); + spyOn(workspaceFileLocks, "withLock").mockImplementationOnce(async (key, operation) => { + entered.resolve(); + await release.promise; + return withLock(key, operation); + }); + const committed = mock(() => undefined); + const accepting = history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }); + await entered.promise; + const foreign = new CompactionCancellation( + new HistoryService(fixture.config).getCompactionCancellationStorage(workspaceId) + ); + try { + await foreign.cancel(); + const record = await foreign.read(); + assert(record); + if (change === "generation") await foreign.retire(record.nonce); + } finally { + release.resolve(); + } + expect(await accepting).toEqual(Ok({ kind: "superseded" })); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + const current = await capture(); + expect(current.generation).not.toBe(expected.generation); + if (change === "generation") expect(current.nonce).toBe(expected.nonce); + else expect(current.nonce).not.toBe(expected.nonce); + } + ); + + it.each(["original", "schema", "signed-schema", "dated-signed-schema"] as const)( + "accepts a %s resume target without its persistence envelope", + async (shape) => { + const original = { + ...createMuxMessage("resume-target", "assistant", "interrupted"), + ...(shape === "dated-signed-schema" ? { createdAt: new Date("2026-09-11T00:00:00Z") } : {}), + }; + if (shape === "signed-schema" || shape === "dated-signed-schema") { + original.parts.unshift({ + type: "reasoning", + text: "thinking", + signature: "provider-signature", + }); + original.metadata = { enqueuedAtMs: 123, goalId: "goal-scope" }; + } + expect((await history.appendToHistory(workspaceId, original)).success).toBe(true); + const target = + shape === "original" + ? original + : MuxMessageSchema.parse({ + ...(await rows()).at(-1)!, + ...(original.createdAt ? { createdAt: original.createdAt } : {}), + }); + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + for (const changed of [ + { ...target, parts: [{ type: "text" as const, text: "different content" }] }, + { ...target, metadata: { ...target.metadata, partial: true } }, + { ...target, metadata: { ...target.metadata, goalId: "different-goal" } }, + { + ...target, + parts: [ + { type: "reasoning" as const, text: "thinking", signature: "different-signature" }, + ...target.parts.slice(1), + ], + }, + ]) { + expect( + await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "resume", message: changed }, + noObserver + ) + ).toEqual(Ok({ kind: "skipped" })); + } + expect( + await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "resume", message: target }, + noObserver + ) + ).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + const persisted = (await rows()).at(-1)!; + if (original.createdAt) { + const persistedRaw = JSON.parse( + (await fs.readFile(chatPath, "utf8")).trimEnd().split("\n").at(-1)! + ) as { createdAt?: string }; + expect(persistedRaw.createdAt).toBe(original.createdAt.toISOString()); + } + expect(persisted.parts).toEqual(original.parts); + expect(persisted.metadata).toMatchObject(original.metadata!); + const fresh = new HistoryService(fixture.config); + expect(await fresh.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + expect(await stop.retireReplacement({ nonce: expected.nonce! })).toBe("applied"); + } + ); + + it.each([ + { kind: "single", value: "bigint" }, + { kind: "single", value: "circular" }, + { kind: "resume", value: "bigint" }, + { kind: "resume", value: "circular" }, + ] as const)( + "returns an Err for $kind $value serialization without changing history", + async ({ kind, value }) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation(kind); + const message = input.kind === "append" ? input.messages[0] : input.message; + const circular: { self?: unknown } = {}; + circular.self = circular; + message.parts.push({ + type: "dynamic-tool", + toolName: "test", + toolCallId: "call", + state: "output-available", + input: {}, + output: value === "bigint" ? 1n : circular, + }); + const before = await fs.readFile(chatPath); + const stopPath = history.getCompactionCancellationStorage(workspaceId).path; + const stopBefore = await fs.readFile(stopPath); + const onCommitted = mock(() => undefined); + const result = await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted, + }); + expect(result.success).toBe(false); + assert(!result.success); + expect(result.error).toContain("Failed to accept compaction replacement:"); + expect(onCommitted).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + expect(await fs.readFile(stopPath)).toEqual(stopBefore); + } + ); + + it.each([false, true])( + "recognizes an unterminated archive replay after repair=%s", + async (repair) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + const active = await fs.readFile(chatPath, "utf8"); + const committed = active.trimEnd().split("\n").at(-1)!; + const archivePath = path.join(path.dirname(chatPath), "chat-archive.jsonl"); + // Archive append stopped after the complete JSON object, before its final delimiter. + await fs.writeFile(archivePath, committed); + if (repair) { + expect( + ( + await history.appendToHistory( + workspaceId, + createMuxMessage("boundary", "assistant", "summary", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ) + ).success + ).toBe(true); + const archived = (await fs.readFile(archivePath, "utf8")).trimEnd().split("\n"); + expect(archived.filter((line) => line === committed)).toHaveLength(2); + } + const fresh = new HistoryService(fixture.config); + expect(await fresh.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + const freshStop = new CompactionCancellation( + fresh.getCompactionCancellationStorage(workspaceId) + ); + expect((await freshStop.read())?.nonce).toBe(expected.nonce!); + expect(await freshStop.retireReplacement({ nonce: expected.nonce! })).toBe("applied"); + expect(await freshStop.read()).toBeNull(); + } + ); + + it.each([false, true])( + "resume rewrites history only when stamping a retained Stop (Stop=%s)", + async (canceled) => { + if (canceled) await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation("resume"); + const before = await fs.readFile(chatPath); + const original = await fs.stat(chatPath); + const committed = mock(() => { + const persisted = JSON.parse(nodeFs.readFileSync(chatPath, "utf8").trim()) as MuxMessage; + expect(persisted.metadata?.compactionReplacementNonce ?? null).toBe(expected.nonce); + // Notification failure cannot revoke acceptance of an existing durable row either. + throw new Error("observer failed"); + }); + const accepted = await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }); + expect(accepted).toEqual( + Ok({ kind: "accepted", witness: canceled ? { nonce: expected.nonce! } : null }) + ); + expect(committed).toHaveBeenCalledTimes(1); + expect((await fs.stat(chatPath)).ino === original.ino).toBe(!canceled); + if (!canceled) expect(await fs.readFile(chatPath)).toEqual(before); + expect(await capture()).toEqual(expected); + } + ); + + it.each([false, true])( + "a held absent-nonce resume refuses a foreign Stop even after retirement (retire=%s)", + async (retire) => { + const expected = await capture(); + const input = await operation("resume"); + const before = await fs.readFile(chatPath); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const withLock = workspaceFileLocks.withLock.bind(workspaceFileLocks); + spyOn(workspaceFileLocks, "withLock").mockImplementationOnce(async (key, operation) => { + entered.resolve(); + await release.promise; + return withLock(key, operation); + }); + const committed = mock(() => undefined); + const accepting = history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }); + await entered.promise; + const foreign = new CompactionCancellation( + new HistoryService(fixture.config).getCompactionCancellationStorage(workspaceId) + ); + try { + await foreign.cancel(); + const stopped = await foreign.read(); + assert(stopped); + if (retire) await foreign.retire(stopped.nonce); + } finally { + release.resolve(); + } + expect(await accepting).toEqual(Ok({ kind: "superseded" })); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + const current = await capture(); + expect(current.generation).not.toBe(expected.generation); + if (retire) expect(current.nonce).toBeNull(); + else expect(current.nonce).not.toBeNull(); + } + ); + + it.each(["admission", "lease"] as const)( + "absent-nonce resume rechecks %s after its exact-target read", + async (lost) => { + const expected = await capture(); + const input = await operation("resume"); + const before = await fs.readFile(chatPath); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const readFile = fs.readFile; + let armed = true; + spyOn(fs, "readFile").mockImplementation( + new Proxy(readFile, { + async apply(target, thisArg, args: Parameters) { + const bytes = await Reflect.apply(target, thisArg, args); + if (armed && args[0] === chatPath) { + armed = false; + entered.resolve(); + await release.promise; + } + return bytes; + }, + }) + ); + let current = true; + const committed = mock(() => undefined); + const accepting = history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => current, + onCommitted: committed, + }); + const lockPath = historyWriteLockPath(fixture.config.rootDir, workspaceId); + await entered.promise; + try { + if (lost === "admission") current = false; + else await fs.writeFile(lockPath, "foreign-owner"); + } finally { + release.resolve(); + } + const result = await accepting; + if (lost === "admission") expect(result).toEqual(Ok({ kind: "superseded" })); + else { + expect(result.success).toBe(false); + await fs.rm(lockPath); + } + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + } + ); + + it("absent-nonce resume still requires an exact unique eligible target", async () => { + const expected = await capture(); + const target = (await rows())[0]; + const original = await fs.readFile(chatPath); + const committed = mock(() => undefined); + for (const message of [ + { ...target, id: "missing" }, + { ...target, parts: [{ type: "text", text: "changed" }] }, + { ...target, metadata: { ...target.metadata, historySequence: undefined } }, + ] satisfies MuxMessage[]) { + expect( + await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "resume", message }, + { isCurrent: () => true, onCommitted: committed } + ) + ).toEqual(Ok({ kind: "skipped" })); + } + for (const extra of [ + { ...target, id: "same-sequence" }, + { ...target, metadata: { ...target.metadata, historySequence: 10 } }, + ]) { + await fs.writeFile( + chatPath, + Buffer.concat([original, Buffer.from(JSON.stringify(extra) + "\n")]) + ); + const before = await fs.readFile(chatPath); + expect( + await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "resume", message: target }, + { isCurrent: () => true, onCommitted: committed } + ) + ).toEqual(Ok({ kind: "skipped" })); + expect(await fs.readFile(chatPath)).toEqual(before); + } + expect(committed).not.toHaveBeenCalled(); + }); + + it.each([false, true])( + "single acceptance preserves append-only history (Stop=%s)", + async (canceled) => { + if (canceled) await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const before = await fs.readFile(chatPath); + const original = await fs.stat(chatPath); + const accepted = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + expect(accepted).toEqual( + Ok({ kind: "accepted", witness: canceled ? { nonce: expected.nonce! } : null }) + ); + expect((await fs.stat(chatPath)).ino).toBe(original.ino); + expect((await fs.readFile(chatPath)).subarray(0, before.length)).toEqual(before); + } + ); + + it("partial single-row writes issue no receipt and the next append repairs the torn tail", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const before = await fs.readFile(chatPath); + const committed = mock(() => undefined); + let writes = 0; + const write = nodeFs.writeSync; + spyOn(nodeFs, "writeSync").mockImplementation( + new Proxy(write, { + apply(target, receiver, args: unknown[]) { + if (writes++ > 0) throw new Error("partial append failed"); + args[3] = 11; + return Reflect.apply(target, receiver, args) as ReturnType; + }, + }) + ); + const failed = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + { + isCurrent: () => true, + onCommitted: committed, + } + ); + expect(failed.success).toBe(false); + expect(committed).not.toHaveBeenCalled(); + const torn = await fs.readFile(chatPath); + expect(torn.length).toBe(before.length + 11); + expect(torn.subarray(0, before.length)).toEqual(before); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok(null) + ); + expect((await stop.read())?.retainUntilReplacement).toBe(true); + mock.restore(); + expect( + await history.acceptCompactionReplacement( + workspaceId, + await capture(), + { + kind: "append", + messages: [createMuxMessage("retry", "user", "new input")], + }, + noObserver + ) + ).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + expect((await fs.readFile(chatPath)).subarray(0, torn.length)).toEqual(torn); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + }); + + it("a missing final delimiter agrees with restart witnessing and cannot duplicate on retry", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation("single"); + const committed = mock(() => undefined); + let writes = 0; + const write = nodeFs.writeSync; + spyOn(nodeFs, "writeSync").mockImplementation( + new Proxy(write, { + apply(target, receiver, args: unknown[]) { + if (writes++ > 0) throw new Error("delimiter write failed"); + assert(typeof args[3] === "number"); + args[3]--; + return Reflect.apply(target, receiver, args) as ReturnType; + }, + }) + ); + const accepted = await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }); + expect(accepted).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + expect(committed).toHaveBeenCalledTimes(1); + mock.restore(); + const bytes = await fs.readFile(chatPath); + expect(bytes.at(-1)).not.toBe(10); + const restarted = new HistoryService(fixture.config); + expect(await restarted.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, noObserver) + ).toEqual(Ok({ kind: "skipped" })); + expect(await fs.readFile(chatPath)).toEqual(bytes); + expect((await rows()).map((row) => row.id)).toEqual(["prior", "accepted"]); + }); + + it.skipIf(process.platform === "win32").each([false, true])( + "new chat acceptance requires directory durability (flush fails=%s)", + async (fails) => { + await stop.cancel({ retainUntilReplacement: true }); + await fs.rm(chatPath); + const expected = await capture(); + let directorySynced = false; + let syncedBeforeReceipt = false; + const sync = nodeFs.fsyncSync; + spyOn(nodeFs, "fsyncSync").mockImplementation((fd) => { + if (nodeFs.fstatSync(fd).isDirectory()) { + if (fails) throw new Error("new chat directory unavailable"); + directorySynced = true; + } + return sync(fd); + }); + const committed = mock(() => { + syncedBeforeReceipt = directorySynced; + return undefined; + }); + const result = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + { + isCurrent: () => true, + onCommitted: committed, + } + ); + expect(result.success).toBe(!fails); + expect(committed).toHaveBeenCalledTimes(fails ? 0 : 1); + if (!fails) expect(syncedBeforeReceipt).toBe(true); + expect(await stop.read()).toMatchObject({ + nonce: expected.nonce, + retainUntilReplacement: true, + }); + } + ); + + it("flushes the exact open append descriptor before publishing acceptance", async () => { + const open = fs.open; + let appendFd: number | undefined; + let flushed = false; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && args[1] === "a") appendFd = handle.fd; + return handle; + }); + const sync = nodeFs.fsyncSync; + spyOn(nodeFs, "fsyncSync").mockImplementation((fd) => { + expect(fd).toBe(appendFd!); + sync(fd); + flushed = true; + }); + const accepted = await history.acceptCompactionReplacement( + workspaceId, + await capture(), + await operation("single"), + { + isCurrent: () => true, + onCommitted: () => { + expect(flushed).toBe(true); + }, + } + ); + expect(accepted).toEqual(Ok({ kind: "accepted", witness: null })); + }); + + it.each([false, true])( + "failed append flush retains Stop through restart verification (missing LF=%s)", + async (missingLf) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation("single"); + const committed = mock(() => undefined); + if (missingLf) { + let writes = 0; + const write = nodeFs.writeSync; + spyOn(nodeFs, "writeSync").mockImplementation( + new Proxy(write, { + apply(target, receiver, args: unknown[]) { + if (writes++ > 0) throw new Error("delimiter write failed"); + assert(typeof args[3] === "number"); + args[3]--; + return Reflect.apply(target, receiver, args) as ReturnType; + }, + }) + ); + } + spyOn(nodeFs, "fsyncSync").mockImplementation(() => { + throw new Error("History flush unavailable"); + }); + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath) + spyOn(handle, "sync").mockRejectedValue(new Error("History flush unavailable")); + return handle; + }); + const failed = await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }); + expect(failed.success).toBe(false); + expect(committed).not.toHaveBeenCalled(); + const appended = await fs.readFile(chatPath); + expect((await rows()).filter((row) => row.id === "accepted")).toHaveLength(1); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, noObserver) + ).toEqual(Ok({ kind: "skipped" })); + expect(await fs.readFile(chatPath)).toEqual(appended); + const restarted = new HistoryService(fixture.config); + expect( + (await restarted.findCompactionReplacementWitness(workspaceId, expected.nonce!)).success + ).toBe(false); + const freshStop = new CompactionCancellation( + restarted.getCompactionCancellationStorage(workspaceId) + ); + await freshStop.read(); + expect( + await freshStop + .retireReplacement({ nonce: expected.nonce! }) + .catch((error: unknown) => error) + ).toBeInstanceOf(Error); + expect( + ( + await new CompactionCancellation( + restarted.getCompactionCancellationStorage(workspaceId) + ).read() + )?.nonce + ).toBe(expected.nonce!); + mock.restore(); + const witness = await restarted.findCompactionReplacementWitness( + workspaceId, + expected.nonce! + ); + expect(witness).toEqual(Ok({ nonce: expected.nonce! })); + expect(await freshStop.retry()).toBe("applied"); + expect( + await new CompactionCancellation( + restarted.getCompactionCancellationStorage(workspaceId) + ).read() + ).toBeNull(); + expect(await fs.readFile(chatPath)).toEqual(appended); + } + ); + + it.skipIf(process.platform === "win32").each(["batch", "resume"] as const)( + "%s flushes the renamed history directory before issuing acceptance", + async (kind) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation(kind); + const directoryFds = new Set(); + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === path.dirname(chatPath)) directoryFds.add(handle.fd); + return handle; + }); + let directoryFlushed = false; + const sync = nodeFs.fsyncSync; + spyOn(nodeFs, "fsyncSync").mockImplementation((fd) => { + sync(fd); + if (directoryFds.has(fd)) { + expect(nodeFs.readFileSync(chatPath, "utf8")).toContain(expected.nonce!); + directoryFlushed = true; + } + }); + let flushedAtCommit = false; + const committed = mock(() => { + flushedAtCommit = directoryFlushed; + return undefined; + }); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }) + ).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + expect(committed).toHaveBeenCalledTimes(1); + expect(flushedAtCommit).toBe(true); + } + ); + + it.skipIf(process.platform === "win32").each(["batch", "resume"] as const)( + "%s directory flush failure retains Stop through fresh-reader recovery", + async (kind) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation(kind); + const committed = mock(() => undefined); + let renamed = false; + const rename = nodeFs.renameSync; + spyOn(nodeFs, "renameSync").mockImplementation((from, to) => { + rename(from, to); + if (to === chatPath) renamed = true; + }); + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === path.dirname(chatPath)) { + const sync = handle.sync.bind(handle); + spyOn(handle, "sync").mockImplementation(async () => { + if (renamed) throw new Error("Directory flush unavailable"); + await sync(); + }); + } + return handle; + }); + const sync = nodeFs.fsyncSync; + spyOn(nodeFs, "fsyncSync").mockImplementation((fd) => { + if (renamed && nodeFs.fstatSync(fd).isDirectory()) + throw new Error("Directory flush unavailable"); + sync(fd); + }); + expect( + ( + await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }) + ).success + ).toBe(false); + expect(committed).not.toHaveBeenCalled(); + const published = await fs.readFile(chatPath); + expect(published.toString()).toContain(expected.nonce!); + expect( + (await history.acceptCompactionReplacement(workspaceId, expected, input, noObserver)) + .success + ).toBe(false); + expect(await fs.readFile(chatPath)).toEqual(published); + const restarted = new HistoryService(fixture.config); + expect( + (await restarted.findCompactionReplacementWitness(workspaceId, expected.nonce!)).success + ).toBe(false); + const freshStop = new CompactionCancellation( + restarted.getCompactionCancellationStorage(workspaceId) + ); + expect((await freshStop.read())?.nonce).toBe(expected.nonce!); + expect( + await freshStop + .retireReplacement({ nonce: expected.nonce! }) + .catch((error: unknown) => error) + ).toBeInstanceOf(Error); + expect( + ( + await new CompactionCancellation( + restarted.getCompactionCancellationStorage(workspaceId) + ).read() + )?.nonce + ).toBe(expected.nonce!); + mock.restore(); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, noObserver) + ).toEqual(Ok({ kind: "skipped" })); + expect( + await restarted.findCompactionReplacementWitness(workspaceId, expected.nonce!) + ).toEqual(Ok({ nonce: expected.nonce! })); + expect(await freshStop.retry()).toBe("applied"); + expect( + await new CompactionCancellation( + restarted.getCompactionCancellationStorage(workspaceId) + ).read() + ).toBeNull(); + expect(await fs.readFile(chatPath)).toEqual(published); + } + ); + + it.each([ + ["lookup", "file"], + ["retire", "file"], + ...(process.platform === "win32" + ? [] + : [ + ["lookup", "directory"], + ["retire", "directory"], + ]), + ])("revalidates %s witness evidence after the %s durability flush", async (use, artifact) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + const open = fs.open; + let changed = false; + let fileFlushed = false; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath || args[0] === path.dirname(chatPath)) { + const sync = handle.sync.bind(handle); + spyOn(handle, "sync").mockImplementation(async () => { + await sync(); + if (args[0] === chatPath) fileFlushed = true; + if ( + !changed && + fileFlushed && + args[0] === (artifact === "file" ? chatPath : path.dirname(chatPath)) + ) { + changed = true; + await fs.appendFile( + chatPath, + JSON.stringify( + createMuxMessage("successor", "user", "later", { historySequence: 2 }) + ) + "\n" + ); + } + }); + } + return handle; + }); + if (use === "lookup") + expect( + (await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).success + ).toBe(false); + else + expect( + await stop.retireReplacement({ nonce: expected.nonce! }).catch((error: unknown) => error) + ).toBeInstanceOf(Error); + expect(changed).toBe(true); + expect( + ( + await new CompactionCancellation( + history.getCompactionCancellationStorage(workspaceId) + ).read() + )?.nonce + ).toBe(expected.nonce!); + }); + + it.each(process.platform === "win32" ? ["file"] : ["file", "directory"])( + "does not issue a lookup witness after lease reclamation during its %s flush", + async (artifact) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + const open = fs.open; + let fileFlushed = false; + let successor: Awaited> | undefined; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath || args[0] === path.dirname(chatPath)) { + const sync = handle.sync.bind(handle); + spyOn(handle, "sync").mockImplementation(async () => { + await sync(); + if (args[0] === chatPath) fileFlushed = true; + if ( + !successor && + fileFlushed && + args[0] === (artifact === "file" ? chatPath : path.dirname(chatPath)) + ) { + const lockPath = historyWriteLockPath(fixture.config.rootDir, workspaceId); + const token = await fs.readFile(lockPath, "utf8"); + await fs.writeFile(lockPath, token.split(":").slice(0, 2).join(":")); + await fs.utimes(lockPath, new Date(0), new Date(0)); + successor = await acquireProcessFileLock({ + lockPath, + timeoutMs: 1000, + label: "witness successor", + }); + } + }); + } + return handle; + }); + try { + expect( + (await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).success + ).toBe(false); + assert(successor); + await successor.assertStillOwned(); + expect( + ( + await new CompactionCancellation( + history.getCompactionCancellationStorage(workspaceId) + ).read() + )?.nonce + ).toBe(expected.nonce!); + } finally { + await successor?.[Symbol.asyncDispose](); + } + } + ); + + it.each(["sync", "close", "certification"] as const)( + "single acceptance survives post-publication %s failure", + async (phase) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + let committed = false; + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && phase === "close" && args[1] === "a") { + const close = handle.close.bind(handle); + spyOn(handle, "close").mockImplementationOnce(async () => { + await close(); + throw new Error("close failed"); + }); + } + if (args[0] === chatPath && phase === "sync" && committed) + spyOn(handle, "sync").mockRejectedValueOnce(new Error("sync failed")); + return handle; + }); + if (phase === "certification") { + const stamps = HistoryAppendProvenance.prototype.stamps; // eslint-disable-line @typescript-eslint/unbound-method -- called with the original receiver + spyOn(HistoryAppendProvenance.prototype, "stamps").mockImplementation(function ( + this: HistoryAppendProvenance + ) { + if (committed) return Promise.reject(new Error("certification failed")); + return stamps.call(this); + }); + } + const result = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + { + isCurrent: () => !committed, + onCommitted: () => { + committed = true; + }, + } + ); + expect(committed).toBe(true); + expect(result).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + mock.restore(); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + } + ); + + it("close failure cannot turn superseded append preparation into acceptance", async () => { + const expected = await capture(); + const before = await fs.readFile(chatPath); + let current = true; + const committed = mock(() => undefined); + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && args[1] === "a") { + current = false; + const close = handle.close.bind(handle); + spyOn(handle, "close").mockImplementationOnce(async () => { + await close(); + throw new Error("close failed"); + }); + } + return handle; + }); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, await operation("single"), { + isCurrent: () => current, + onCommitted: committed, + }) + ).toEqual(Ok({ kind: "superseded" })); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + }); + + function beforeFileRead(filePath: string, action: (args: unknown[]) => void | Promise) { + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === filePath) { + const read = handle.read.bind(handle); + spyOn(handle, "read").mockImplementation( + new Proxy(read, { + async apply(target, thisArg, args: unknown[]) { + await action(args); + return Reflect.apply(target, thisArg, args) as ReturnType; + }, + }) + ); + } + return handle; + }); + } + + it.each(["none", "terminated", "unterminated", "repeated"] as const)( + "accepts a stamped Resume after failed retirement and %s archive replay without rewriting its row", + async (replay) => { + const storage = history.getCompactionCancellationStorage(workspaceId); + stop = new CompactionCancellation(storage); + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const first = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("resume"), + noObserver + ); + assert(first.success && first.data.kind === "accepted" && first.data.witness); + spyOn(storage, "mutate").mockRejectedValueOnce(new Error("retirement unavailable")); + const retirement = await stop + .retireReplacement(first.data.witness) + .catch((error: unknown) => error); + expect(retirement).toEqual(new Error("retirement unavailable")); + mock.restore(); + const archivePath = path.join(path.dirname(chatPath), "chat-archive.jsonl"); + const stampedRow = (await fs.readFile(chatPath, "utf8")).trimEnd().split("\n").at(-1)!; + if (replay !== "none") + await fs.writeFile( + archivePath, + replay === "terminated" + ? stampedRow + "\n" + : replay === "repeated" + ? stampedRow + "\n" + stampedRow + : stampedRow + ); + const archiveBefore = replay === "none" ? null : await fs.readFile(archivePath); + + history = new HistoryService(fixture.config); + const freshStop = new CompactionCancellation( + history.getCompactionCancellationStorage(workspaceId) + ); + await freshStop.read(); + expect(await capture()).toEqual(expected); + const before = await fs.readFile(chatPath); + const committed = mock(() => undefined); + const retried = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("resume"), + { isCurrent: () => true, onCommitted: committed } + ); + expect(retried).toEqual(first); + expect(committed).toHaveBeenCalledTimes(1); + expect(await fs.readFile(chatPath)).toEqual(before); + assert(retried.success && retried.data.kind === "accepted" && retried.data.witness); + expect(await freshStop.retireReplacement(retried.data.witness)).toBe("applied"); + expect(await freshStop.read()).toBeNull(); + if (archiveBefore) expect(await fs.readFile(archivePath)).toEqual(archiveBefore); + } + ); + + it.each(["flush failure", "supersession"] as const)( + "refuses a stamped Resume after witness %s", + async (failure) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const first = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("resume"), + noObserver + ); + assert(first.success && first.data.kind === "accepted"); + const input = await operation("resume"); + const before = await fs.readFile(chatPath); + let current = true; + const open = fs.open; + spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const handle = await open(...args); + if (args[0] === chatPath && args[1] === "r+") { + const sync = handle.sync.bind(handle); + spyOn(handle, "sync").mockImplementation(async () => { + if (failure === "flush failure") throw new Error("witness flush unavailable"); + await sync(); + current = false; + }); + } + return handle; + }); + const committed = mock(() => undefined); + const result = await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => current, + onCommitted: committed, + }); + if (failure === "flush failure") { + expect(result.success).toBe(false); + assert(!result.success); + expect(result.error).toContain("witness flush unavailable"); + } else expect(result).toEqual(Ok({ kind: "superseded" })); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + } + ); + + it.each([ + "changed", + "duplicate", + "archive", + "archive nonce", + "archive id", + "archive sequence", + "archive formatting", + "foreign", + "stale", + ] as const)("refuses a stamped Resume with %s evidence", async (conflict) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const first = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("resume"), + noObserver + ); + assert(first.success && first.data.kind === "accepted"); + const input = await operation("resume"); + assert(input.kind === "resume"); + if (conflict === "changed") input.message.parts = [{ type: "text", text: "different" }]; + const persisted = (await fs.readFile(chatPath, "utf8")).trimEnd().split("\n").at(-1)!; + if (conflict === "duplicate") await fs.appendFile(chatPath, persisted + "\n"); + if (conflict.startsWith("archive ")) { + const archived = JSON.parse(persisted) as MuxMessage; + if (conflict === "archive nonce") archived.metadata!.compactionReplacementNonce = "foreign"; + if (conflict === "archive id") archived.id = "foreign-id"; + if (conflict === "archive sequence") archived.metadata!.historySequence! += 1; + await fs.writeFile( + path.join(path.dirname(chatPath), "chat-archive.jsonl"), + (conflict === "archive formatting" ? " " + persisted : JSON.stringify(archived)) + "\n" + ); + } + if (conflict === "archive") + await fs.writeFile( + path.join(path.dirname(chatPath), "chat-archive.jsonl"), + JSON.stringify({ ...input.message, parts: [{ type: "text", text: "collision" }] }) + "\n" + ); + if (conflict === "foreign") await stop.cancel({ retainUntilReplacement: true }); + const before = await fs.readFile(chatPath); + const committed = mock(() => undefined); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => conflict !== "stale", + onCommitted: committed, + }) + ).toEqual( + Ok({ kind: conflict === "foreign" || conflict === "stale" ? "superseded" : "skipped" }) + ); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + }); + + it("refuses an unstamped Resume with an exact archive replay", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation("resume"); + const before = await fs.readFile(chatPath); + await fs.writeFile(path.join(path.dirname(chatPath), "chat-archive.jsonl"), before); + const committed = mock(() => undefined); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: committed, + }) + ).toEqual(Ok({ kind: "skipped" })); + expect(committed).not.toHaveBeenCalled(); + expect(await fs.readFile(chatPath)).toEqual(before); + expect(await capture()).toEqual(expected); + }); + + it("keeps an accepted receipt verifiable after repeated budget rejection and restart", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation("single"); + const accepted = await history.acceptCompactionReplacement( + workspaceId, + expected, + input, + noObserver + ); + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + const unlink = nodeFs.rmSync; + const cancellationPath = history.getCompactionCancellationStorage(workspaceId).path; + spyOn(nodeFs, "rmSync").mockImplementation((file, options) => { + if (file === cancellationPath) throw new Error("injected unlink failure"); + return unlink(file, options); + }); + const retirement = await stop + .retireReplacement(accepted.data.witness) + .catch((error: unknown) => error); + expect(retirement).toBeInstanceOf(Error); + mock.restore(); + for (let attempt = 0; attempt < 2; attempt++) { + const rejected = await history.rejectContextBudgetRequest( + workspaceId, + (await rows()).at(-1)! + ); + assert(rejected.success); + const fresh = new HistoryService(fixture.config); + expect(await fresh.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + } + const freshStop = new CompactionCancellation( + new HistoryService(fixture.config).getCompactionCancellationStorage(workspaceId) + ); + await freshStop.read(); + expect(await freshStop.retireReplacement(accepted.data.witness)).toBe("applied"); + }); + + it.each([false, true])( + "exhausts short reads while distinguishing ambiguous same-nonce witnesses (%s)", + async (sameNonce) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const message = createMuxMessage("ambiguous", "user", "bad occurrence", { + historySequence: 10, + compactionReplacementNonce: sameNonce ? expected.nonce! : "unrelated-nonce", + }); + const valid = createMuxMessage("later", "user", "🙂 valid occurrence", { + historySequence: 11, + compactionReplacementNonce: expected.nonce!, + }); + await fs.writeFile( + chatPath, + [message, message, valid].map((row) => JSON.stringify(row)).join("\n") + ); + let reads = 0; + beforeFileRead(chatPath, (args) => { + if (typeof args[2] === "number") args[2] = Math.min(args[2], 17); + reads++; + }); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok(sameNonce ? null : { nonce: expected.nonce! }) + ); + expect(reads).toBeGreaterThan(10); + const collected: string[] = []; + expect( + ( + await history.iterateFullHistory(workspaceId, "forward", (rows) => { + collected.push(...rows.map((row) => row.id)); + }) + ).success + ).toBe(true); + expect(collected).toEqual([message.id, message.id, valid.id]); + } + ); + + it("reports a forward visitor failure at an unterminated tail instead of success", async () => { + await fs.writeFile(chatPath, JSON.stringify(createMuxMessage("tail", "user", "input"))); + const result = await history.iterateFullHistory(workspaceId, "forward", () => + Promise.reject(new Error("visitor I/O failed")) + ); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("visitor I/O failed"); + }); + + it("refuses truncated streaming evidence instead of treating unread bytes as absence", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + let truncated = false; + beforeFileRead(chatPath, async () => { + if (!truncated) { + truncated = true; + await fs.truncate(chatPath, 0); + } + }); + expect( + (await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).success + ).toBe(false); + }); + + it.each(["accept", "lookup", "retire"] as const)( + "%s scans a multi-chunk archive without whole-file allocation", + async (use) => { + const archivePath = path.join(path.dirname(chatPath), "chat-archive.jsonl"); + await fs.writeFile( + archivePath, + Array.from( + { length: 128 }, + (_, index) => + JSON.stringify( + createMuxMessage(`old-${index}`, "assistant", "x".repeat(8192), { + historySequence: index + 100, + }) + ) + "\n" + ).join("") + ); + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + if (use !== "accept") + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + const read = fs.readFile; + spyOn(fs, "readFile").mockImplementation( + new Proxy(read, { + apply(target, thisArg, args: Parameters) { + if (args[0] === archivePath) throw new Error("whole-file archive allocation refused"); + return Reflect.apply(target, thisArg, args); + }, + }) + ); + if (use === "accept") + expect( + ( + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ) + ).success + ).toBe(true); + else if (use === "lookup") + expect( + await history.findCompactionReplacementWitness(workspaceId, expected.nonce!) + ).toEqual(Ok({ nonce: expected.nonce! })); + else expect(await stop.retireReplacement({ nonce: expected.nonce! })).toBe("applied"); + } + ); + + it.each(["accept", "lookup", "retire"] as const)( + "%s streams giant archive evidence without whole-row allocation", + async (use) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const archivePath = path.join(path.dirname(chatPath), "chat-archive.jsonl"); + const archived = createMuxMessage( + "giant-archive", + "user", + "x".repeat(3 * SESSION_HISTORY_MAX_LINE_BYTES), + { + historySequence: 10, + ...(use === "accept" ? {} : { compactionReplacementNonce: expected.nonce! }), + } + ); + await fs.writeFile(archivePath, JSON.stringify(archived)); + const input = await operation("single"); + // Ordinary append's sequence allocation remains outside this witness-reader change. + // Keep the acceptance guard active throughout preparation, before its real write lock. + let lockDepth = 0; + const withLock = workspaceFileLocks.withLock.bind(workspaceFileLocks); + spyOn(workspaceFileLocks, "withLock").mockImplementation((key, body) => + withLock(key, async () => { + lockDepth++; + try { + return await body(); + } finally { + lockDepth--; + } + }) + ); + const mustStream = () => use !== "accept" || lockDepth === 0; + const parse = JSON.parse; + const concat = Buffer.concat.bind(Buffer); + spyOn(JSON, "parse").mockImplementation((...args: Parameters) => { + if (mustStream() && Buffer.byteLength(args[0]) > SESSION_HISTORY_MAX_LINE_BYTES) + throw new Error("whole-row parse refused"); + return parse(...args) as unknown; + }); + spyOn(Buffer, "concat").mockImplementation((...args: Parameters) => { + if ( + mustStream() && + args[0].reduce((sum, chunk) => sum + chunk.byteLength, 0) > SESSION_HISTORY_MAX_LINE_BYTES + ) + throw new Error("whole-row concatenation refused"); + return concat(...args); + }); + if (use === "accept") + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, noObserver) + ).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + else if (use === "lookup") + expect( + await history.findCompactionReplacementWitness(workspaceId, expected.nonce!) + ).toEqual(Ok({ nonce: expected.nonce! })); + else expect(await stop.retireReplacement({ nonce: expected.nonce! })).toBe("applied"); + } + ); + + it.each([ + "id", + "sequence", + "invalid-utf8", + "protected", + "invalid-parts", + "malformed", + "replay", + ] as const)("retains giant identity collision accounting for %s rows", async (conflict) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const id = "identity-" + "x".repeat(SESSION_HISTORY_MAX_LINE_BYTES); + const candidate = createMuxMessage(id, "user", "candidate", { + historySequence: 10, + compactionReplacementNonce: expected.nonce!, + }); + await fs.writeFile(chatPath, JSON.stringify(candidate)); + const other = createMuxMessage( + conflict === "sequence" ? "other" : id, + "user", + conflict === "sequence" ? "x".repeat(SESSION_HISTORY_MAX_LINE_BYTES) : "collision", + { + historySequence: conflict === "sequence" ? 10 : 11, + ...(conflict === "protected" ? { contextBoundaryKind: "reset" as const } : {}), + } + ); + let raw = Buffer.from(JSON.stringify(other)); + if (conflict === "invalid-parts") + raw = Buffer.from(raw.toString().replace('"type":"text"', '"type":"bad"')); + if (conflict === "malformed") raw = raw.subarray(0, raw.length - 1); + if (conflict === "invalid-utf8") + raw = Buffer.concat([ + raw.subarray(0, -1), + Buffer.from(',"extra":"'), + Buffer.from([255]), + Buffer.from('"}'), + ]); + if (conflict === "replay") + raw = Buffer.from(JSON.stringify(candidate) + "\n" + JSON.stringify(candidate)); + await fs.writeFile(path.join(path.dirname(chatPath), "chat-archive.jsonl"), raw); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok( + conflict === "invalid-parts" || conflict === "malformed" || conflict === "replay" + ? { nonce: expected.nonce! } + : null + ) + ); + }); + + it.each([ + "commit", + "foreign-before", + "foreign-after", + "flush-failure", + "observer-failure", + "acceptance-observer-failure", + ] as const)( + "owned reset successor is authorized only by its history receipt (%s)", + async (phase) => { + const expected = await capture(); + const foreign = new CompactionCancellation( + new HistoryService(fixture.config).getCompactionCancellationStorage(workspaceId) + ); + if (phase === "foreign-before") await foreign.cancel({ retainUntilReplacement: true }); + if (phase === "flush-failure") { + const sync = nodeFs.fsyncSync; + spyOn(nodeFs, "fsyncSync").mockImplementation((fd) => { + if (nodeFs.readFileSync(chatPath, "utf8").includes('"id":"trigger"')) + throw new Error("history flush failed"); + sync(fd); + }); + } + let acceptedReceipt = false; + let receiptPrecededReset = false; + let predecessor: Awaited> | undefined; + let successor: Awaited> | undefined; + const result = await history.acceptCompactionReplacement( + workspaceId, + expected, + { + kind: "append", + preserveCancellation: true, + messages: [ + createMuxMessage("reset", "assistant", "", { contextBoundaryKind: "reset" }), + createMuxMessage("trigger", "user", "New context"), + ], + }, + { + isCurrent: () => true, + onCommitted: () => { + acceptedReceipt = true; + if (phase === "acceptance-observer-failure") + throw new Error("accepted observer failed"); + }, + onContextResetCommitted: (before, after) => { + receiptPrecededReset = acceptedReceipt; + predecessor = before; + successor = after; + if (phase === "observer-failure") throw new Error("observer failed"); + }, + } + ); + if (phase === "foreign-before" || phase === "flush-failure") { + expect(successor).toBeUndefined(); + expect(acceptedReceipt).toBe(false); + if (phase === "flush-failure") { + expect(result.success).toBe(false); + expect(await fs.readFile(chatPath, "utf8")).toContain('"id":"trigger"'); + } else expect(result).toEqual(Ok({ kind: "superseded" })); + return; + } + expect(result).toEqual(Ok({ kind: "accepted", witness: null })); + expect(receiptPrecededReset).toBe(true); + expect(predecessor).toEqual(expected); + assert(successor); + expect(successor.nonce).toBe(expected.nonce); + expect(successor.generation).not.toBe(expected.generation); + expect(await capture()).toEqual(successor); + if (phase === "foreign-after") await foreign.cancel({ retainUntilReplacement: true }); + const next = await history.acceptCompactionReplacement( + workspaceId, + successor, + { + kind: "append", + preserveCancellation: true, + messages: [createMuxMessage("next", "user", "Queued input")], + }, + noObserver + ); + expect(next).toEqual( + Ok(phase === "foreign-after" ? { kind: "superseded" } : { kind: "accepted", witness: null }) + ); + expect((await rows()).some((row) => row.id === "next")).toBe(phase !== "foreign-after"); + } + ); + + it.each(["staging", "lease check"] as const)( + "a replacement reset losing logical ownership during generation %s leaves its frontier unchanged", + async (phase) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const journal = history.getContinuousCompactionJournal(workspaceId); + const generation = await journal.captureGeneration(); + const before = await fs.readFile(chatPath); + let current = true; + const atomic = atomicWrite.default; + spyOn(atomicWrite, "default").mockImplementation( + new Proxy(atomic, { + async apply(target, _thisArg, args: Parameters) { + const result = await target(...args); + if (String(args[0]).includes(`${CONTINUOUS_COMPACTION_GENERATION_FILE}.continuous-`)) { + if (phase === "staging") current = false; + else { + const read = fs.readFile; + spyOn(fs, "readFile").mockImplementation( + new Proxy(read, { + async apply(target, thisArg, args: Parameters) { + const result = await Reflect.apply(target, thisArg, args); + if (args[0] === historyWriteLockPath(fixture.config.rootDir, workspaceId)) + current = false; + return result; + }, + }) + ); + } + } + return result; + }, + }) + ); + const result = await history.acceptCompactionReplacement( + workspaceId, + expected, + { + kind: "append", + messages: [ + createMuxMessage("reset", "assistant", "", { contextBoundaryKind: "reset" }), + createMuxMessage("trigger", "user", "New context"), + ], + }, + { isCurrent: () => current, onCommitted: () => undefined } + ); + expect(result).toEqual(Ok({ kind: "superseded" })); + expect(await journal.captureGeneration()).toBe(generation); + expect(await fs.readFile(chatPath)).toEqual(before); + } + ); + + it("a reset refused after its generation fence preserves user history and permits fresh replacement", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const nonce = expected.nonce; + assert(nonce); + const journal = history.getContinuousCompactionJournal(workspaceId); + const prepared: ContinuousCompactionJournal = { + version: 1, + publicationGeneration: expected.generation, + boundary: createMuxMessage("summary", "assistant", "Prepared summary", { + compactionBoundary: true, + compactionEpoch: 1, + }), + staticCopies: [], + liveTailCopySpec: { + sourceMessageId: "live", + sourceHistorySequence: 1, + copyId: "copy", + partIndex: 0, + metadataTemplate: { synthetic: true, rlmPreservedTailCopy: true }, + }, + postCompactionAttachments: [], + prefixSourceRows: await rows(), + systemPrefix: [], + cacheEnabled: false, + preparation: { + modelString: "anthropic:test", + providerForMessages: "anthropic", + effectiveThinkingLevel: "off", + effectiveAgentId: "exec", + toolNamesForSentinel: [], + }, + providerFamily: "anthropic", + parentModel: "anthropic:test", + summaryModel: "anthropic:test", + headFingerprint: "head", + sourceFingerprint: "source", + headEnd: { id: "prior", sequence: 0 }, + epoch: 0, + streamMessageId: "live", + streamHistorySequence: 1, + stepNumber: 0, + firstTailToolCallId: "tool", + }; + assert( + await journal.write(prepared, [{ role: "user", content: "Prepared prefix" }], () => true) + ); + assert(await journal.read(), "The old journal must be usable before the reset fence"); + const archivePath = path.join(path.dirname(chatPath), "chat-archive.jsonl"); + await fs.writeFile( + archivePath, + JSON.stringify(createMuxMessage("archived", "user", "Older input", { historySequence: 7 })) + + "\n" + ); + const beforeChat = await fs.readFile(chatPath); + const beforeArchive = await fs.readFile(archivePath); + let current = true; + let staged = false; + afterStaging(() => { + staged = true; + current = false; + }); + const committed = mock(() => undefined); + const resetCommitted = mock(() => undefined); + const replacement = () => ({ + kind: "append" as const, + messages: [ + createMuxMessage("reset", "assistant", "", { contextBoundaryKind: "reset" }), + createMuxMessage("trigger", "user", "New context"), + ], + }); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, replacement(), { + isCurrent: () => current, + onCommitted: committed, + onContextResetCommitted: resetCommitted, + }) + ).toEqual(Ok({ kind: "superseded" })); + expect(resetCommitted).not.toHaveBeenCalled(); + expect(staged).toBe(true); + expect(await journal.captureGeneration()).not.toBe(expected.generation); + expect(await fs.readFile(chatPath)).toEqual(beforeChat); + expect(await fs.readFile(archivePath)).toEqual(beforeArchive); + expect(committed).not.toHaveBeenCalled(); + expect(await history.findCompactionReplacementWitness(workspaceId, nonce)).toEqual(Ok(null)); + expect((await history.getCompactionCancellationStorage(workspaceId).read())?.nonce).toBe(nonce); + expect(stop.blocksRecovery).toBe(false); + expect((await stop.read())?.retainUntilReplacement).toBe(true); + expect(await journal.read()).toBeNull(); + expect(await journal.write(prepared, [], () => true)).toBeNull(); + mock.restore(); + const fresh = await capture(); + expect(fresh.generation).not.toBe(expected.generation); + const accepted = await history.acceptCompactionReplacement( + workspaceId, + fresh, + replacement(), + noObserver + ); + assert(accepted.success && accepted.data.kind === "accepted" && accepted.data.witness); + expect(await stop.retireReplacement(accepted.data.witness)).toBe("applied"); + expect(await stop.read()).toBeNull(); + expect((await rows()).at(-1)?.id).toBe("trigger"); + }); + + for (const kind of ["single", "batch", "resume"] as const) { + it(`${kind} commits the receipt with the row before observers run`, async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation(kind); + let commits = 0; + const accepted = await history.acceptCompactionReplacement(workspaceId, expected, input, { + isCurrent: () => true, + onCommitted: (receipt) => { + commits++; + const committed = nodeFs + .readFileSync(chatPath, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as MuxMessage); + expect(committed.at(-1)?.metadata?.compactionReplacementNonce).toBe(expected.nonce!); + expect(receipt.witness?.nonce).toBe(expected.nonce!); + // Notification cannot mutate the internal receipt or turn publication into failure. + Object.assign(receipt.witness!, { nonce: "changed by observer" }); + throw new Error("observer failed"); + }, + }); + expect(accepted).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + expect(commits).toBe(1); + expect(await history.getCompactionCancellationStorage(workspaceId).read()).not.toBeNull(); + const loaded = await rows(); + expect(loaded).toHaveLength(kind === "batch" ? 3 : kind === "single" ? 2 : 1); + if (input.kind === "append") + expect(input.messages.at(-1)?.metadata?.historySequence).toBe( + loaded.at(-1)?.metadata?.historySequence + ); + const foreign = new HistoryService(fixture.config); + expect(await foreign.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + expect(await stop.retireReplacement({ nonce: expected.nonce! })).toBe("applied"); + }); + + it(`${kind} distinguishes local supersession from staging I/O failure`, async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const before = await fs.readFile(chatPath); + let current = true; + let commits = 0; + afterStaging(() => { + current = false; + }); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, await operation(kind), { + isCurrent: () => current, + onCommitted: () => { + commits++; + }, + }) + ).toEqual(Ok({ kind: "superseded" })); + expect(await fs.readFile(chatPath)).toEqual(before); + mock.restore(); + afterStaging(() => { + throw new Error("disk failed"); + }); + const failed = await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation(kind), + { + isCurrent: () => true, + onCommitted: () => { + commits++; + }, + } + ); + assert(!failed.success); + expect(failed.error).toContain("disk failed"); + expect(commits).toBe(0); + expect(await fs.readFile(chatPath)).toEqual(before); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok(null) + ); + expect(await stop.read()).toMatchObject({ + nonce: expected.nonce, + retainUntilReplacement: true, + }); + }); + } + + it("accepts proven absence, but rejects a Stop/retirement ABA and a foreign newer Stop", async () => { + const absent = await capture(); + expect( + await history.acceptCompactionReplacement( + workspaceId, + absent, + await operation("single"), + noObserver + ) + ).toEqual(Ok({ kind: "accepted", witness: null })); + await stop.cancel(); + const first = await capture(); + await stop.retire(first.nonce!); + expect((await capture()).nonce).toBeNull(); + expect( + await history.acceptCompactionReplacement( + workspaceId, + absent, + await operation("single"), + noObserver + ) + ).toEqual(Ok({ kind: "superseded" })); + await stop.cancel({ retainUntilReplacement: true }); + const stale = await capture(); + const foreign = new CompactionCancellation( + new HistoryService(fixture.config).getCompactionCancellationStorage(workspaceId) + ); + await foreign.cancel(); + expect( + await history.acceptCompactionReplacement( + workspaceId, + stale, + await operation("single"), + noObserver + ) + ).toEqual(Ok({ kind: "superseded" })); + expect(await rows()).toHaveLength(2); + }); + + it("does not reinterpret cancellation or generation read failures as absence", async () => { + const expected = await capture(); + const sidecar = history.getCompactionCancellationStorage(workspaceId).path; + await fs.mkdir(sidecar); + expect((await captureFailure()).success).toBe(false); + expect( + ( + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ) + ).success + ).toBe(false); + await fs.rmdir(sidecar); + await fs.mkdir(path.join(path.dirname(chatPath), CONTINUOUS_COMPACTION_GENERATION_FILE)); + expect((await captureFailure()).success).toBe(false); + expect( + ( + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ) + ).success + ).toBe(false); + expect(await rows()).toHaveLength(1); + function captureFailure() { + return history.captureCompactionReplacement(workspaceId); + } + }); + + it("skips empty acceptance and missing, changed, or ambiguous resume rows without retiring retained Stop", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const target = (await rows())[0]; + for (const input of [ + { kind: "append", messages: [] }, + { kind: "resume", message: { ...target, id: "missing" } }, + { kind: "resume", message: { ...target, parts: [{ type: "text", text: "changed" }] } }, + { + kind: "resume", + message: { ...target, metadata: { ...target.metadata, historySequence: undefined } }, + }, + ] satisfies CompactionReplacementOperation[]) { + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, noObserver) + ).toEqual(Ok({ kind: "skipped" })); + } + const original = await fs.readFile(chatPath); + for (const extra of [ + target, + { ...target, id: "same-sequence" }, + { ...target, metadata: { ...target.metadata, historySequence: 10 } }, + ]) { + await fs.writeFile( + chatPath, + Buffer.concat([original, Buffer.from(JSON.stringify(extra) + "\n")]) + ); + const before = await fs.readFile(chatPath); + expect( + await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "resume", message: target }, + noObserver + ) + ).toEqual(Ok({ kind: "skipped" })); + expect(await fs.readFile(chatPath)).toEqual(before); + } + await history.clearHistory(workspaceId); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok(null) + ); + expect(await stop.retire(expected.nonce!)).toBeUndefined(); + expect(await stop.read()).toMatchObject({ + nonce: expected.nonce, + retainUntilReplacement: true, + }); + }); + + it("keeps accepted outcomes across real lock completion errors and cancellation unlink failure", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation("single"); + const withLock = workspaceFileLocks.withLock.bind(workspaceFileLocks); + spyOn(workspaceFileLocks, "withLock").mockImplementationOnce(async (key, operation) => { + await withLock(key, operation); + throw new Error("outer lock completion failed"); + }); + expect( + await history.acceptCompactionReplacement(workspaceId, expected, input, noObserver) + ).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + spyOn(nodeFs, "rmSync").mockImplementationOnce(() => { + throw new Error("unlink failed"); + }); + expect(stop.retireReplacement({ nonce: expected.nonce! })).rejects.toThrow("unlink failed"); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + await stop.cancel(); + const newer = await capture(); + expect(newer.nonce).not.toBe(expected.nonce); + expect( + await history.getCompactionCancellationStorage(workspaceId).mutate( + { kind: "retire", nonce: expected.nonce!, replacementWitness: { nonce: expected.nonce! } }, + () => true, + () => undefined + ) + ).toBe("superseded"); + expect((await capture()).nonce).toBe(newer.nonce); + }); + + it("verifies archived receipts after same-row finalization and rejects ambiguous or forged witnesses", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const input = await operation("resume"); + await history.acceptCompactionReplacement(workspaceId, expected, input, noObserver); + assert(input.kind === "resume"); + await history.updateHistory(workspaceId, { + ...input.message, + parts: [{ type: "text", text: "finalized" }], + }); + await history.appendToHistory( + workspaceId, + createMuxMessage("boundary", "assistant", "summary", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + expect( + await fs.readFile(path.join(path.dirname(chatPath), "chat-archive.jsonl"), "utf8") + ).toContain(expected.nonce!); + const foreign = new HistoryService(fixture.config); + expect(await foreign.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + const storage = foreign.getCompactionCancellationStorage(workspaceId); + expect( + storage.mutate( + { kind: "retire", nonce: expected.nonce!, replacementWitness: { nonce: "forged" } }, + () => true, + () => undefined + ) + ).rejects.toThrow("not verified"); + await fs.appendFile(chatPath, JSON.stringify({ ...input.message, id: "collision" }) + "\n"); + expect(await foreign.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok(null) + ); + expect(await storage.read()).toMatchObject({ nonce: expected.nonce! }); + }); + + it("waits on the real cross-instance lock and rejects a reclaimed lease before publication", async () => { + const expected = await capture(); + const lockPath = historyWriteLockPath(fixture.config.rootDir, workspaceId); + const held = await acquireProcessFileLock({ + lockPath, + timeoutMs: 1000, + label: "foreign writer", + }); + const writing = history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + await fs.writeFile( + history.getCompactionCancellationStorage(workspaceId).path, + JSON.stringify({ version: 1, nonce: "foreign", scope: { kind: "unresolved" } }) + ); + await held[Symbol.asyncDispose](); + expect(await writing).toEqual(Ok({ kind: "superseded" })); + const current = await capture(); + afterStaging(async () => { + await fs.writeFile(lockPath, "foreign-owner"); + }); + const before = await fs.readFile(chatPath); + let commits = 0; + const rejected = await history.acceptCompactionReplacement( + workspaceId, + current, + await operation("single"), + { + isCurrent: () => true, + onCommitted: () => { + commits++; + }, + } + ); + expect(rejected.success).toBe(false); + expect(commits).toBe(0); + expect(await fs.readFile(chatPath)).toEqual(before); + await fs.rm(lockPath); + }); + + it.each([ + ["active", "trigger"], + ["archive", "trigger"], + ["active", "payload"], + ["archive", "payload"], + ] as const)( + "refuses a receipt for an append identity already in %s (%s)", + async (location, collision) => { + if (location === "archive") + await history.appendToHistory( + workspaceId, + createMuxMessage("boundary", "assistant", "summary", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const result = await history.acceptCompactionReplacement( + workspaceId, + expected, + { + kind: "append", + messages: + collision === "trigger" + ? [createMuxMessage("prior", "user", "another occurrence")] + : [ + createMuxMessage("prior", "assistant", "conflicting payload"), + createMuxMessage("new-trigger", "user", "new input"), + ], + }, + noObserver + ); + expect(result).toEqual(Ok({ kind: "skipped" })); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok(null) + ); + } + ); + + it("recognizes archive replays while rejecting conflicting rows or duplicate active rows", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + const active = await fs.readFile(chatPath, "utf8"); + const committed = active.trimEnd().split("\n").at(-1)! + "\n"; + const archivePath = path.join(path.dirname(chatPath), "chat-archive.jsonl"); + await fs.writeFile(archivePath, committed); + const foreign = new HistoryService(fixture.config); + expect(await foreign.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + const conflicting = JSON.parse(committed) as MuxMessage; + conflicting.parts = [{ type: "text", text: "different occurrence bytes" }]; + await fs.writeFile(archivePath, JSON.stringify(conflicting) + "\n"); + expect(await foreign.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok(null) + ); + await fs.writeFile(archivePath, committed + committed + committed); + expect(await foreign.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + await fs.writeFile(archivePath, ""); + await fs.appendFile(chatPath, committed); + expect(await foreign.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok(null) + ); + await fs.writeFile(chatPath, active); + await fs.writeFile(archivePath, committed); + expect(await stop.retireReplacement({ nonce: expected.nonce! })).toBe("applied"); + }); + + it("keeps lifetime archive reads outside the history lock for lookup and witnessed retirement", async () => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + const archivePath = path.join(path.dirname(chatPath), "chat-archive.jsonl"); + const archived = + Array.from({ length: 2000 }, (_, i) => + JSON.stringify( + createMuxMessage(`archived-${i}`, "assistant", "x".repeat(1024), { + historySequence: i + 2, + }) + ) + ).join("\n") + "\n"; + await fs.writeFile(archivePath, archived); + let holdingLock = false; + let archiveBytesReadUnderLock = 0; + const withLock = workspaceFileLocks.withLock.bind(workspaceFileLocks); + spyOn(workspaceFileLocks, "withLock").mockImplementation((key, operation) => + withLock(key, async () => { + holdingLock = true; + try { + return await operation(); + } finally { + holdingLock = false; + } + }) + ); + beforeFileRead(archivePath, (args) => { + if (holdingLock && typeof args[2] === "number") archiveBytesReadUnderLock += args[2]; + }); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + expect(await stop.retireReplacement({ nonce: expected.nonce! })).toBe("applied"); + expect(archiveBytesReadUnderLock).toBe(0); + }); + + it.each(["Stop", "append"] as const)( + "lets a foreign %s finish during witnessed retirement's archive scan", + async (change) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + const archivePath = path.join(path.dirname(chatPath), "chat-archive.jsonl"); + await fs.writeFile( + archivePath, + JSON.stringify( + createMuxMessage("archive", "assistant", "old", { + historySequence: 99, + }) + ) + "\n" + ); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + let armed = true; + beforeFileRead(archivePath, async () => { + if (armed) { + armed = false; + entered.resolve(); + await release.promise; + } + }); + const retiring = stop.retireReplacement({ nonce: expected.nonce! }).then( + (result) => result, + (error: unknown) => error + ); + await entered.promise; + const foreign = new HistoryService(fixture.config); + const foreignStop = new CompactionCancellation( + foreign.getCompactionCancellationStorage(workspaceId) + ); + try { + expect(nodeFs.existsSync(historyWriteLockPath(fixture.config.rootDir, workspaceId))).toBe( + false + ); + if (change === "Stop") await foreignStop.cancel({ retainUntilReplacement: true }); + else + await foreign.appendToHistory( + workspaceId, + createMuxMessage("foreign", "assistant", "concurrent input") + ); + } finally { + release.resolve(); + await retiring; + } + if (change === "Stop") { + expect(await retiring).toBe("superseded"); + expect((await foreignStop.read())?.nonce).not.toBe(expected.nonce); + expect((await history.getCompactionCancellationStorage(workspaceId).read())?.nonce).toBe( + (await foreignStop.read())?.nonce + ); + } else { + expect(await retiring).toBeInstanceOf(Error); + expect(stop.needsPersistence).toBe(true); + expect(stop.blocksRecovery).toBe(false); + expect((await foreignStop.read())?.nonce).toBe(expected.nonce!); + expect(await stop.retry()).toBe("applied"); + expect(await foreignStop.read()).toBeNull(); + } + } + ); + + it.each([ + ["append", "lookup"], + ["rotate", "lookup"], + ["reset", "lookup"], + ["append", "accept"], + ] as const)( + "%s between %s evidence preparation and locking invalidates evidence without claiming absence", + async (mutation, use) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + await history.acceptCompactionReplacement( + workspaceId, + expected, + await operation("single"), + noObserver + ); + const foreign = new HistoryService(fixture.config); + const withLock = workspaceFileLocks.withLock.bind(workspaceFileLocks); + let armed = true; + spyOn(workspaceFileLocks, "withLock").mockImplementation(async (key, operation) => { + if (armed) { + armed = false; + if (mutation === "reset") await foreign.clearHistory(workspaceId); + else + await foreign.appendToHistory( + workspaceId, + createMuxMessage( + "foreign", + "assistant", + "changed", + mutation === "rotate" + ? { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + } + : undefined + ) + ); + } + return withLock(key, operation); + }); + const finding = + use === "lookup" + ? await history.findCompactionReplacementWitness(workspaceId, expected.nonce!) + : await history.acceptCompactionReplacement( + workspaceId, + expected, + { + kind: "append", + messages: [createMuxMessage("next-trigger", "user", "next input")], + }, + noObserver + ); + expect(finding.success).toBe(false); + expect((await rows()).some((row) => row.id === "next-trigger")).toBe(false); + expect((await history.getCompactionCancellationStorage(workspaceId).read())?.nonce).toBe( + expected.nonce! + ); + const refreshed = await history.findCompactionReplacementWitness( + workspaceId, + expected.nonce! + ); + expect(refreshed).toEqual(Ok(mutation === "reset" ? null : { nonce: expected.nonce! })); + } + ); + + it.each([0, 1])("acceptance and lookup agree at the row limit plus %d byte(s)", async (extra) => { + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + const message = createMuxMessage("limit", "user", "x"); + message.parts = [{ type: "text", text: "" }]; + const representation = { + ...message, + workspaceId, + metadata: { + ...message.metadata, + compactionReplacementNonce: expected.nonce!, + historySequence: 1, + }, + }; + message.parts = [ + { + type: "text", + text: "x".repeat( + SESSION_HISTORY_MAX_LINE_BYTES + extra - Buffer.byteLength(JSON.stringify(representation)) + ), + }, + ]; + let commits = 0; + const result = await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "append", messages: [message] }, + { + isCurrent: () => true, + onCommitted: () => { + commits++; + }, + } + ); + expect(result).toEqual(Ok({ kind: "accepted", witness: { nonce: expected.nonce! } })); + expect(commits).toBe(1); + const committedLine = (await fs.readFile(chatPath, "utf8")).trimEnd().split("\n").at(-1)!; + expect(Buffer.byteLength(committedLine)).toBe(SESSION_HISTORY_MAX_LINE_BYTES + extra); + const fresh = new HistoryService(fixture.config); + expect(await fresh.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + expect( + await fresh.getCompactionCancellationStorage(workspaceId).mutate( + { + kind: "retire", + nonce: expected.nonce!, + replacementWitness: { nonce: expected.nonce! }, + }, + () => true, + () => undefined + ) + ).toBe("applied"); + if (extra > 0) { + // The witness check does not gratuitously restrict ordinary large history inputs. + const absentWorkspace = "large-without-stop"; + const absent = await history.captureCompactionReplacement(absentWorkspace); + assert(absent.success); + expect( + await history.acceptCompactionReplacement( + absentWorkspace, + absent.data, + { + kind: "append", + messages: [ + { ...message, metadata: { ...message.metadata, historySequence: undefined } }, + ], + }, + noObserver + ) + ).toEqual(Ok({ kind: "accepted", witness: null })); + } + }); + + it.each(["update", "boundary"] as const)( + "preserves a resumed assistant receipt through stale %s finalization", + async (writer) => { + await history.appendToHistory( + workspaceId, + createMuxMessage("assistant", "assistant", "interrupted") + ); + const original = (await rows()).at(-1)!; + await stop.cancel({ retainUntilReplacement: true }); + const old = await capture(); + await history.acceptCompactionReplacement( + workspaceId, + old, + { kind: "resume", message: original }, + noObserver + ); + const stale = (await rows()).at(-1)!; + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + await history.acceptCompactionReplacement( + workspaceId, + expected, + { kind: "resume", message: stale }, + noObserver + ); + if (writer === "update") + expect(await history.updateHistory(workspaceId, stale)).toEqual(Ok(undefined)); + else + expect( + await history.persistBoundaryWithTailCopies( + workspaceId, + { + ...stale, + metadata: { + ...stale.metadata, + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }, + }, + [], + true + ) + ).toEqual(Ok(undefined)); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok({ nonce: expected.nonce! }) + ); + expect(await stop.retireReplacement({ nonce: expected.nonce! })).toBe("applied"); + } + ); + + it("preserves raw protected history and refuses to issue a receipt from a protected trigger", async () => { + const protectedRow = createMuxMessage( + "raw-floor", + "user", + "x".repeat(SESSION_HISTORY_MAX_LINE_BYTES), + { + contextBoundaryKind: "reset", + } + ); + await history.appendToHistory(workspaceId, protectedRow); + const original = await fs.readFile(chatPath); + await stop.cancel({ retainUntilReplacement: true }); + const expected = await capture(); + let commits = 0; + const failed = await history.acceptCompactionReplacement( + workspaceId, + expected, + { + kind: "append", + messages: [ + { ...protectedRow, id: "new-floor", metadata: { contextBoundaryKind: "reset" } }, + ], + }, + { + isCurrent: () => true, + onCommitted: () => { + commits++; + }, + } + ); + expect(failed.success).toBe(false); + expect(commits).toBe(0); + expect(await fs.readFile(chatPath)).toEqual(original); + expect(await history.findCompactionReplacementWitness(workspaceId, expected.nonce!)).toEqual( + Ok(null) + ); + const next = await capture(); + expect( + await history.acceptCompactionReplacement( + workspaceId, + next, + await operation("single"), + noObserver + ) + ).toEqual(Ok({ kind: "accepted", witness: { nonce: next.nonce! } })); + expect((await fs.readFile(chatPath)).subarray(0, original.length)).toEqual(original); + }); +}); diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 4d4ef681bf..78c509ec6c 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2178,9 +2178,10 @@ describe("HistoryService", () => { const advancing = spyOn(store, "advanceGenerationUnderHistoryLock").mockImplementationOnce( async () => { await fs.access(historyWriteLockPath(config.rootDir, ws)); - await advance(); + const result = await advance(); entered.resolve(); await release.promise; + return result; } ); const deleting = diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index ac1a08ddcc..1814326633 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -21,12 +21,18 @@ import { type HistoryControlRow, type BoundedHistoryScanOptions, } from "./historyScanner"; +import { + scanHistoryReplacementRows, + equalHistoryReplacementRows, + type HistoryReplacementRow, +} from "./historyReplacementRows"; +import { MuxMessageSchema } from "@/common/orpc/schemas/message"; import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import { isManualHistoryReset } from "@/common/utils/messages/contextWindows"; import { createContextBudgetRejectedMessage } from "@/common/utils/messages/contextBudgetRejection"; import * as path from "path"; import { createHash, randomUUID } from "node:crypto"; -import { renameSync, rmdirSync, unlinkSync } from "node:fs"; +import { fsyncSync, renameSync, rmdirSync, unlinkSync, writeSync } from "node:fs"; import { isDeepStrictEqual } from "node:util"; import * as fs from "fs/promises"; import type { @@ -39,6 +45,13 @@ import { type ContinuousCompactionPublication, } from "./continuousCompactionJournal"; import { CONTINUOUS_COMPACTION_JOURNAL_FILE } from "@/constants/continuousCompaction"; +import { + FileCompactionCancellationStorage, + type CompactionCancellationReplacementWitness, + type CompactionReplacementCapture, + type CompactionReplacementOperation, + type CompactionReplacementOutcome, +} from "./compactionCancellation"; import writeFileAtomic from "write-file-atomic"; import assert from "node:assert"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; @@ -107,7 +120,10 @@ interface HistoryTruncateTransaction extends HistoryTruncateHashes { interface HistoryPublicationObserver { assertStillOwned: () => Promise; isCurrent: () => boolean; - // Returning undefined excludes async callbacks: receipt capture must not yield after rename. + // Observable JSON prevents replay even when the durability barrier fails. It grants no receipt. + onPublished?: () => undefined; + onGenerationAdvanced?: (generation: string) => undefined; + // Returning undefined excludes async callbacks: receipt capture must not yield after publication. onCommitted: () => undefined; } @@ -117,6 +133,22 @@ interface HistoryRewriteRow { protectedMessage?: MuxMessage; } +interface ReplacementHistoryRow extends HistoryReplacementRow { + artifact: "chat" | "archive"; +} +type ReplacementHistoryScan = ( + visit: (row: ReplacementHistoryRow) => boolean | void | Promise, + nonce?: string +) => Promise; + +function replacementIdKey( + id: string | NonNullable["id"] +): string { + return typeof id === "string" + ? `${id.length}:${createHash("sha256").update(id, "utf16le").digest("hex")}` + : `${id.length}:${id.sha256}`; +} + function splitHistoryLines(raw: Buffer): Buffer[] { const lines: Buffer[] = []; for (let start = 0; start < raw.length; ) { @@ -260,6 +292,20 @@ function getCompactionMetadataToPreserve( return preserved; } +function getReplacementMetadataToPreserve( + existing: MuxMessage, + incoming: MuxMessage +): Partial { + // A stale finalizer cannot replace the receipt of this exact accepted occurrence. + const nonce = existing.metadata?.compactionReplacementNonce; + return existing.id === incoming.id && + existing.role === incoming.role && + typeof nonce === "string" && + nonce.length > 0 + ? { compactionReplacementNonce: nonce } + : {}; +} + /** * Whether a partial message's parts are durable enough to commit to * chat.jsonl. Exported so StreamManager's abort path can apply the SAME @@ -1664,9 +1710,11 @@ export class HistoryService { while (readPos < fileSize) { const remaining = fileSize - readPos; const toRead = Math.min(HistoryService.REVERSE_READ_CHUNK_SIZE, remaining); - const rawChunk = Buffer.alloc(toRead); - await fh.read(rawChunk, 0, toRead, readPos); - readPos += toRead; + const allocation = Buffer.alloc(toRead); + const { bytesRead } = await fh.read(allocation, 0, toRead, readPos); + if (bytesRead === 0) throw new Error("History ended before its captured size"); + const rawChunk = allocation.subarray(0, bytesRead); + readPos += bytesRead; const buffer = carryoverBytes.length > 0 ? Buffer.concat([carryoverBytes, rawChunk]) : rawChunk; @@ -1718,13 +1766,14 @@ export class HistoryService { if (carryoverBytes.length > 0) { const line = carryoverBytes.toString("utf-8").trim(); if (line.length > 0) { + let msg: MuxMessage; try { - const msg = normalizeLegacyMuxMetadata(JSON.parse(line) as MuxMessage); - const shouldContinue = await visitor([msg], [line], carryoverBytes); - if (shouldContinue === false) return false; + msg = normalizeLegacyMuxMetadata(JSON.parse(line) as MuxMessage); } catch { - // Skip malformed line + return true; // Skip malformed JSON, but never swallow a visitor's I/O failure. } + const shouldContinue = await visitor([msg], [line], carryoverBytes); + if (shouldContinue === false) return false; } } return true; @@ -2932,7 +2981,8 @@ export class HistoryService { private async fenceContextResetUnderHistoryLock( workspaceId: string, - messages: readonly MuxMessage[] + messages: readonly MuxMessage[], + publication?: HistoryPublicationObserver ): Promise { if ( messages.some((message) => getContextBoundaryKind(message) === CONTEXT_BOUNDARY_KINDS.RESET) @@ -2940,7 +2990,13 @@ export class HistoryService { // Reset/rollover discards context captured by foreign compactors too. Advance // after admission but before appending under the same lock, so a failed // generation write cannot leave a durable reset open to stale publication. - await this.getContinuousCompactionJournal(workspaceId).advanceGenerationUnderHistoryLock(); + // If later history staging loses ownership, keep this conservative fence: it + // may require recomputing a summary, but only the history receipt accepts input. + await this.getContinuousCompactionJournal(workspaceId).advanceGenerationUnderHistoryLock( + publication?.onGenerationAdvanced, + publication?.assertStillOwned, + publication?.isCurrent + ); } } @@ -2954,30 +3010,32 @@ export class HistoryService { messages: MuxMessage[]; }> { const raw = (await this.readExistingFileBytes(filePath)) ?? Buffer.alloc(0); - const rows = splitHistoryLines(raw).map((line) => { - // Match the provider scanner's row budget without the JSONL delimiter. - const content = line.at(-1) === 10 ? line.subarray(0, -1) : line; - const text = content.toString("utf8"); - const parsed = this.parseMessages(text, filePath, (value) => - isReadableHistoryMessage(value) ? normalizeLegacyMuxMetadata(value) : null - )[0]; - const protectedReset = - parsed !== undefined && - ((hasRawResetMarker(text) && hasAmbiguousResetKeys(text)) || - // Oversized rows use the provider scanner's token probe, even when - // intervening bytes prevent a contiguous raw reset marker match. - (content.length > SESSION_HISTORY_MAX_LINE_BYTES && - hasUnreadableHistoryResetEvidence([line]))); - return { - raw: line, - message: protectedReset ? undefined : parsed, - // Keep identity and sequence accounting even when the raw floor cannot be rewritten. - protectedMessage: protectedReset ? parsed : undefined, - }; - }); + const rows = splitHistoryLines(raw).map((line) => this.parseHistoryRewriteRow(line, filePath)); return { rows, messages: rows.flatMap((row) => (row.message ? [row.message] : [])) }; } + private parseHistoryRewriteRow(line: Buffer, filePath: string): HistoryRewriteRow { + // Match the provider scanner's row budget without the JSONL delimiter. + const content = line.at(-1) === 10 ? line.subarray(0, -1) : line; + const text = content.toString("utf8"); + const parsed = this.parseMessages(text, filePath, (value) => + isReadableHistoryMessage(value) ? normalizeLegacyMuxMetadata(value) : null + )[0]; + const protectedReset = + parsed !== undefined && + ((hasRawResetMarker(text) && hasAmbiguousResetKeys(text)) || + // Oversized rows use the provider scanner's token probe, even when + // intervening bytes prevent a contiguous raw reset marker match. + (content.length > SESSION_HISTORY_MAX_LINE_BYTES && + hasUnreadableHistoryResetEvidence([line]))); + return { + raw: line, + message: protectedReset ? undefined : parsed, + // Keep identity and sequence accounting even when the raw floor cannot be rewritten. + protectedMessage: protectedReset ? parsed : undefined, + }; + } + private getProtectedRewriteMaxSequence(rows: readonly HistoryRewriteRow[]): number { // These parsed rows survive every partial rewrite as raw bytes, even beyond a cut. // Their sequences remain occupied regardless of whether they are transformable. @@ -3223,6 +3281,504 @@ export class HistoryService { ); } + /** Inactive until all Stop/send/resume/recovery consumers adopt the same authority. */ + getCompactionCancellationStorage(workspaceId: string): FileCompactionCancellationStorage { + return new FileCompactionCancellationStorage(this, workspaceId, async (witness, signal) => { + const evidence = await this.prepareCompactionReplacementWitness( + workspaceId, + witness.nonce, + signal + ); + if (!evidence.success) throw new Error(evidence.error); + return async () => (await evidence.data()) !== null; + }); + } + + private async captureCompactionReplacementUnderHistoryLock( + workspaceId: string + ): Promise { + const cancellation = await this.getCompactionCancellationStorage(workspaceId).read(); + const generation = + await this.getContinuousCompactionJournal(workspaceId).captureGenerationUnderHistoryLock(); + return { nonce: cancellation?.nonce ?? null, generation }; + } + + captureCompactionReplacement( + workspaceId: string, + options?: { onRepaired: () => void; replaceUnreadable?: boolean } + ): Promise> { + if (options) + return this.withCompactionStorageLock(workspaceId, async (_dir, checkLock) => { + // Repair and capture share one storage acquisition. Neither a foreign Stop nor + // malformed-state recovery may replace the request's frontier during preflight. + await this.getCompactionCancellationStorage(workspaceId).repairUnderHistoryLock( + () => true, + options.onRepaired, + checkLock, + options.replaceUnreadable + ); + return this.getAppendProvenance(workspaceId).runMutation(async () => { + await this.recoverTruncateTransactionUnlocked(workspaceId, checkLock); + return Ok(await this.captureCompactionReplacementUnderHistoryLock(workspaceId)); + }, checkLock); + }).catch((error: unknown) => Err(`Failed to capture replacement: ${getErrorMessage(error)}`)); + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to capture replacement", + async () => Ok(await this.captureCompactionReplacementUnderHistoryLock(workspaceId)) + ); + } + + /** The caller's capture fences preparation, including a Stop/retirement back to absence. */ + async acceptCompactionReplacement( + workspaceId: string, + capture: CompactionReplacementCapture, + operation: CompactionReplacementOperation, + observer: { + isCurrent: () => boolean; + onContextResetCommitted?: ( + predecessor: CompactionReplacementCapture, + successor: CompactionReplacementCapture + ) => undefined; + onCommitted: ( + accepted: Extract + ) => undefined; + } + ): Promise> { + const expected = { ...capture }; + // Compare persisted JSON values in this realm: native structuredClone can return + // host-prototype objects in a VM, which falsely fail exact history comparisons. + let prepared: CompactionReplacementOperation; + let originalMessages: MuxMessage[]; + try { + originalMessages = operation.kind === "append" ? [...operation.messages] : []; + // Snapshot before any await, while keeping invalid unknown payloads in the Result contract. + prepared = JSON.parse(JSON.stringify(operation)) as CompactionReplacementOperation; + } catch (error) { + return Err(`Failed to accept compaction replacement: ${getErrorMessage(error)}`); + } + // Ordinary input still compares the captured Stop and generation under the write lock, + // but its durable append receipt must not grant authority to replace that Stop. + const replacementNonce = + prepared.kind === "append" && prepared.preserveCancellation ? null : expected.nonce; + const reusesWitness = + prepared.kind === "resume" && + replacementNonce !== null && + prepared.message.metadata?.compactionReplacementNonce === replacementNonce; + if (!observer.isCurrent()) return Ok({ kind: "superseded" }); + let verifyIdentities: (() => Promise) | undefined; + if (replacementNonce !== null) { + const ids = new Set( + (prepared.kind === "append" + ? prepared.messages.map((message) => message.id) + : [prepared.message.id] + ) + .filter((id) => typeof id === "string") + .map(replacementIdKey) + ); + const evidence = await this.prepareCompactionHistoryEvidence(workspaceId, async (scan) => { + // Payload identities must be fresh too: rollback/delete selects every matching id. + let matches = 0; + let chatMatches = 0; + let firstMatch: ReplacementHistoryRow | undefined; + const expectedMatches = prepared.kind === "resume" ? 1 : 0; + await scan(async (row) => { + const other = row.identity; + const matchesIdentity = + other && + (ids.has(replacementIdKey(other.id)) || + (prepared.kind === "resume" && + other.sequence === prepared.message.metadata?.historySequence)); + // An accepted replacement consumes this Stop even before sidecar retirement. + // Only the exact stamped Resume may reuse its byte-identical archive replays. + // Legacy ineligible stamps cannot consume authority or strand a fresh replacement. + if ( + row.matchesNonce && + row.replacementCandidate && + (!reusesWitness || !matchesIdentity) + ) { + matches = expectedMatches + 1; + return false; + } + if (matchesIdentity) { + if (row.artifact === "chat") chatMatches++; + // Interrupted rotation can replay an already stamped Resume into the archive. + // Only exact bytes may share its identity; two active rows still conflict. + if ( + !reusesWitness || + !firstMatch || + chatMatches > 1 || + !(await equalHistoryReplacementRows(firstMatch, row)) + ) + matches++; + firstMatch ??= row; + } + return matches <= expectedMatches; + }, replacementNonce); + return matches === expectedMatches; + }); + if (!evidence.success) return evidence; + verifyIdentities = evidence.data; + } + let verifyExistingWitness: + | (() => Promise) + | undefined; + if (reusesWitness) { + const evidence = await this.prepareCompactionReplacementWitness( + workspaceId, + replacementNonce + ); + if (!evidence.success) return evidence; + verifyExistingWitness = evidence.data; + } + let accepted: Extract | undefined; + const result = await this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to accept compaction replacement", + async (assertStillOwned) => { + if (!observer.isCurrent()) return Ok({ kind: "superseded" }); + const current = await this.captureCompactionReplacementUnderHistoryLock(workspaceId); + if (expected.nonce !== current.nonce || expected.generation !== current.generation) + return Ok({ kind: "superseded" }); + let messages: MuxMessage[]; + if (prepared.kind === "append") { + messages = prepared.messages; + if (messages.length === 0) return Ok({ kind: "skipped" }); + const trigger = messages.at(-1)!; + if ( + new Set(messages.map((message) => message.id)).size !== messages.length || + messages.some( + (message) => !message.id || message.metadata?.historySequence !== undefined + ) || + // Rejected inputs use inert assistant capsules for older-reader compatibility. + (trigger.role !== "user" && + !( + prepared.preserveCancellation && + trigger.role === "assistant" && + trigger.metadata?.contextBudgetRejected + )) + ) + return Ok({ kind: "skipped" }); + for (const message of messages) { + const { compactionReplacementNonce: _receipt, ...metadata } = message.metadata ?? {}; + message.metadata = metadata; + } + } else { + const target = prepared.message; + if (!isNonNegativeInteger(target.metadata?.historySequence)) + return Ok({ kind: "skipped" }); + const { rows } = await this.readHistoryForRewrite(this.getChatHistoryPath(workspaceId)); + const matches = rows.filter((row) => { + const message = row.message ?? row.protectedMessage; + return ( + message && + (message.id === target.id || + message.metadata?.historySequence === target.metadata?.historySequence) + ); + }); + const row = matches[0]; + // Persistence adds workspaceId; the wire schema also removes transient text + // state. Preserve every other field when proving the caller's exact target. + const messageShape = (message: MuxMessage) => { + const { workspaceId: _workspaceId, ...declared } = message as MuxMessage & { + workspaceId?: unknown; + }; + return { + ...declared, + parts: declared.parts.map((part) => { + if (part.type !== "text" && part.type !== "reasoning") return part; + const { state: _state, ...persisted } = part as typeof part & { state?: unknown }; + return persisted; + }), + }; + }; + const matchesTarget = (message: MuxMessage) => { + if (isDeepStrictEqual(messageShape(message), messageShape(target))) return true; + // Wire callers cannot attest fields omitted by the schema (for example a + // reasoning signature). Accept only that exact projection, not two projected + // objects: any differing field supplied by the caller must still reject. + // JSONL dates need the same normalization as transcript reads. Copy first: + // comparison must not mutate the persisted row that receives the witness. + const wire = MuxMessageSchema.safeParse( + this.normalizeTranscriptMessage({ ...message }) + ); + return wire.success && isDeepStrictEqual(JSON.parse(JSON.stringify(wire.data)), target); + }; + // updateHistory selects by sequence. Prove its unique target, including protected + // raw rows, before using that shared writer; never fall back to id-only resume. + if ( + matches.length !== 1 || + !row || + !this.isCompactionReplacementRow(row) || + !matchesTarget(row.message!) + ) + return Ok({ kind: "skipped" }); + // Stamp the verified disk row so normalization never erases persisted fields. + messages = [row.message!]; + } + const trigger = messages.at(-1)!; + // Ordinary input records the rejection without gaining authority to replace a Stop. + if ( + trigger.metadata?.contextBudgetRejected && + !(prepared.kind === "append" && prepared.preserveCancellation) + ) + return Ok({ kind: "skipped" }); + if (prepared.kind === "resume" && expected.nonce === null) { + // Ordinary Retry/Resume has nothing to stamp. Keep the captured generation and + // exact-target checks above, but accept the existing row without rewriting history. + await assertStillOwned(); + if (!observer.isCurrent()) return Ok({ kind: "superseded" }); + accepted = { kind: "accepted", witness: null }; + observer.onCommitted(structuredClone(accepted)); + return Ok(accepted); + } + if (replacementNonce !== null) { + if (!(await verifyIdentities!())) return Ok({ kind: "skipped" }); + if (verifyExistingWitness) { + // A failed sidecar retirement must not make explicit Retry inert. Reuse the + // exact durable stamp only after the same identity, generation and lock checks; + // witness verification flushes and revalidates the existing publication. + const witness = await verifyExistingWitness(); + if (!witness) return Ok({ kind: "skipped" }); + await assertStillOwned(); + if (!observer.isCurrent()) return Ok({ kind: "superseded" }); + accepted = { kind: "accepted", witness }; + observer.onCommitted(structuredClone(accepted)); + return Ok(accepted); + } + trigger.metadata = { ...trigger.metadata, compactionReplacementNonce: replacementNonce }; + } + let superseded = false; + const rememberPublished = (): undefined => { + // Caller queue mutations cannot change the batch receiving publication metadata. + originalMessages.forEach((message, index) => { + message.metadata = messages[index].metadata; + }); + }; + let resetGeneration: string | undefined; + const publication: HistoryPublicationObserver = { + onGenerationAdvanced: (generation) => { + resetGeneration = generation; + }, + onPublished: rememberPublished, + assertStillOwned: async () => { + if (replacementNonce !== null) { + const message = { ...trigger, workspaceId }; + const raw = Buffer.from(JSON.stringify(message)); + // Check the actual assigned sequence and representation. Large ordinary rows + // remain supported; a protected raw privacy floor cannot issue a witness. + if ( + !this.isCompactionReplacementRow({ raw, message }) || + (raw.length > SESSION_HISTORY_MAX_LINE_BYTES && + hasUnreadableHistoryResetEvidence([raw])) + ) + throw new Error("Replacement row cannot establish a durable witness"); + } + await assertStillOwned(); + }, + isCurrent: () => { + superseded = !observer.isCurrent(); + return !superseded; + }, + onCommitted: () => { + // This receipt is authoritative even if notification, provenance finalization, + // or lock disposal fails later. No await may separate publication from this capture. + accepted = { + kind: "accepted", + witness: replacementNonce === null ? null : { nonce: replacementNonce }, + }; + rememberPublished(); + try { + observer.onCommitted(structuredClone(accepted)); + } finally { + // A generation write alone grants no authority. Observer failure cannot undo + // this history receipt or strand queued work on its committed predecessor. + if (resetGeneration !== undefined) + observer.onContextResetCommitted?.( + { ...expected }, + { ...expected, generation: resetGeneration } + ); + } + }, + }; + const written = + prepared.kind === "append" + ? await this.appendManyToHistoryUnderWriteLock(workspaceId, messages, publication) + : await this.updateHistoryUnderWriteLock(workspaceId, trigger, publication); + if (accepted) return Ok(accepted); + if (superseded) return Ok({ kind: "superseded" }); + return written.success + ? Err("History publication did not issue an acceptance receipt") + : written; + } + ); + return accepted ? Ok(accepted) : result; + } + + private isCompactionReplacementRow(row: HistoryRewriteRow): boolean { + const message = row.message; + const content = row.raw.at(-1) === 10 ? row.raw.subarray(0, -1) : row.raw; + const text = content.toString("utf8"); + return ( + isReadableHistoryMessage(message) && + message.id.length > 0 && + // The readable-history predicate also accepts legacy array-coerced roles. + String(message.role) !== "system" && + isNonNegativeInteger(message.metadata?.historySequence) && + // Canonical round trips prove unambiguous keys without imposing the bounded + // scanner's limit on ordinary large input. Both paths exclude the JSONL delimiter. + (content.equals(Buffer.from(JSON.stringify(message))) || + (Buffer.from(text).equals(content) && !hasAmbiguousResetKeys(text))) + ); + } + + async findCompactionReplacementWitness( + workspaceId: string, + nonce: string + ): Promise> { + const evidence = await this.prepareCompactionReplacementWitness(workspaceId, nonce); + if (!evidence.success) return evidence; + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to verify replacement", + async (assertStillOwned) => { + const witness = await evidence.data(); + await assertStillOwned(); + return Ok(witness); + } + ); + } + + private async prepareCompactionReplacementWitness( + workspaceId: string, + nonce: string, + signal?: AbortSignal + ) { + const artifacts = new Set(); + const evidence = await this.prepareCompactionHistoryEvidence( + workspaceId, + (rows) => this.findCompactionReplacementInRows(rows, nonce, artifacts, signal), + signal + ); + if (!evidence.success) return evidence; + return Ok(async () => { + const witness = await evidence.data(); + if (witness) { + // A visible nonce may survive a failed file or rename flush. Both lookup and Stop + // retirement must establish durability before authority, then reject changed evidence. + for (const artifact of artifacts) { + const file = + artifact === "chat" + ? this.getChatHistoryPath(workspaceId) + : this.getChatArchivePath(workspaceId); + // Windows FlushFileBuffers requires write access even though verification writes no bytes. + const handle = await fs.open(file, "r+"); + try { + await handle.sync(); + } finally { + await handle.close(); + } + } + await using directory = await this.openHistoryPublicationDirectory( + this.getChatHistoryPath(workspaceId) + ); + await directory?.sync(); + await evidence.data(); + } + return witness; + }); + } + + private async prepareCompactionHistoryEvidence( + workspaceId: string, + inspect: (scan: ReplacementHistoryScan) => Promise, + signal?: AbortSignal + ): Promise Promise>> { + try { + signal?.throwIfAborted(); + const provenance = this.getAppendProvenance(workspaceId); + const before = await provenance.stamps(); + signal?.throwIfAborted(); + // Lifetime witness verification must remain complete without buffering a giant row or + // borrowing the session_history tool's paging budget. Only bounded evidence escapes a scan. + const scan: ReplacementHistoryScan = async (visit, nonce) => { + for (const artifact of ["chat", "archive"] as const) { + const file = artifact === "chat" ? provenance.chatPath : provenance.archivePath; + const complete = await scanHistoryReplacementRows( + file, + (row) => visit({ ...row, artifact }), + { nonce, signal } + ); + if (!complete) return; + } + }; + const value = await inspect(scan); + const assertUnchanged = async () => { + signal?.throwIfAborted(); + const after = await provenance.stamps(); + signal?.throwIfAborted(); + if (!isDeepStrictEqual(before, after)) + throw new Error( + "Compaction history changed during verification; retry with fresh evidence" + ); + }; + await assertUnchanged(); + return Ok(async () => { + // The cancellation adapter holds a bare lock; only the public history path recovers. + if (await this.truncateRecoveryArtifactsPresent(workspaceId)) + throw new Error("Compaction history requires recovery before verification"); + await assertUnchanged(); + return value; + }); + } catch (error) { + return Err(`Failed to prepare replacement evidence: ${getErrorMessage(error)}`); + } + } + + private async findCompactionReplacementInRows( + scan: ReplacementHistoryScan, + nonce: string, + witnessArtifacts: Set, + signal?: AbortSignal + ): Promise { + let witness: CompactionCancellationReplacementWitness | null = null; + // The outer iterator is the candidate cursor. Each candidate needs an exhaustive identity + // pass, trading extra reads for bounded memory even when every earlier candidate collides. + await scan(async (row) => { + const identity = row.identity; + if (!nonce || !row.matchesNonce || !row.replacementCandidate || !identity) return; + let chatMatches = 0; + let identical = true; + const artifacts = new Set(); + await scan(async (candidate) => { + const other = candidate.identity; + if ( + other && + (replacementIdKey(other.id) === replacementIdKey(identity.id) || + other.sequence === identity.sequence || + // Every eligible occurrence must prove the same receipt, not just this identity. + (candidate.matchesNonce && candidate.replacementCandidate)) + ) { + if (candidate.artifact === "chat") chatMatches++; + // Fingerprint collisions only add conservative conflicts; replay authority still + // requires exact bytes from both captured ranges, including their original encoding. + identical &&= await equalHistoryReplacementRows(candidate, row, signal); + artifacts.add(candidate.artifact); + } + return identical && chatMatches <= 1; + }, nonce); + // Rotation may finish an unterminated archive row and then append its source + // again. Identical archive replays prove one occurrence, with or without LF; + // conflicting identity bytes or multiple active-chat rows remain ambiguous. + if (identical && chatMatches <= 1 && artifacts.size > 0) { + witness = { nonce }; + for (const artifact of artifacts) witnessArtifacts.add(artifact); + return false; + } + }, nonce); + return witness; + } + // Replacement acceptance can reuse allocation and provenance under the already-held locks. private async appendManyToHistoryUnderWriteLock( workspaceId: string, @@ -3255,12 +3811,20 @@ export class HistoryService { // temp-and-rename helper the other history mutations use, under the // cross-process append lock (r50) so a foreign backend's row cannot // land between this read and the replace and be silently deleted. - await this.fenceContextResetUnderHistoryLock(workspaceId, messages); + await this.fenceContextResetUnderHistoryLock(workspaceId, messages, publication); + // A single accepted trigger needs no batch rewrite. Keep ordinary typing append-only + // while provenance still owns torn-tail handling and exact byte certification. + const atomic = !publication || messages.length !== 1; await this.getAppendProvenance(workspaceId).appendChat( Buffer.from(this.serializeHistoryEntries(messages, workspaceId)), - true, - publication && - ((filePath, bytes) => this.publishHistoryUnderWriteLock(filePath, bytes, publication)) + atomic, + publication && atomic + ? (filePath, bytes) => this.publishHistoryUnderWriteLock(filePath, bytes, publication) + : undefined, + publication && !atomic + ? (filePath, bytes, createsFile) => + this.appendHistoryUnderWriteLock(filePath, bytes, publication, createsFile) + : undefined ); // Publish the entire batch before sealing its previous epoch. Rotation // is best-effort: a storage failure must not invite a duplicate batch. @@ -3542,6 +4106,60 @@ export class HistoryService { ); } + private async appendHistoryUnderWriteLock( + historyPath: string, + bytes: Buffer, + publication: HistoryPublicationObserver, + createsFile: boolean + ): Promise { + assert(bytes.at(-1) === 10, "History append requires a JSONL delimiter"); + const handle = await fs.open(historyPath, "a", 0o600); + let committed = false; + try { + await using directory = createsFile + ? await this.openHistoryPublicationDirectory(historyPath) + : undefined; + await publication.assertStillOwned(); + if (!publication.isCurrent()) throw new Error("History publication no longer owned"); + // Stop cannot enter between the final check, complete row append and receipt. + // Recovery accepts a complete unterminated JSON row, so its final delimiter + // cannot define acceptance. Incomplete JSON still receives no receipt. + let offset = 0; + while (offset < bytes.length) { + const written = writeSync(handle.fd, bytes, offset, bytes.length - offset); + assert(written > 0, "History append must make progress"); + offset += written; + if (!committed && offset >= bytes.length - 1) { + publication.onPublished?.(); + // Flush this exact descriptor before acceptance; no await may admit Stop between + // the guarded append, durability barrier and receipt (even without the final LF). + fsyncSync(handle.fd); + // A new inode's name must survive a crash before its receipt can retire Stop. + if (directory) fsyncSync(directory.fd); + committed = true; + try { + publication.onCommitted(); + } catch (error) { + log.warn("History appended but commit observer failed", { error }); + } + } + } + } finally { + await handle.close().catch((error: unknown) => { + if (!committed) throw error; + log.warn("History appended but handle close failed", { error }); + }); + } + } + + private openHistoryPublicationDirectory(historyPath: string) { + // Match append provenance: Windows cannot fsync directory handles. It retains its + // existing rename boundary; POSIX acceptance additionally requires directory durability. + return process.platform === "win32" + ? Promise.resolve(undefined) + : fs.open(path.dirname(historyPath), "r"); + } + /** Caller holds both history locks. Ordinary writes retain their existing publication behavior. */ private async publishHistoryUnderWriteLock( historyPath: string, @@ -3554,12 +4172,17 @@ export class HistoryService { let committed = false; try { await writeFileAtomic(stagedPath, bytes, { mode: 0o600 }); + await using directory = await this.openHistoryPublicationDirectory(historyPath); // Staging can outlive a filesystem lease even while the logical owner is current. await publication.assertStillOwned(); // Replacement acceptance must capture its receipt in the same synchronous // turn as ownership validation and rename, before any observer can yield. if (!publication.isCurrent()) throw new Error("History publication no longer owned"); renameSync(stagedPath, historyPath); + // Visible rows forbid duplicate retries, but only the durable rename grants acceptance. + // Keep the final admission, rename, flush, and receipt in one synchronous region. + publication.onPublished?.(); + if (directory) fsyncSync(directory.fd); committed = true; try { publication.onCommitted(); @@ -3621,6 +4244,7 @@ export class HistoryService { metadata: { ...message.metadata, ...(preservedCompactionMetadata ?? {}), + ...(publication ? {} : getReplacementMetadataToPreserve(existingMessage, message)), historySequence: targetSequence, }, }; @@ -3771,6 +4395,7 @@ export class HistoryService { metadata: { ...summaryMessage.metadata, ...(preservedCompactionMetadata ?? {}), + ...getReplacementMetadataToPreserve(messages[i], summaryMessage), historySequence: targetSequence, }, }; diff --git a/tests/ipc/streaming/compactionReplacement.mock.test.ts b/tests/ipc/streaming/compactionReplacement.mock.test.ts new file mode 100644 index 0000000000..1815ec82ee --- /dev/null +++ b/tests/ipc/streaming/compactionReplacement.mock.test.ts @@ -0,0 +1,75 @@ +import { createTestEnvironment, cleanupTestEnvironment } from "../setup"; +import { + createTempGitRepo, + cleanupTempGitRepo, + createWorkspace, + generateBranchName, + createStreamCollector, + HAIKU_MODEL, +} from "../helpers"; +import { createMuxMessage } from "@/common/types/message"; +import { CompactionCancellation } from "@/node/services/compactionCancellation"; +import { HistoryService } from "@/node/services/historyService"; + +// Jest supplies native structuredClone from another realm. Exercise the real acceptance +// seam and IPC resume with mock streaming so value equality cannot regress unnoticed. +describe("replacement acceptance before mock IPC resume", () => { + test.each(["append", "resume"] as const)("accepts an exact %s replacement", async (kind) => { + const env = await createTestEnvironment(); + env.services.aiService.enableMockMode(); + const repo = await createTempGitRepo(); + try { + const workspace = await createWorkspace(env, repo, generateBranchName("replacement")); + if (!workspace.success) throw new Error(String(workspace.error)); + const workspaceId = workspace.metadata.id; + const history = new HistoryService(env.config); + const original = createMuxMessage("original", "user", "mock input"); + expect((await history.appendToHistory(workspaceId, original)).success).toBe(true); + const cancellation = new CompactionCancellation( + history.getCompactionCancellationStorage(workspaceId) + ); + await cancellation.cancel({ retainUntilReplacement: true }); + const capture = await history.captureCompactionReplacement(workspaceId); + if (!capture.success) throw new Error(capture.error); + const loaded = await history.getLastMessages(workspaceId, 1); + if (!loaded.success) throw new Error(loaded.error); + const result = await history.acceptCompactionReplacement( + workspaceId, + capture.data, + kind === "append" + ? { kind, messages: [createMuxMessage("replacement", "user", "mock replacement")] } + : { kind, message: loaded.data[0] }, + { isCurrent: () => true, onCommitted: () => undefined } + ); + expect(result).toEqual({ + success: true, + data: { kind: "accepted", witness: { nonce: capture.data.nonce } }, + }); + if (!result.success || result.data.kind !== "accepted" || !result.data.witness) + throw new Error("Expected a replacement witness"); + await cancellation.retireReplacement(result.data.witness); + const collector = createStreamCollector(env.orpc, workspaceId); + collector.start(); + try { + await collector.waitForSubscription(5000); + expect( + ( + await env.orpc.workspace.resumeStream({ + workspaceId, + options: { model: HAIKU_MODEL, agentId: "exec" }, + }) + ).success + ).toBe(true); + await collector.waitForEvent("stream-end", 15000); + expect( + collector.getEvents().filter((event) => "type" in event && event.type === "stream-error") + ).toEqual([]); + } finally { + collector.stop(); + } + } finally { + await cleanupTestEnvironment(env); + await cleanupTempGitRepo(repo); + } + }); +});