diff --git a/.omo/evidence/20260727-v24r13-capacity-aware-compaction/QA.md b/.omo/evidence/20260727-v24r13-capacity-aware-compaction/QA.md new file mode 100644 index 00000000..ea9fe7e2 --- /dev/null +++ b/.omo/evidence/20260727-v24r13-capacity-aware-compaction/QA.md @@ -0,0 +1,67 @@ +# V24R13 capacity-aware compaction QA + +## What was tested + +1. A new uneven protected-frame counterexample was built and run on the V24R12 + base before the Core implementation. +2. The same focused context suite was rebuilt and rerun after implementation, + including aggregate retention, pair integrity, newest-frame, oversized + newest-frame, and no-summarizable boundaries. +3. The preserved V24R12 Case 09 durable session shape was replayed through both + built engines with a local mock summary model. The replay emitted statistics + only. +4. Focused AgentLoop, Progress Lease, real local Gateway, O1/router observation, + and Legal Coverage control suites were run together. +5. The complete repository build and test suite plus `git diff --check` were run. + +## What was observed + +- Red counterexample: 9 existing tests passed; only the new aggregate-retention + assertion failed on V24R12 because exact retained tokens exceeded 35%. +- Green context suite: 11/11 passed in 437.1065 ms. +- Preserved replay input: 27 durable messages, 88,280 estimated tokens, 30,897 + exact-retention target. +- V24R12 replay: 42,098 exact retained tokens (47.6869%), 42,157 projected + post-message tokens, and no protected agent pair entered summary. +- V24R13 replay: 27,232 exact retained tokens (30.8473%), 27,291 projected + post-message tokens, and both older protected agent pairs entered summary. +- Both replay partitions retained complete tool call/result pairs and the newest + pair remained exact. +- Focused controls: 95/95 passed, including real local Gateway/O1 legal flows. +- Full repository: 313/313 passed after a clean build. +- Patch hygiene: `git diff --check` passed; the frozen dependency lock was not + modified. + +Artifacts: + +- `red-counterexample.log` +- `green-counterexample.log` +- `preserved-session-comparison.md` +- `replay-preserved-case09-context.mjs` +- `focused-controls.log` +- `full-suite.log` + +## Why it is enough for the code gate + +The red/green test isolates the previous count-based bypass and proves the new +aggregate token ceiling. The preserved replay demonstrates the same behavior +on the actual failed session's durable message shape without copying content. +Pair assertions cover provider validity, and the focused real-Gateway controls +show that dynamic context, Lease accounting, O1, and Legal Coverage still +compose. The full suite covers unrelated regressions. + +## Remaining product risk + +This QA does not prove that a real model summary preserves every semantic fact, +that the complete request including system/tools/dynamic context clears the +blocking threshold in every trajectory, or that Case 09 reaches completion. +Those are product-level claims and require a fresh immutable Gateway/O1 +campaign. V25 and the 85-case campaign remain unauthorized until full Case 09 +passes with completion proof and a substantive validated report. + +## What was omitted + +No API key, token, auth header, environment dump, private source content, +prompt body, legal report text, or model reasoning was recorded. The replay +script reads the preserved session locally but emits only aggregate metrics and +booleans. diff --git a/.omo/evidence/20260727-v24r13-capacity-aware-compaction/preserved-session-comparison.md b/.omo/evidence/20260727-v24r13-capacity-aware-compaction/preserved-session-comparison.md new file mode 100644 index 00000000..4e86edea --- /dev/null +++ b/.omo/evidence/20260727-v24r13-capacity-aware-compaction/preserved-session-comparison.md @@ -0,0 +1,59 @@ +# Preserved Case 09 context-shape replay + +The same content-silent replay script loaded the preserved V24R12 session and +ran against the built V24R12 and V24R13 `CompactionEngine` implementations. +The script emitted only counts, token estimates, ratios, and booleans. It did +not emit source content, prompts, reasoning, credentials, or tool-call IDs. + +Command shape: + +```text +node replay-preserved-case09-context.mjs +``` + +Shared input metrics: + +```text +durable messages: 27 +estimated message tokens: 88,280 +keepTailRatio: 0.35 +exact-retention target: 30,897 tokens +``` + +V24R12 (`d2953974`): + +```text +exact retained messages: 14 +exact retained tokens: 42,098 +exact retained ratio: 0.476869 +projected post-message tokens: 42,157 +summary input messages: 13 +summary input tokens: 46,182 +protected agent calls summarized: 0 +protected agent calls retained: 2 +summary tool pairs complete: true +retained tool pairs complete: true +newest tool pair retained: true +``` + +V24R13 working tree: + +```text +exact retained messages: 8 +exact retained tokens: 27,232 +exact retained ratio: 0.308473 +projected post-message tokens: 27,291 +summary input messages: 19 +summary input tokens: 61,048 +protected agent calls summarized: 2 +protected agent calls retained: 0 +summary tool pairs complete: true +retained tool pairs complete: true +newest tool pair retained: true +``` + +The old planner exceeded its 35% target because count-selected tail and +protected-prefix retention were independent. The new planner stays below the +aggregate target, moves the older protected pairs into the summary input, and +retains the newest complete pair. This replay diagnoses compaction planning; +it does not by itself prove the final legal report or completion contract. diff --git a/.omo/evidence/20260727-v24r13-capacity-aware-compaction/replay-preserved-case09-context.mjs b/.omo/evidence/20260727-v24r13-capacity-aware-compaction/replay-preserved-case09-context.mjs new file mode 100644 index 00000000..e42b7c90 --- /dev/null +++ b/.omo/evidence/20260727-v24r13-capacity-aware-compaction/replay-preserved-case09-context.mjs @@ -0,0 +1,93 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +const [runtimeRoot, sessionPath] = process.argv.slice(2); +if (!runtimeRoot || !sessionPath) { + throw new Error("usage: node replay-preserved-case09-context.mjs "); +} + +const importFromRuntime = (relativePath) => import(pathToFileURL(resolve(runtimeRoot, relativePath)).href); +const { CompactionEngine } = await importFromRuntime("dist/src/context/compaction/CompactionEngine.js"); +const { TokenBudgetManager } = await importFromRuntime("dist/src/context/budget/TokenBudgetManager.js"); +const { collectToolCallIds, collectToolResultIds } = await importFromRuntime( + "dist/src/context/compaction/toolPairIntegrity.js", +); + +const entries = (await readFile(sessionPath, "utf8")) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +const messages = entries.flatMap((entry) => { + if (entry.type === "accepted_input" && Array.isArray(entry.messages)) return entry.messages; + if ((entry.type === "assistant_message" || entry.type === "tool_result_message") && entry.message) { + return [entry.message]; + } + return []; +}); + +const requests = []; +const tokenBudget = new TokenBudgetManager(); +const engine = new CompactionEngine({ + provider: "preserved-replay", + model_: "preserved-replay", + tokenBudget, + maxProtectedPrefixTurns: 8, + model: { + async *stream(request) { + requests.push(request); + yield { + type: "text_delta", + text: "## Objective\nPreserved replay\n## Current State\nMeasured\n## Remaining\nNone\n## Files And Artifacts\nNone", + }; + }, + }, +}); + +const keepRatio = 0.35; +const result = await engine.run({ trigger: "auto", messages, keepTailRatio: keepRatio }); +const summarizedMessages = requests[0]?.messages.slice(0, -1) ?? []; +const totalTokens = tokenBudget.estimateMessagesTokens(messages); +const retainedTokens = tokenBudget.estimateMessagesTokens(result.messagesToKeep); +const summaryInputTokens = tokenBudget.estimateMessagesTokens(summarizedMessages); +const summarizedCalls = collectToolCallIds(summarizedMessages); +const summarizedResults = collectToolResultIds(summarizedMessages); +const retainedCalls = collectToolCallIds(result.messagesToKeep); +const retainedResults = collectToolResultIds(result.messagesToKeep); +const newestToolCallId = [...collectToolCallIds(messages)].at(-1); + +const countAgentCalls = (candidateMessages) => candidateMessages.reduce( + (count, message) => count + message.content.filter( + (block) => block.type === "tool_call" && ["agent", "Agent", "Task"].includes(block.name), + ).length, + 0, +); +const setsEqual = (left, right) => left.size === right.size && [...left].every((value) => right.has(value)); + +console.log(JSON.stringify({ + schemaVersion: 1, + sanitization: { + sourceContentEmitted: false, + promptContentEmitted: false, + reasoningEmitted: false, + credentialsEmitted: false, + toolCallIdsEmitted: false, + }, + totalMessages: messages.length, + totalTokens, + keepRatio, + keepTargetTokens: Math.floor(totalTokens * keepRatio), + retainedMessages: result.messagesToKeep.length, + retainedTokens, + retainedRatio: retainedTokens / totalTokens, + projectedPostTokens: result.postTokens, + summarizedMessages: summarizedMessages.length, + summaryInputTokens, + summarizedAgentCalls: countAgentCalls(summarizedMessages), + retainedAgentCalls: countAgentCalls(result.messagesToKeep), + summarizedPairsComplete: setsEqual(summarizedCalls, summarizedResults), + retainedPairsComplete: setsEqual(retainedCalls, retainedResults), + newestToolPairRetained: newestToolCallId !== undefined + && retainedCalls.has(newestToolCallId) + && retainedResults.has(newestToolCallId), +}, null, 2)); diff --git a/docs/PILOTDECK_CONVERGENCE_V24R13_CAPACITY_AWARE_COMPACTION.md b/docs/PILOTDECK_CONVERGENCE_V24R13_CAPACITY_AWARE_COMPACTION.md new file mode 100644 index 00000000..6c11b058 --- /dev/null +++ b/docs/PILOTDECK_CONVERGENCE_V24R13_CAPACITY_AWARE_COMPACTION.md @@ -0,0 +1,77 @@ +# PilotDeck Convergence V24R13: Capacity-Aware Compaction + +## Decision + +V24R13 fixes one domain-neutral full-compaction planning defect exposed by the +V24R12 Case 09 product gate. Exact post-compact retention is bounded by token +capacity instead of message and protected-frame counts alone. It does not +change Legal Coverage, domain validators, Progress Lease thresholds `8/2`, +Router, Memory, model, Skills, corpus, deadlines, tool execution, durable +receipts, or completion authority. + +The failed run completed full summaries, but the planner restored a 35% tail +selected by message count and up to eight additional protected frames without +accounting for their aggregate size. A few large, valid tool-result frames +therefore survived verbatim and left the projected request blocking. + +## Protocol + +Full compaction computes one exact-retention budget from the estimated token +size of the current canonical messages and the existing `keepTailRatio`. + +- Atomic frames remain the unit of selection, so tool calls and results are + never split. +- The newest complete frame is retained even when it alone exceeds the target. +- Newest tail frames consume the budget first. +- Eligible protected prefix frames may use only the remaining budget. +- Protected frames that do not fit are included in the same summary input; + they are not silently dropped. +- The boundary marker, summary, exact retained messages, attachments, hook + results, and trailing-user normalization keep their existing order. + +Protected context remains a preference for exact retention, not authority to +construct a request that still exceeds the model boundary. The existing +post-compact full-request evaluator remains the final fail-closed check. + +## Boundaries + +- No legal field, Case 09 identity, plugin name, or domain state is inspected. +- No blocking or warning threshold is relaxed. +- No additional model request, Lease grace, retry loop, or completion path is + introduced. +- No tool result is truncated or deleted by this planner. Non-retained frames + are visible to the summary model and represented by its handoff. +- Existing per-result persistence and references are unchanged. +- The count bound of eight protected prefix frames remains as a second ceiling; + the token budget is an additional aggregate ceiling. +- A single newest atomic frame can still exceed the target. The post-compact + evaluator must reject it if the complete request remains blocking. + +## Counterexamples + +Tests must prove that: + +- uneven, large protected frames cannot bypass the aggregate retention budget; +- protected frames excluded from exact retention are present in the summary + request; +- every summarized and retained tool call has its exact result pair; +- the newest atomic frame remains exact and in chronological order; +- ordinary small protected trajectories still retain useful recent frames; +- no-summarizable, warning-only, dynamic prompt, Progress Lease, and legal + contracts retain their existing behavior; and +- a sanitized Case 09-shaped replay changes a rejected blocking projection + into a non-blocking projection without changing the domain or Lease layers. + +## Verification Gate + +1. Record the uneven-frame counterexample failing on the V24R12 base. +2. Apply the smallest Core compaction-planning change and rerun focused context, + Agent runtime, Progress Lease, O1, Gateway, and Legal Coverage controls. +3. Replay the sanitized preserved-session shape and prove token reduction plus + pair integrity without copying source content or reasoning. +4. Run the complete repository suite and patch hygiene checks, then record + reviewer-readable QA evidence. +5. Push a stacked draft PR on V24R12. Create a fresh immutable campaign and run + Gate 0, paired smoke, Case 05, and full Case 09. V25 and the 85-case campaign + remain blocked until Case 09 completes with a substantive report and + completion proof. diff --git a/src/context/compaction/CompactionEngine.ts b/src/context/compaction/CompactionEngine.ts index ad37fef4..a0d3edd8 100644 --- a/src/context/compaction/CompactionEngine.ts +++ b/src/context/compaction/CompactionEngine.ts @@ -90,7 +90,7 @@ export type CompactionResult = { postTokens?: number; summaryMessage?: CanonicalMessage; boundaryMarker: CanonicalMessage; - /** Messages preserved verbatim across the boundary (kept tail). */ + /** Messages preserved verbatim across the boundary (bounded protected prefix plus tail). */ messagesToKeep: CanonicalMessage[]; /** Attachments to be re-injected post-compact (memory / hooks). */ attachments: CanonicalMessage[]; @@ -143,10 +143,11 @@ export class CompactionEngine { async run(input: CompactionInput): Promise { const preTokens = this.estimateMessages(input.messages); const tailRatio = clamp(input.keepTailRatio ?? DEFAULT_KEEP_TAIL_RATIO, 0, 1); - const keepCount = Math.max(1, Math.floor(input.messages.length * tailRatio)); + const keepTokenBudget = Math.max(1, Math.floor(preTokens * tailRatio)); const compactPlan = planFullCompactionMessages( input.messages, - keepCount, + keepTokenBudget, + (candidate) => this.estimateMessages(candidate), this.protectedToolNames, this.maxProtectedPrefixTurns, ); @@ -339,28 +340,47 @@ export function buildPostCompactMessages(result: CompactionResult): CanonicalMes function planFullCompactionMessages( messages: CanonicalMessage[], - keepCount: number, + keepTokenBudget: number, + estimateMessages: (messages: CanonicalMessage[]) => number, protectedToolNames?: Iterable, maxProtectedPrefixTurns = DEFAULT_MAX_PROTECTED_PREFIX_TURNS, ): { messagesToSummarize: CanonicalMessage[]; messagesToKeep: CanonicalMessage[] } { const frames = splitMessagesIntoAtomicFrames(messages); + const frameTokens = frames.map((frame) => estimateMessages(frame.messages)); let tailStart = frames.length; - let keptMessages = 0; - while (tailStart > 0 && keptMessages < keepCount) { - tailStart -= 1; - keptMessages += frames[tailStart]?.messages.length ?? 0; + let retainedTokens = 0; + while (tailStart > 0) { + const candidateIndex = tailStart - 1; + const candidateTokens = frameTokens[candidateIndex] ?? 0; + if (tailStart < frames.length && retainedTokens + candidateTokens > keepTokenBudget) { + break; + } + tailStart = candidateIndex; + retainedTokens += candidateTokens; + if (retainedTokens >= keepTokenBudget) { + break; + } } - const prefix = frames.slice(0, tailStart).flatMap((frame) => frame.messages); + const prefixFrames = frames.slice(0, tailStart); + const prefix = prefixFrames.flatMap((frame) => frame.messages); const tail = frames.slice(tailStart).flatMap((frame) => frame.messages); const protectedIndexes = collectProtectedFrameIndexes(prefix, { protectedToolNames, maxProtectedTurns: maxProtectedPrefixTurns, }); + const retainedProtectedIndexes = new Set(); + for (let index = prefixFrames.length - 1; index >= 0; index -= 1) { + if (!protectedIndexes.has(index)) continue; + const candidateTokens = frameTokens[index] ?? 0; + if (retainedTokens + candidateTokens > keepTokenBudget) continue; + retainedProtectedIndexes.add(index); + retainedTokens += candidateTokens; + } const protectedMessages: CanonicalMessage[] = []; const summaryCandidates: CanonicalMessage[] = []; - for (const frame of splitMessagesIntoAtomicFrames(prefix)) { - if (protectedIndexes.has(frame.index)) { + for (const frame of prefixFrames) { + if (retainedProtectedIndexes.has(frame.index)) { protectedMessages.push(...frame.messages); } else { summaryCandidates.push(...frame.messages); diff --git a/tests/context/auto-compaction-observability.spec.ts b/tests/context/auto-compaction-observability.spec.ts index 6a54fcf1..b3889dca 100644 --- a/tests/context/auto-compaction-observability.spec.ts +++ b/tests/context/auto-compaction-observability.spec.ts @@ -6,7 +6,7 @@ import { resolve } from "node:path"; import { DefaultContextRuntime } from "../../src/context/DefaultContextRuntime.js"; import { CompactionEngine } from "../../src/context/compaction/CompactionEngine.js"; import { collectToolCallIds, collectToolResultIds } from "../../src/context/compaction/toolPairIntegrity.js"; -import type { TokenBudgetSnapshot } from "../../src/context/budget/TokenBudgetManager.js"; +import { TokenBudgetManager, type TokenBudgetSnapshot } from "../../src/context/budget/TokenBudgetManager.js"; import type { CanonicalMessage, CanonicalModelRequest } from "../../src/model/index.js"; import { ProgressLease } from "../../src/agent/convergence/ProgressLease.js"; @@ -67,7 +67,7 @@ test("progress policy can force a full boundary while the token budget is still assert.equal(result.trace?.appliedTier, "full"); }); -test("full compaction distinguishes a protected prefix with no summarizable messages from model failure", async () => { +test("full compaction distinguishes a fully retained trajectory from model failure", async () => { let modelCalls = 0; const statuses: string[] = []; const engine = new CompactionEngine({ @@ -87,7 +87,7 @@ test("full compaction distinguishes a protected prefix with no summarizable mess const result = await engine.run({ trigger: "auto", messages: protectedAgentPairs(4), - keepTailRatio: 0.25, + keepTailRatio: 1, }); assert.equal(result.outcome, "no_summarizable_messages"); @@ -164,6 +164,88 @@ test("bounded protected-prefix retention summarizes old agent turns and preserve assert.deepEqual(collectToolCallIds(result.messagesToKeep), collectToolResultIds(result.messagesToKeep)); }); +test("full compaction shares one token budget across the exact tail and protected prefix", async () => { + const requests: CanonicalModelRequest[] = []; + const tokenBudget = new TokenBudgetManager(); + const engine = new CompactionEngine({ + provider: "test", + model_: "test", + tokenBudget, + maxProtectedPrefixTurns: 8, + model: { + async *stream(request) { + requests.push(request); + yield { + type: "text_delta" as const, + text: "## Objective\nO\n## Current State\nS\n## Remaining\nR\n## Files And Artifacts\nF", + }; + }, + }, + }); + const trajectory = unevenProtectedAgentTrajectory(10, 12_000); + const keepRatio = 0.35; + + const result = await engine.run({ + trigger: "auto", + messages: trajectory, + keepTailRatio: keepRatio, + }); + + assert.equal(result.outcome, "summarized"); + assert.equal(requests.length, 1); + const summarizedMessages = requests[0]!.messages.slice(0, -1); + const totalTokens = tokenBudget.estimateMessagesTokens(trajectory); + const retainedTokens = tokenBudget.estimateMessagesTokens(result.messagesToKeep); + assert.ok(retainedTokens <= Math.floor(totalTokens * keepRatio)); + assert.ok(collectToolCallIds(summarizedMessages).size > 0); + assert.deepEqual(collectToolCallIds(summarizedMessages), collectToolResultIds(summarizedMessages)); + assert.deepEqual(collectToolCallIds(result.messagesToKeep), collectToolResultIds(result.messagesToKeep)); + assert.ok(collectToolCallIds(result.messagesToKeep).has("agent-9")); +}); + +test("full compaction keeps one oversized newest atomic frame intact", async () => { + const requests: CanonicalModelRequest[] = []; + const estimateMessages = (candidate: CanonicalMessage[]) => Buffer.byteLength(JSON.stringify(candidate), "utf8"); + const engine = new CompactionEngine({ + provider: "test", + model_: "test", + tokenAccounting: { estimateMessages } as never, + maxProtectedPrefixTurns: 0, + model: { + async *stream(request) { + requests.push(request); + yield { type: "text_delta" as const, text: "oversized-newest summary" }; + }, + }, + }); + const trajectory = [ + ...singlePromptToolTrajectory(3), + { + role: "assistant" as const, + content: [{ type: "tool_call" as const, id: "newest", name: "read_file", input: {} }], + }, + { + role: "user" as const, + content: [{ + type: "tool_result" as const, + toolCallId: "newest", + content: [{ type: "text" as const, text: "x".repeat(8_000) }], + }], + }, + ]; + + const result = await engine.run({ trigger: "auto", messages: trajectory, keepTailRatio: 0.05 }); + const summarizedMessages = requests[0]!.messages.slice(0, -1); + + assert.ok( + estimateMessages(result.messagesToKeep) + > Math.floor(estimateMessages(trajectory) * 0.05), + ); + assert.deepEqual(collectToolCallIds(result.messagesToKeep), new Set(["newest"])); + assert.deepEqual(collectToolResultIds(result.messagesToKeep), new Set(["newest"])); + assert.deepEqual(collectToolCallIds(summarizedMessages), collectToolResultIds(summarizedMessages)); +}); + test("full compaction summarizes a real-shaped single-prompt Agent trajectory", async () => { const requests: CanonicalModelRequest[] = []; const engine = new CompactionEngine({ @@ -194,7 +276,7 @@ test("full compaction summarizes a real-shaped single-prompt Agent trajectory", assert.deepEqual(collectToolCallIds(result.messagesToKeep), collectToolResultIds(result.messagesToKeep)); }); -test("full compaction aligns a message-count tail boundary to complete tool turns", async () => { +test("full compaction aligns a token-budget tail boundary to complete tool turns", async () => { const requests: CanonicalModelRequest[] = []; const engine = new CompactionEngine({ provider: "test", @@ -341,6 +423,29 @@ function singlePromptAgentTrajectory(count: number): CanonicalMessage[] { ]; } +function unevenProtectedAgentTrajectory(count: number, resultCharacters: number): CanonicalMessage[] { + return [ + { role: "user", content: [{ type: "text", text: "one bounded task with several large results" }] }, + ...Array.from({ length: count }, (_, index): CanonicalMessage[] => { + const id = `agent-${index}`; + return [ + { + role: "assistant", + content: [{ type: "tool_call", id, name: "agent", input: { task: `task-${index}` } }], + }, + { + role: "user", + content: [{ + type: "tool_result", + toolCallId: id, + content: [{ type: "text", text: String(index).repeat(resultCharacters) }], + }], + }, + ]; + }).flat(), + ]; +} + function singlePromptToolTrajectory(count: number): CanonicalMessage[] { return [ { role: "user", content: [{ type: "text", text: "one long tool task" }] },