From cf1605bdff2f52abffc4ef58a0c28c7178baff89 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 00:14:18 +0100 Subject: [PATCH 01/27] feat(analysis): index direct task and steering instructions as citable spans The initial task message and later user steering become addressable instruction units (TASK-1 / STEER-N) with source kind, turn, a direct-task/direct-steering authority tier, and a lifetime closed only by an explicit revision signal. Spans render in the analysis prompt by turn reference, never by duplicating turn text; only the authoritative envelope of a user message indexes (fenced blocks and blockquote lines are structurally excluded, assistant/tool text never is). The fold validates cites against the spans each session issued and renders them apart from memory-instruction rows. ANALYSIS_INDEX_VERSION 3 -> 4. --- AGENTS.md | 5 + src/analyze.js | 21 ++- src/directives.js | 180 +++++++++++++++++++++++ src/distill.js | 6 + src/fold.js | 92 +++++++++++- src/prompts/analysis.md | 25 +++- src/proposal.js | 6 +- test/directives.test.js | 223 +++++++++++++++++++++++++++++ test/fixtures/directive-spans.json | 15 ++ 9 files changed, 564 insertions(+), 9 deletions(-) create mode 100644 src/directives.js create mode 100644 test/directives.test.js create mode 100644 test/fixtures/directive-spans.json diff --git a/AGENTS.md b/AGENTS.md index 22c14a5a..fb47e235 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,11 @@ Each owning file's comments and tests hold the detail; read them before touching - **Quotes must be found in the trace they cite** (`sanitizeEvidence` in `src/analyze.js`), so fake agents in tests must quote real session text. Bump `ANALYSIS_INDEX_VERSION` (`src/state.js`) for any change to what analysis accepts. +- **Direct task/steering instructions are addressable** (`src/directives.js`): the first substantive user turn is + `TASK-1`, later ones `STEER-`, each with `direct-task`/`direct-steering` authority and a lifetime from + its turn onward. Only the carved envelope indexes (fenced blocks and `>` quotes excluded); assistant/tool text + never indexes. Spans render by turn reference, never duplicated text; only ids plus metadata persist. The fold + has no directive rows: cites stay session authority, and recurrence reaches synthesis as ordinary gaps. - **Sampling is deterministic and sticky** (`src/sample.js`): a per-transcript hash of `transcriptIdentity` and `config.seed`, never `Math.random()` or an index/position-derived draw. - **Corpus mix is interactive vs non-interactive, never an unknown bucket** (`src/interaction.js`). diff --git a/src/analyze.js b/src/analyze.js index 0b532ed2..c2942efe 100644 --- a/src/analyze.js +++ b/src/analyze.js @@ -3,6 +3,7 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; import { extractJson, runModelCall, usageRecord } from "./acpx.js"; +import { directiveMetadata, extractDirectives, renderDirectiveIndex } from "./directives.js"; import { distill } from "./distill.js"; import { classifyInteraction } from "./interaction.js"; import { getAdapter, readTranscript } from "./discovery/index.js"; @@ -149,7 +150,7 @@ function foldSpace(text) { * `quotesNotInTrace` counts what the trace check rejected, so a run whose analysis model * paraphrases everything reads as that rather than as a clean repo. */ -export function sanitizeEvidence(parsed, memoryFile = null, trace = null) { +export function sanitizeEvidence(parsed, memoryFile = null, trace = null, directiveIds = null) { const clean = { positive: [], negative: [], @@ -160,6 +161,11 @@ export function sanitizeEvidence(parsed, memoryFile = null, trace = null) { if (!parsed || typeof parsed !== "object") return clean; const validInstructions = memoryFile ? new Set(instructionUnits(memoryFile).map((unit) => unit.id)) : null; + // Direct task/steering spans are session-scoped instruction sources alongside the + // memory index; a citation of one is addressable evidence, not a stale reference. + if (validInstructions && Array.isArray(directiveIds)) { + for (const id of directiveIds) validInstructions.add(id); + } const foldedTrace = typeof trace === "string" && !clean.usedRawTranscript ? foldSpace(trace) : null; const hasQuote = (item) => { if (typeof item?.quote !== "string" || item.quote.trim().length < 8) return false; @@ -299,11 +305,13 @@ async function analyzeOne({ fs.writeFileSync(rawFile, `${lines.join("\n")}\n`, { mode: 0o600, flag: "wx" }); } + const directives = extractDirectives(distilled.turns); const prompt = renderPrompt("analysis", { MEMORY_PATH: memoryFile.path, INSTRUCTION_INDEX: renderInstructionIndex(memoryFile), ALSO_LOADED: alsoLoaded, SKILLS: skillIndex, + DIRECTIVES: renderDirectiveIndex(directives), OPEN_GAPS: openGapIndex, TRACE: distilled.trace, }); @@ -336,9 +344,18 @@ async function analyzeOne({ throw new Error("analysis returned no parseable JSON"); } + const evidence = sanitizeEvidence( + parsed, + memoryFile, + distilled.trace, + directives.map((span) => span.id), + ); return { status: "ok", - evidence: sanitizeEvidence(parsed, memoryFile, distilled.trace), + // The record carries this session's directive span metadata - ids, turn, authority + // and lifetime, never envelope text - so a cited TASK-1 / STEER-N id stays + // resolvable in persisted evidence (see `src/directives.js`). + evidence: { ...evidence, directives: directiveMetadata(directives) }, usage: usageRecord(ranWith, result), distilled, }; diff --git a/src/directives.js b/src/directives.js new file mode 100644 index 00000000..5169af78 --- /dev/null +++ b/src/directives.js @@ -0,0 +1,180 @@ +/** + * Direct task and steering instructions as addressable instruction sources. + * + * Backpass indexes project memory files as instruction sources, but in real sessions + * much of what governs agent behaviour comes straight from the user: the initial task + * message and later steering turns. Without ids those read only as undifferentiated + * transcript evidence, so analysis cannot say "the agent followed the user's direct + * instruction" apart from "the agent followed the memory file". This module makes each + * substantive user turn citable: + * + * - the first user turn's authoritative envelope is the task span (`TASK-1`) + * - each later substantive user turn is a steering span (`STEER-`) + * - every span carries source kind, turn, an authority tier distinct from project + * memory (`direct-task` / `direct-steering`), and a lifetime (from its turn + * onward, closed only by later steering with an explicit revision signal) + * + * Only the authoritative envelope of a user message becomes a span. Quoted transcripts + * (`>` blockquotes), pasted tool output and code (fenced blocks), retry reasons and + * planned actions embedded as quotes or pastes are structurally excluded: indexing + * them as authority would invert instructions that say to treat that text as + * untrusted evidence. Assistant messages and tool results never become spans. + * + * Spans are extracted from the distilled message turns, so turn numbers match the + * trace the analysis model reads. The prompt index renders them by reference (id plus + * a `see turn N` pointer), never by duplicating turn text: task and steering text is + * private by default, and only ids plus metadata (`directives` on the evidence + * record) ever persist - envelope text never leaves the in-memory prompt. + */ + +export const TASK_ID = "TASK-1"; +export const TASK_AUTHORITY = "direct-task"; +export const STEERING_AUTHORITY = "direct-steering"; + +/** Envelopes shorter than this carry no judgable instruction (acks, greetings). */ +export const MIN_ENVELOPE_WORDS = 3; + +/** + * A later steering envelope supersedes earlier spans only on an explicit revision + * signal - a structural admission that the user is revising, not merely adding. + * Deliberately narrow: a bare "instead" also appears in fresh instructions, and an + * over-eager supersede would silently close a task lifetime that still governs. + */ +const REVISION_SIGNAL = + /\b(actually|scratch that|forget (that|it|what i said)|ignore my (previous|last|earlier) (instructions?|message|request)|on second thought|change of plans?|disregard that|correction:)/i; + +const FENCE = /^\s*(```|~~~)/; +const QUOTE = /^\s*>/; + +/** A span id is stable by construction: kind plus the distilled turn it points at. */ +export function isDirectiveId(id) { + return id === TASK_ID || /^STEER-\d+$/.test(String(id || "")); +} + +function steeringId(turn) { + return `STEER-${turn}`; +} + +/** + * Split a user message into its authoritative envelope and the embedded untrusted + * blocks. Fenced code blocks (pasted tool output, code, retry logs) and blockquote + * lines (quoted transcripts, pasted replies, planned actions quoted back) are + * evidence-only, never authority. Returns the envelope plus what was excluded, so + * tests can prove a quoted transcript never becomes instruction text. + */ +export function carveEnvelope(text) { + const excluded = []; + const kept = []; + let inFence = false; + let fenceBuffer = []; + + const flushFence = () => { + if (fenceBuffer.length) excluded.push({ kind: "code-block", text: fenceBuffer.join("\n").trim() }); + fenceBuffer = []; + }; + + for (const line of String(text ?? "").split("\n")) { + if (FENCE.test(line)) { + if (inFence) { + fenceBuffer.push(line); + flushFence(); + } else { + fenceBuffer.push(line); + } + inFence = !inFence; + continue; + } + if (inFence) { + fenceBuffer.push(line); + continue; + } + if (QUOTE.test(line)) { + excluded.push({ kind: "blockquote", text: line.replace(/^\s*>\s?/, "").trim() }); + continue; + } + kept.push(line); + } + // An unclosed fence still withholds the rest: a pasted log without a closing + // marker is still a paste, not an instruction. + if (inFence) flushFence(); + + return { envelope: kept.join("\n").trim(), excluded }; +} + +function envelopeWords(envelope) { + return envelope.split(/\s+/).filter(Boolean).length; +} + +/** + * Index the substantive user turns of one session. `turns` are the distilled message + * turns (`{ turn, role, text }`) in trace order, so ids stay stable across runs of the + * same transcript and always point at text the model was actually sent. + */ +export function extractDirectives(turns) { + const spans = []; + const envelopes = new Map(); + let seenFirstUser = false; + for (const entry of Array.isArray(turns) ? turns : []) { + if (!entry || entry.role !== "user") continue; + const { envelope } = carveEnvelope(entry.text); + if (envelopeWords(envelope) < MIN_ENVELOPE_WORDS) continue; + const span = !seenFirstUser + ? { id: TASK_ID, kind: "task", turn: entry.turn, authority: TASK_AUTHORITY } + : { id: steeringId(entry.turn), kind: "steering", turn: entry.turn, authority: STEERING_AUTHORITY }; + seenFirstUser = true; + spans.push({ ...span, lifetime: { fromTurn: entry.turn, toTurn: null }, supersededBy: null }); + envelopes.set(entry.turn, envelope); + } + // Close lifetimes where a revision is detectable: each span is superseded by the + // next steering span whose envelope carries an explicit revision signal. + const revisers = spans.filter( + (span) => span.kind === "steering" && REVISION_SIGNAL.test(envelopes.get(span.turn) || ""), + ); + for (const span of spans) { + const superseder = revisers.find((other) => other.turn > span.turn); + if (superseder) { + span.supersededBy = superseder.id; + span.lifetime.toTurn = superseder.turn; + } + } + return spans; +} + +/** Metadata that may persist on an evidence record: ids and spans, never turn text. */ +export function directiveMetadata(spans) { + return (Array.isArray(spans) ? spans : []).map((span) => ({ + id: span.id, + kind: span.kind, + turn: span.turn, + authority: span.authority, + lifetime: { ...span.lifetime }, + supersededBy: span.supersededBy, + })); +} + +/** + * The analysis-prompt index. By reference only: each entry names the turn whose text + * is already in the trace, so no user-turn text is duplicated into the prompt. + */ +export function renderDirectiveIndex(spans, { elided = false } = {}) { + const list = Array.isArray(spans) ? spans : []; + if (!list.length) return "(none - no substantive user instruction found in the trace)"; + const lines = list.map((span) => { + const life = + span.lifetime?.toTurn != null + ? `lifetime turns ${span.lifetime.fromTurn}-${span.lifetime.toTurn - 1}` + : `lifetime turns ${span.lifetime?.fromTurn ?? span.turn}+`; + const superseded = span.supersededBy ? `, superseded by [${span.supersededBy}] (explicit revision)` : ""; + return ( + `[${span.id}] ${span.kind} · turn ${span.turn} · authority ${span.authority} · ` + + `${life}${superseded} - see turn ${span.turn} in the trace` + ); + }); + if (elided) { + lines.push( + "The trace middle was elided; a cited turn may sit in the elided span - " + + "open the raw transcript when a claim needs its text.", + ); + } + return lines.join("\n"); +} diff --git a/src/distill.js b/src/distill.js index b890d478..444d578f 100644 --- a/src/distill.js +++ b/src/distill.js @@ -100,6 +100,10 @@ export function isBoilerplate(text) { export function distill(events, meta, options = {}) { const maxTraceTokens = options.maxTraceTokens ?? 12000; const lines = []; + // Message turns in trace order, with the same turn numbers the trace prints: the + // single source of truth for anything that points at a turn (directive spans in + // `src/directives.js`). Text is the redacted form the model actually sees. + const turns = []; let userTurns = 0; let assistantTurns = 0; let toolCalls = 0; @@ -113,6 +117,7 @@ export function distill(events, meta, options = {}) { turn += 1; if (event.role === "user") userTurns += 1; else assistantTurns += 1; + turns.push({ turn, role: event.role, text }); lines.push(`### turn ${turn} · ${event.role}`); lines.push(text); lines.push(""); @@ -152,6 +157,7 @@ export function distill(events, meta, options = {}) { return { trace, + turns, stats: { userTurns, assistantTurns, diff --git a/src/fold.js b/src/fold.js index e03b783e..d44930b6 100644 --- a/src/fold.js +++ b/src/fold.js @@ -2,6 +2,7 @@ import fs from "node:fs"; import { loadConfig } from "./config.js"; import { classifyInteraction, INTERACTIVE, NON_INTERACTIVE } from "./interaction.js"; +import { isDirectiveId } from "./directives.js"; import { findInstructionUnit, instructionUnits, resolveMemoryFiles, similarity } from "./memory.js"; import { rootOwnsGap } from "./nested.js"; import { @@ -131,6 +132,41 @@ export function foldEvidence( ]); const recordSources = issuedSources.slice(0, usable.length); const observationSources = issuedSources.slice(usable.length); + // Declared direct-instruction spans per session: only a TASK-/STEER- id this + // session's own analysis was issued counts as a directive citation. Anything else + // in that shape is a hallucinated id and keeps today's stale-reference row. + const declaredDirectives = new Map(); + for (const record of usable) { + const sessionIdentity = record.transcript.identity || record.transcript.id; + if (!Array.isArray(record.directives)) continue; + const byId = new Map(); + for (const span of record.directives) { + if (span && isDirectiveId(span.id)) byId.set(span.id, span); + } + if (byId.size) declaredDirectives.set(sessionIdentity, byId); + } + // Session-scoped directive cites: one row per (session, span), never merged + // across sessions and never mixed into memory-instruction relevance. + const directiveBuckets = new Map(); + const touchDirective = (sessionIdentity, source, span) => { + const key = `${sessionIdentity} ${span.id}`; + if (!directiveBuckets.has(key)) { + directiveBuckets.set(key, { + id: span.id, + kind: span.kind ?? null, + turn: span.turn ?? null, + authority: span.authority ?? null, + lifetime: span.lifetime ?? null, + supersededBy: span.supersededBy ?? null, + source, + sessionId: sessionIdentity, + positive: 0, + negative: 0, + quotes: [], + }); + } + return directiveBuckets.get(key); + }; for (const [index, record] of usable.entries()) { const sessionIdentity = record.transcript.identity || record.transcript.id; const corroborationIdentity = corroborationIdentityOf(record.transcript); @@ -138,9 +174,27 @@ export function foldEvidence( const source = recordSources[index]; sources.add(source); if (record.transcript.project) sourceProjects[source] = record.transcript.project; + const sessionIdentity = record.transcript.identity || record.transcript.id; + const declared = declaredDirectives.get(sessionIdentity); for (const polarity of ["positive", "negative"]) { for (const item of record[polarity] || []) { + const span = declared?.get(item.instruction); + if (span) { + const entry = touchDirective(sessionIdentity, source, span); + entry[polarity] += 1; + entry.quotes.push({ + polarity, + text: item.quote, + effect: item.effect, + moment: item.moment, + class: polarity === "negative" ? (item.class ?? null) : undefined, + source, + }); + if (polarity === "positive") positiveCount += 1; + else negativeCount += 1; + continue; + } const entry = touch(item.instruction); entry[polarity] += 1; const category = classifyInteraction(record.transcript); @@ -347,12 +401,18 @@ export function foldEvidence( droppedGapSingletons, orchestrationGapSightings, // The existing-instruction lane's candidate count: how many instructions the - // negatives land on. Display only; no gate reads it. - instructionsWithNegatives: instructionRows.filter((row) => row.negative > 0).length, + // negatives land on. Display only; no gate reads it. Directive-shaped ids are + // never existing memory instructions - declared ones have their own section, + // undeclared ones are stale references - so neither enters this lane. + instructionsWithNegatives: instructionRows.filter((row) => row.negative > 0 && !isDirectiveId(row.instruction)) + .length, usedRawTranscript: usedRawCount, crossSurfaceDuplicates: duplicates.length, }, instructions: instructionRows, + directives: [...directiveBuckets.values()] + .map((entry) => ({ ...entry, quotes: entry.quotes.slice(0, 6) })) + .sort((a, b) => String(a.source).localeCompare(String(b.source)) || (a.turn ?? 0) - (b.turn ?? 0)), sources: [...sources], sourceProjects, sourceObservers, @@ -674,6 +734,34 @@ function renderEvidence(summary, { includeReportOnly }) { } } + if (Array.isArray(summary.directives) && summary.directives.length) { + lines.push(""); + lines.push("### Direct task/steering instructions cited (session authority, not memory content)"); + lines.push( + "These ids name one session's own task message (TASK-1) or later steering (STEER-N). " + + "They explain behaviour the memory file did not steer. They are not memory content: " + + "never rewrite them, and a steering theme recurring across sessions is a gap signal, " + + "not an edit to these rows. Counts are cites within that session, not cross-session relevance.", + ); + for (const row of summary.directives) { + const life = + row.lifetime?.toTurn != null + ? `lifetime turns ${row.lifetime.fromTurn}-${row.lifetime.toTurn - 1}` + : `lifetime turns ${row.turn}+`; + const superseded = row.supersededBy ? ` superseded by [${row.supersededBy}]` : ""; + lines.push( + `- [${row.id}] ${row.kind ?? "steering"} · turn ${row.turn} · authority ${row.authority ?? "direct"} · ` + + `${life}${superseded} +${row.positive} -${row.negative} (from ${row.source})`, + ); + for (const quote of row.quotes.slice(0, 3)) { + const sign = quote.polarity === "negative" ? "-" : "+"; + const cls = quote.polarity === "negative" ? ` [${quote.class ?? "unclassified"}]` : ""; + const effect = quote.effect ? ` :: ${oneLine(quote.effect, 200)}` : ""; + lines.push(` ${sign}${cls} "${oneLine(quote.text)}"${effect} (${quote.source})`); + } + } + } + const representedOverlaps = new Set( summary.instructions.filter((row) => row.skillOverlap).map((row) => row.instruction), ); diff --git a/src/prompts/analysis.md b/src/prompts/analysis.md index 3bbda0ca..fa5811f6 100644 --- a/src/prompts/analysis.md +++ b/src/prompts/analysis.md @@ -6,7 +6,7 @@ have prevented. This is the loss signal for a backward pass over the memory file ## The memory file under audit: {{MEMORY_PATH}} -Each instruction has a stable id in [brackets]. Refer to instructions ONLY by these ids. +Each memory instruction has a stable id in [brackets]. Refer to memory instructions ONLY by these ids. {{INSTRUCTION_INDEX}}{{ALSO_LOADED}} @@ -20,6 +20,21 @@ only when you need its body to confirm the coverage. {{SKILLS}} +## Direct instructions in this session + +The user's own task message (TASK-1) and later steering (STEER-N) are addressable +instruction sources with an authority of their own, separate from the memory file. +Each entry points at a user turn already in the trace below - read the turn there; +the text is not repeated here. + +{{DIRECTIVES}} + +When the agent was clearly following one of these instead of (or as well as) a memory +instruction, cite its id in `instruction`. A mistake no memory instruction AND no +direct instruction covers is a gap; a direct instruction the agent ignored or violated +is a `negative` citing that id, not a gap. Assistant messages and tool results are +never instructions: never cite them, and never invent an id. + ## Gaps already on the books Earlier sessions reported these gaps; each has a stable id. If a gap you found is the @@ -73,9 +88,11 @@ Rules, in order of importance: 5. **Do not confabulate influence.** Only call something positive when the trace shows the agent doing the specific thing the instruction asks for. An outcome that would have happened anyway is not evidence. -6. `gaps` are mistakes NOT covered by any current instruction. If an instruction exists - and was ignored, that is `negative` with `class: "non-compliance"`, not a gap. A - mistake covered only by a SKILL's content is still a gap - skills have no +6. `gaps` are mistakes NOT covered by any current instruction, memory or direct. If a + memory instruction exists and was ignored, that is `negative` with + `class: "non-compliance"`, not a gap - and the same holds for a direct instruction + the agent ignored: cite its TASK-1 / STEER-N id as the `negative`'s instruction. + A mistake covered only by a SKILL's content is still a gap - skills have no instruction ids - but cite the skill in `coveredBySkill`: that is a failed trigger, and the fix is that skill's description, not new memory-file text. 7. `proposedInstruction` must be one imperative sentence, specific enough to act on and diff --git a/src/proposal.js b/src/proposal.js index 947d26dc..5db6d45d 100644 --- a/src/proposal.js +++ b/src/proposal.js @@ -1,6 +1,7 @@ import path from "node:path"; import { renderHunkLines } from "./diff.js"; +import { isDirectiveId } from "./directives.js"; import { normalizeSourceLabel } from "./gap-ledger.js"; import { mixFromCounts } from "./interaction.js"; import { memoryTextHash } from "./memory.js"; @@ -915,7 +916,10 @@ function funnelInstructionOutcome(summary, accepted, suppressed) { typeof summary.totals.reportOnlyByReason !== "object" ) return null; - const candidates = summary.instructions.filter((row) => row.negative > 0); + // Directive-shaped ids are never existing memory instructions (declared ones render + // in their own fold section; undeclared ones are stale references), so the + // existing-instruction lane of the funnel skips them. Display only; no gate reads it. + const candidates = summary.instructions.filter((row) => row.negative > 0 && !isDirectiveId(row.instruction)); const candidateIds = new Set(candidates.map((row) => row.instruction)); const acceptedIds = new Set(accepted.flatMap((edit) => edit.instructions || []).filter((id) => candidateIds.has(id))); const suppressedIds = new Set( diff --git a/test/directives.test.js b/test/directives.test.js new file mode 100644 index 00000000..69cb9c4b --- /dev/null +++ b/test/directives.test.js @@ -0,0 +1,223 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { distill } from "../src/distill.js"; +import { sanitizeEvidence } from "../src/analyze.js"; +import { foldEvidence, renderEvidenceForPrompt } from "../src/fold.js"; +import { parseMemoryUnits } from "../src/memory.js"; +import { estimateTokens } from "../src/tokens.js"; +import { + TASK_ID, + carveEnvelope, + directiveMetadata, + extractDirectives, + isDirectiveId, + renderDirectiveIndex, +} from "../src/directives.js"; +import * as claude from "../src/discovery/adapters/claude.js"; +import * as pi from "../src/discovery/adapters/pi.js"; + +const FIXTURES = path.dirname(fileURLToPath(import.meta.url)); +const META = { + id: "directive-demo", + harness: "claude", + association: { tier: 1, confidence: "exact" }, + rawPath: "/tmp/directive-spans.jsonl", +}; + +function fixtureEvents() { + return JSON.parse(fs.readFileSync(path.join(FIXTURES, "fixtures/directive-spans.json"), "utf8")); +} + +test("only the authoritative envelope becomes instruction text", () => { + const { envelope, excluded } = carveEnvelope(fixtureEvents()[0].text); + assert.match(envelope, /Migrate the auth module/); + assert.match(envelope, /Keep the public function names unchanged/); + assert.ok(!envelope.includes("previous assistant said"), "quoted transcript must not be authority"); + assert.ok(!envelope.includes("refreshSession"), "pasted tool output must not be authority"); + assert.deepEqual(excluded.map((entry) => entry.kind).sort(), ["blockquote", "blockquote", "code-block"]); +}); + +test("an unclosed fence still withholds the paste", () => { + const { envelope } = carveEnvelope("Do the migration.\n```\nError: boom"); + assert.equal(envelope, "Do the migration."); +}); + +test("the first substantive user turn is the task, later ones are steering", () => { + const { turns } = distill(fixtureEvents(), META); + const spans = extractDirectives(turns); + assert.deepEqual( + spans.map((span) => [span.id, span.kind, span.turn, span.authority]), + [ + [TASK_ID, "task", 1, "direct-task"], + ["STEER-4", "steering", 4, "direct-steering"], + ], + ); + // "thanks" is an ack, not an instruction; the assistant reply and tool result never index. + assert.ok(spans.every((span) => span.turn !== 2 && span.turn !== 3)); + assert.ok(isDirectiveId("TASK-1") && isDirectiveId("STEER-4")); + assert.ok(!isDirectiveId("AG-001") && !isDirectiveId("TASK-2") && !isDirectiveId("banana")); +}); + +test("an explicit revision closes earlier lifetimes and nothing else does", () => { + const { turns } = distill(fixtureEvents(), META); + const [task, steering] = extractDirectives(turns); + assert.equal(task.supersededBy, "STEER-4"); + assert.deepEqual(task.lifetime, { fromTurn: 1, toTurn: 4 }); + assert.equal(steering.supersededBy, null); + assert.deepEqual(steering.lifetime, { fromTurn: 4, toTurn: null }); + + const plain = extractDirectives([ + { turn: 1, role: "user", text: "Migrate the auth module to tokens." }, + { turn: 2, role: "assistant", text: "Done." }, + { turn: 3, role: "user", text: "Also update the changelog for this release." }, + ]); + assert.equal(plain[0].supersededBy, null, "an addition is not a revision"); + assert.deepEqual(plain[0].lifetime, { fromTurn: 1, toTurn: null }); +}); + +test("a message that is only quotes and pastes yields no span", () => { + const spans = extractDirectives([ + { turn: 1, role: "user", text: "> quoted transcript line\n\n```\npasted output\n```" }, + { turn: 2, role: "assistant", text: "Noted." }, + ]); + assert.deepEqual(spans, []); + assert.match(renderDirectiveIndex(spans), /\(none/); +}); + +test("the index renders by reference and never repeats turn text", () => { + const { turns, trace } = distill(fixtureEvents(), META); + const spans = extractDirectives(turns); + const section = renderDirectiveIndex(spans); + assert.match(section, /\[TASK-1\] task · turn 1 · authority direct-task/); + assert.match(section, /see turn 1 in the trace/); + assert.match(section, /superseded by \[STEER-4\]/); + for (const entry of turns.filter((candidate) => candidate.role === "user")) { + const { envelope } = carveEnvelope(entry.text); + if (envelope.length > 200) { + assert.ok(!section.includes(envelope), `turn ${entry.turn} text must not be duplicated`); + } + } + assert.ok(trace.includes("Migrate the auth module"), "the trace itself still carries the text"); +}); + +test("sanitizeEvidence accepts issued directive ids and drops the rest", () => { + const memoryFile = { units: parseMemoryUnits("# T\n\n- First rule\n") }; + const clean = sanitizeEvidence( + { + positive: [ + { instruction: "TASK-1", quote: "Migrate the auth module", effect: "followed the brief" }, + { instruction: "STEER-4", quote: "keep the old refresh-token flow", effect: "kept it" }, + { instruction: "AG-001", quote: "First rule", effect: "followed memory too" }, + { instruction: "AG-999", quote: "First rule", effect: "hallucinated memory id" }, + { instruction: "STEER-9", quote: "First rule", effect: "hallucinated steering id" }, + ], + usedRawTranscript: true, + }, + memoryFile, + null, + ["TASK-1", "STEER-4"], + ); + assert.deepEqual( + clean.positive.map((item) => item.instruction), + ["TASK-1", "STEER-4", "AG-001"], + ); +}); + +test("the fold keeps directive cites out of memory-instruction rows", () => { + const spans = extractDirectives(distill(fixtureEvents(), META).turns); + const meta = directiveMetadata(spans); + assert.ok( + JSON.stringify(meta).length < 800 && !JSON.stringify(meta).includes("Migrate"), + "persisted metadata carries ids and spans, never turn text", + ); + const memoryFile = { units: parseMemoryUnits("# T\n\n- First rule\n") }; + const base = (id) => ({ + status: "ok", + transcript: { id, harness: "claude", startedAt: Date.parse("2026-08-01T00:00:00Z") }, + positive: [], + negative: [], + gaps: [], + }); + const summary = foldEvidence( + [ + { + ...base("s1"), + directives: meta, + positive: [{ instruction: "TASK-1", quote: "Migrate the auth module", effect: "followed it" }], + negative: [{ instruction: "STEER-4", quote: "old refresh-token flow", effect: "dropped it", class: "harm" }], + }, + { + ...base("s2"), + directives: [], + negative: [{ instruction: "TASK-1", quote: "Migrate the auth module", effect: "no span issued here" }], + }, + ], + { memoryFile }, + ); + assert.equal(summary.directives.length, 2, "one row per session span, never merged across sessions"); + const task = summary.directives.find((row) => row.sessionId === "s1" && row.id === "TASK-1"); + assert.equal(task.positive, 1); + assert.equal(task.authority, "direct-task"); + assert.equal(task.supersededBy, "STEER-4"); + const stale = summary.instructions.find((row) => row.instruction === "TASK-1"); + assert.ok(stale && stale.known === false, "the undeclared TASK-1 cite keeps today's stale-reference row"); + assert.equal( + summary.totals.instructionsWithNegatives, + 0, + "directive negatives never enter the existing-instruction lane", + ); + const prompt = renderEvidenceForPrompt(summary); + assert.match(prompt, /Direct task\/steering instructions cited/); + assert.match(prompt, /\[TASK-1\] task · turn 1/); + assert.match(prompt, /never rewrite them/); +}); + +test("the directive index costs a fraction of duplicating the turns it points at", () => { + const cases = [ + { file: "claude-session.jsonl", read: (ref) => claude.read(ref) }, + { file: "pi-session.jsonl", read: (ref) => pi.read(ref) }, + ]; + for (const { file, read } of cases) { + const candidate = { key: file, path: path.join(FIXTURES, "fixtures", file) }; + const stat = fs.statSync(candidate.path); + candidate.mtimeMs = stat.mtimeMs; + candidate.bytes = stat.size; + const { events } = read({ path: candidate.path }); + const distilled = distill(events, { + ...META, + harness: candidate.path.includes("pi-") ? "pi" : "claude", + }); + const spans = extractDirectives(distilled.turns); + const section = renderDirectiveIndex(spans, { elided: distilled.stats.elided }); + const sectionTokens = estimateTokens(section); + // By-reference cost scales with the span count, never the turn length: one + // bounded entry per span, whatever the turn says. + if (spans.length) { + assert.ok( + sectionTokens <= spans.length * 50, + `${file}: ${spans.length} span(s) must cost at most ~50 tok each, got ${sectionTokens} tok`, + ); + } + console.log( + `directives ${file}: +${sectionTokens} tok index for ${spans.length} span(s) ` + + `on ${estimateTokens(distilled.trace)} tok trace`, + ); + } + // Where it matters - long task turns - by-reference beats duplicating the text. + const long = distill(fixtureEvents(), META); + const longSpans = extractDirectives(long.turns); + const longSection = estimateTokens(renderDirectiveIndex(longSpans)); + const duplicated = longSpans.reduce( + (sum, span) => sum + estimateTokens(long.turns.find((entry) => entry.turn === span.turn)?.text || ""), + 0, + ); + assert.ok( + longSection < duplicated, + `directive-spans fixture: index (${longSection} tok) must beat duplication (${duplicated} tok)`, + ); + console.log(`directives directive-spans.json: +${longSection} tok index vs ${duplicated} tok duplicated`); +}); diff --git a/test/fixtures/directive-spans.json b/test/fixtures/directive-spans.json new file mode 100644 index 00000000..abb7ba6e --- /dev/null +++ b/test/fixtures/directive-spans.json @@ -0,0 +1,15 @@ +[ + { + "kind": "message", + "role": "user", + "text": "Migrate the auth module to the new token API.\n\n> previous assistant said: I will keep using the old session API\n> tool output was: Error: 401 Unauthorized at auth.js:12\n\n```\nError: 401 Unauthorized\n at refreshSession (auth.js:12:8)\n```\n\nKeep the public function names unchanged." + }, + { "kind": "message", "role": "assistant", "text": "On it - migrating to the token API now." }, + { "kind": "tool", "name": "Bash", "input": { "command": "npm test" }, "result": "ok" }, + { "kind": "message", "role": "user", "text": "thanks" }, + { + "kind": "message", + "role": "user", + "text": "Actually, keep the old refresh-token flow as well for the mobile client." + } +] From 81353d03d65e0b4fe121dd4ac9cbc9302b0665eb Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 00:26:09 +0100 Subject: [PATCH 02/27] no-mistakes(review): keep directive-covered mistakes as gaps; drop supersede heuristic --- src/directives.js | 88 +++++++++-------------------------------- src/fold.js | 19 +++------ src/prompts/analysis.md | 21 +++++----- src/proposal.js | 6 +-- test/directives.test.js | 77 +++++++++++++++++++++++------------- 5 files changed, 85 insertions(+), 126 deletions(-) diff --git a/src/directives.js b/src/directives.js index 5169af78..c47295ee 100644 --- a/src/directives.js +++ b/src/directives.js @@ -11,8 +11,8 @@ * - the first user turn's authoritative envelope is the task span (`TASK-1`) * - each later substantive user turn is a steering span (`STEER-`) * - every span carries source kind, turn, an authority tier distinct from project - * memory (`direct-task` / `direct-steering`), and a lifetime (from its turn - * onward, closed only by later steering with an explicit revision signal) + * memory (`direct-task` / `direct-steering`), and a lifetime running from its turn + * onward * * Only the authoritative envelope of a user message becomes a span. Quoted transcripts * (`>` blockquotes), pasted tool output and code (fenced blocks), retry reasons and @@ -25,6 +25,9 @@ * a `see turn N` pointer), never by duplicating turn text: task and steering text is * private by default, and only ids plus metadata (`directives` on the evidence * record) ever persist - envelope text never leaves the in-memory prompt. + * + * A span is session authority, never durable project memory: a mistake only a direct + * instruction covered is still a gap, because the next session starts without it. */ export const TASK_ID = "TASK-1"; @@ -34,15 +37,6 @@ export const STEERING_AUTHORITY = "direct-steering"; /** Envelopes shorter than this carry no judgable instruction (acks, greetings). */ export const MIN_ENVELOPE_WORDS = 3; -/** - * A later steering envelope supersedes earlier spans only on an explicit revision - * signal - a structural admission that the user is revising, not merely adding. - * Deliberately narrow: a bare "instead" also appears in fresh instructions, and an - * over-eager supersede would silently close a task lifetime that still governs. - */ -const REVISION_SIGNAL = - /\b(actually|scratch that|forget (that|it|what i said)|ignore my (previous|last|earlier) (instructions?|message|request)|on second thought|change of plans?|disregard that|correction:)/i; - const FENCE = /^\s*(```|~~~)/; const QUOTE = /^\s*>/; @@ -56,49 +50,27 @@ function steeringId(turn) { } /** - * Split a user message into its authoritative envelope and the embedded untrusted - * blocks. Fenced code blocks (pasted tool output, code, retry logs) and blockquote - * lines (quoted transcripts, pasted replies, planned actions quoted back) are - * evidence-only, never authority. Returns the envelope plus what was excluded, so - * tests can prove a quoted transcript never becomes instruction text. + * The authoritative envelope of a user message. Fenced code blocks (pasted tool output, + * code, retry logs) and blockquote lines (quoted transcripts, pasted replies, planned + * actions quoted back) are evidence-only, never authority, so they are stripped. */ export function carveEnvelope(text) { - const excluded = []; const kept = []; let inFence = false; - let fenceBuffer = []; - - const flushFence = () => { - if (fenceBuffer.length) excluded.push({ kind: "code-block", text: fenceBuffer.join("\n").trim() }); - fenceBuffer = []; - }; for (const line of String(text ?? "").split("\n")) { if (FENCE.test(line)) { - if (inFence) { - fenceBuffer.push(line); - flushFence(); - } else { - fenceBuffer.push(line); - } inFence = !inFence; continue; } - if (inFence) { - fenceBuffer.push(line); - continue; - } - if (QUOTE.test(line)) { - excluded.push({ kind: "blockquote", text: line.replace(/^\s*>\s?/, "").trim() }); - continue; - } + // An unclosed fence still withholds the rest: a pasted log without a closing + // marker is still a paste, not an instruction. + if (inFence) continue; + if (QUOTE.test(line)) continue; kept.push(line); } - // An unclosed fence still withholds the rest: a pasted log without a closing - // marker is still a paste, not an instruction. - if (inFence) flushFence(); - return { envelope: kept.join("\n").trim(), excluded }; + return kept.join("\n").trim(); } function envelopeWords(envelope) { @@ -112,30 +84,15 @@ function envelopeWords(envelope) { */ export function extractDirectives(turns) { const spans = []; - const envelopes = new Map(); let seenFirstUser = false; for (const entry of Array.isArray(turns) ? turns : []) { if (!entry || entry.role !== "user") continue; - const { envelope } = carveEnvelope(entry.text); - if (envelopeWords(envelope) < MIN_ENVELOPE_WORDS) continue; + if (envelopeWords(carveEnvelope(entry.text)) < MIN_ENVELOPE_WORDS) continue; const span = !seenFirstUser ? { id: TASK_ID, kind: "task", turn: entry.turn, authority: TASK_AUTHORITY } : { id: steeringId(entry.turn), kind: "steering", turn: entry.turn, authority: STEERING_AUTHORITY }; seenFirstUser = true; - spans.push({ ...span, lifetime: { fromTurn: entry.turn, toTurn: null }, supersededBy: null }); - envelopes.set(entry.turn, envelope); - } - // Close lifetimes where a revision is detectable: each span is superseded by the - // next steering span whose envelope carries an explicit revision signal. - const revisers = spans.filter( - (span) => span.kind === "steering" && REVISION_SIGNAL.test(envelopes.get(span.turn) || ""), - ); - for (const span of spans) { - const superseder = revisers.find((other) => other.turn > span.turn); - if (superseder) { - span.supersededBy = superseder.id; - span.lifetime.toTurn = superseder.turn; - } + spans.push({ ...span, lifetime: { fromTurn: entry.turn, toTurn: null } }); } return spans; } @@ -148,7 +105,6 @@ export function directiveMetadata(spans) { turn: span.turn, authority: span.authority, lifetime: { ...span.lifetime }, - supersededBy: span.supersededBy, })); } @@ -159,17 +115,11 @@ export function directiveMetadata(spans) { export function renderDirectiveIndex(spans, { elided = false } = {}) { const list = Array.isArray(spans) ? spans : []; if (!list.length) return "(none - no substantive user instruction found in the trace)"; - const lines = list.map((span) => { - const life = - span.lifetime?.toTurn != null - ? `lifetime turns ${span.lifetime.fromTurn}-${span.lifetime.toTurn - 1}` - : `lifetime turns ${span.lifetime?.fromTurn ?? span.turn}+`; - const superseded = span.supersededBy ? `, superseded by [${span.supersededBy}] (explicit revision)` : ""; - return ( + const lines = list.map( + (span) => `[${span.id}] ${span.kind} · turn ${span.turn} · authority ${span.authority} · ` + - `${life}${superseded} - see turn ${span.turn} in the trace` - ); - }); + `lifetime turns ${span.lifetime?.fromTurn ?? span.turn}+ - see turn ${span.turn} in the trace`, + ); if (elided) { lines.push( "The trace middle was elided; a cited turn may sit in the elided span - " + diff --git a/src/fold.js b/src/fold.js index d44930b6..47df423b 100644 --- a/src/fold.js +++ b/src/fold.js @@ -133,8 +133,7 @@ export function foldEvidence( const recordSources = issuedSources.slice(0, usable.length); const observationSources = issuedSources.slice(usable.length); // Declared direct-instruction spans per session: only a TASK-/STEER- id this - // session's own analysis was issued counts as a directive citation. Anything else - // in that shape is a hallucinated id and keeps today's stale-reference row. + // session's own analysis was issued counts as a directive citation. const declaredDirectives = new Map(); for (const record of usable) { const sessionIdentity = record.transcript.identity || record.transcript.id; @@ -157,7 +156,6 @@ export function foldEvidence( turn: span.turn ?? null, authority: span.authority ?? null, lifetime: span.lifetime ?? null, - supersededBy: span.supersededBy ?? null, source, sessionId: sessionIdentity, positive: 0, @@ -401,11 +399,8 @@ export function foldEvidence( droppedGapSingletons, orchestrationGapSightings, // The existing-instruction lane's candidate count: how many instructions the - // negatives land on. Display only; no gate reads it. Directive-shaped ids are - // never existing memory instructions - declared ones have their own section, - // undeclared ones are stale references - so neither enters this lane. - instructionsWithNegatives: instructionRows.filter((row) => row.negative > 0 && !isDirectiveId(row.instruction)) - .length, + // negatives land on. Display only; no gate reads it. + instructionsWithNegatives: instructionRows.filter((row) => row.negative > 0).length, usedRawTranscript: usedRawCount, crossSurfaceDuplicates: duplicates.length, }, @@ -744,14 +739,10 @@ function renderEvidence(summary, { includeReportOnly }) { "not an edit to these rows. Counts are cites within that session, not cross-session relevance.", ); for (const row of summary.directives) { - const life = - row.lifetime?.toTurn != null - ? `lifetime turns ${row.lifetime.fromTurn}-${row.lifetime.toTurn - 1}` - : `lifetime turns ${row.turn}+`; - const superseded = row.supersededBy ? ` superseded by [${row.supersededBy}]` : ""; lines.push( `- [${row.id}] ${row.kind ?? "steering"} · turn ${row.turn} · authority ${row.authority ?? "direct"} · ` + - `${life}${superseded} +${row.positive} -${row.negative} (from ${row.source})`, + `lifetime turns ${row.lifetime?.fromTurn ?? row.turn}+ ` + + `+${row.positive} -${row.negative} (from ${row.source})`, ); for (const quote of row.quotes.slice(0, 3)) { const sign = quote.polarity === "negative" ? "-" : "+"; diff --git a/src/prompts/analysis.md b/src/prompts/analysis.md index fa5811f6..db7ff824 100644 --- a/src/prompts/analysis.md +++ b/src/prompts/analysis.md @@ -30,10 +30,10 @@ the text is not repeated here. {{DIRECTIVES}} When the agent was clearly following one of these instead of (or as well as) a memory -instruction, cite its id in `instruction`. A mistake no memory instruction AND no -direct instruction covers is a gap; a direct instruction the agent ignored or violated -is a `negative` citing that id, not a gap. Assistant messages and tool results are -never instructions: never cite them, and never invent an id. +instruction, cite its id in `instruction`; a direct instruction the agent ignored or +violated is a `negative` citing that id. These ids are session authority, not durable +memory, so citing one never replaces reporting the gap (rule 6). Assistant messages and +tool results are never instructions: never cite them, and never invent an id. ## Gaps already on the books @@ -88,13 +88,14 @@ Rules, in order of importance: 5. **Do not confabulate influence.** Only call something positive when the trace shows the agent doing the specific thing the instruction asks for. An outcome that would have happened anyway is not evidence. -6. `gaps` are mistakes NOT covered by any current instruction, memory or direct. If a - memory instruction exists and was ignored, that is `negative` with - `class: "non-compliance"`, not a gap - and the same holds for a direct instruction - the agent ignored: cite its TASK-1 / STEER-N id as the `negative`'s instruction. - A mistake covered only by a SKILL's content is still a gap - skills have no +6. `gaps` are mistakes NOT covered by a memory instruction. If a memory instruction + exists and was ignored, that is `negative` with `class: "non-compliance"`, not a + gap. A mistake covered only by a SKILL's content is still a gap - skills have no instruction ids - but cite the skill in `coveredBySkill`: that is a failed trigger, - and the fix is that skill's description, not new memory-file text. + and the fix is that skill's description, not new memory-file text. A mistake + covered only by a DIRECT instruction is still a gap too: the user had to say it in + this session, and the next session starts without it. Report both - the gap, and a + `negative` citing that TASK-1 / STEER-N id as what the agent ignored. 7. `proposedInstruction` must be one imperative sentence, specific enough to act on and general enough to apply beyond this one session. 8. An empty array is a valid and useful answer. Report nothing rather than something weak. diff --git a/src/proposal.js b/src/proposal.js index 5db6d45d..947d26dc 100644 --- a/src/proposal.js +++ b/src/proposal.js @@ -1,7 +1,6 @@ import path from "node:path"; import { renderHunkLines } from "./diff.js"; -import { isDirectiveId } from "./directives.js"; import { normalizeSourceLabel } from "./gap-ledger.js"; import { mixFromCounts } from "./interaction.js"; import { memoryTextHash } from "./memory.js"; @@ -916,10 +915,7 @@ function funnelInstructionOutcome(summary, accepted, suppressed) { typeof summary.totals.reportOnlyByReason !== "object" ) return null; - // Directive-shaped ids are never existing memory instructions (declared ones render - // in their own fold section; undeclared ones are stale references), so the - // existing-instruction lane of the funnel skips them. Display only; no gate reads it. - const candidates = summary.instructions.filter((row) => row.negative > 0 && !isDirectiveId(row.instruction)); + const candidates = summary.instructions.filter((row) => row.negative > 0); const candidateIds = new Set(candidates.map((row) => row.instruction)); const acceptedIds = new Set(accepted.flatMap((edit) => edit.instructions || []).filter((id) => candidateIds.has(id))); const suppressedIds = new Set( diff --git a/test/directives.test.js b/test/directives.test.js index 69cb9c4b..70bc9272 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -33,17 +33,16 @@ function fixtureEvents() { } test("only the authoritative envelope becomes instruction text", () => { - const { envelope, excluded } = carveEnvelope(fixtureEvents()[0].text); + const envelope = carveEnvelope(fixtureEvents()[0].text); assert.match(envelope, /Migrate the auth module/); assert.match(envelope, /Keep the public function names unchanged/); assert.ok(!envelope.includes("previous assistant said"), "quoted transcript must not be authority"); + assert.ok(!envelope.includes("401 Unauthorized"), "quoted tool output must not be authority"); assert.ok(!envelope.includes("refreshSession"), "pasted tool output must not be authority"); - assert.deepEqual(excluded.map((entry) => entry.kind).sort(), ["blockquote", "blockquote", "code-block"]); }); test("an unclosed fence still withholds the paste", () => { - const { envelope } = carveEnvelope("Do the migration.\n```\nError: boom"); - assert.equal(envelope, "Do the migration."); + assert.equal(carveEnvelope("Do the migration.\n```\nError: boom"), "Do the migration."); }); test("the first substantive user turn is the task, later ones are steering", () => { @@ -62,21 +61,11 @@ test("the first substantive user turn is the task, later ones are steering", () assert.ok(!isDirectiveId("AG-001") && !isDirectiveId("TASK-2") && !isDirectiveId("banana")); }); -test("an explicit revision closes earlier lifetimes and nothing else does", () => { +test("every lifetime runs from its own turn onward; later steering never closes it", () => { const { turns } = distill(fixtureEvents(), META); const [task, steering] = extractDirectives(turns); - assert.equal(task.supersededBy, "STEER-4"); - assert.deepEqual(task.lifetime, { fromTurn: 1, toTurn: 4 }); - assert.equal(steering.supersededBy, null); + assert.deepEqual(task.lifetime, { fromTurn: 1, toTurn: null }); assert.deepEqual(steering.lifetime, { fromTurn: 4, toTurn: null }); - - const plain = extractDirectives([ - { turn: 1, role: "user", text: "Migrate the auth module to tokens." }, - { turn: 2, role: "assistant", text: "Done." }, - { turn: 3, role: "user", text: "Also update the changelog for this release." }, - ]); - assert.equal(plain[0].supersededBy, null, "an addition is not a revision"); - assert.deepEqual(plain[0].lifetime, { fromTurn: 1, toTurn: null }); }); test("a message that is only quotes and pastes yields no span", () => { @@ -94,12 +83,12 @@ test("the index renders by reference and never repeats turn text", () => { const section = renderDirectiveIndex(spans); assert.match(section, /\[TASK-1\] task · turn 1 · authority direct-task/); assert.match(section, /see turn 1 in the trace/); - assert.match(section, /superseded by \[STEER-4\]/); + assert.match(section, /lifetime turns 1\+/); for (const entry of turns.filter((candidate) => candidate.role === "user")) { - const { envelope } = carveEnvelope(entry.text); - if (envelope.length > 200) { - assert.ok(!section.includes(envelope), `turn ${entry.turn} text must not be duplicated`); - } + assert.ok( + !section.includes(carveEnvelope(entry.text)), + `turn ${entry.turn} text must not be duplicated`, + ); } assert.ok(trace.includes("Migrate the auth module"), "the trace itself still carries the text"); }); @@ -152,23 +141,25 @@ test("the fold keeps directive cites out of memory-instruction rows", () => { }, { ...base("s2"), - directives: [], - negative: [{ instruction: "TASK-1", quote: "Migrate the auth module", effect: "no span issued here" }], + directives: meta, + negative: [{ instruction: "STEER-4", quote: "old refresh-token flow", effect: "dropped it again" }], }, ], { memoryFile }, ); - assert.equal(summary.directives.length, 2, "one row per session span, never merged across sessions"); + assert.equal(summary.directives.length, 3, "one row per session span, never merged across sessions"); const task = summary.directives.find((row) => row.sessionId === "s1" && row.id === "TASK-1"); assert.equal(task.positive, 1); assert.equal(task.authority, "direct-task"); - assert.equal(task.supersededBy, "STEER-4"); - const stale = summary.instructions.find((row) => row.instruction === "TASK-1"); - assert.ok(stale && stale.known === false, "the undeclared TASK-1 cite keeps today's stale-reference row"); + assert.deepEqual(task.lifetime, { fromTurn: 1, toTurn: null }); + assert.ok( + !summary.instructions.some((row) => isDirectiveId(row.instruction)), + "a directive cite never becomes a memory-instruction row", + ); assert.equal( summary.totals.instructionsWithNegatives, 0, - "directive negatives never enter the existing-instruction lane", + "no memory instruction drew a negative here", ); const prompt = renderEvidenceForPrompt(summary); assert.match(prompt, /Direct task\/steering instructions cited/); @@ -221,3 +212,33 @@ test("the directive index costs a fraction of duplicating the turns it points at ); console.log(`directives directive-spans.json: +${longSection} tok index vs ${duplicated} tok duplicated`); }); + +test("the same steer ignored in two sessions still clusters into a proposal-eligible gap", () => { + const meta = directiveMetadata(extractDirectives(distill(fixtureEvents(), META).turns)); + const memoryFile = { units: parseMemoryUnits("# T\n\n- First rule\n") }; + const session = (id) => ({ + status: "ok", + transcript: { id, harness: "claude", startedAt: Date.parse("2026-08-01T00:00:00Z") }, + directives: meta, + positive: [], + // The user had to steer it in this session, and the agent ignored the steer: a + // directive negative for provenance, and a gap because a directive is not memory. + negative: [ + { instruction: "STEER-4", quote: "old refresh-token flow", effect: "dropped it", class: "non-compliance" }, + ], + gaps: [ + { + mistake: "dropped the refresh-token flow the user asked to keep", + proposedInstruction: "Keep the old refresh-token flow when migrating auth.", + recurrenceRisk: "high", + domain: "project", + quote: "old refresh-token flow", + }, + ], + }); + const summary = foldEvidence([session("s1"), session("s2")], { memoryFile, minGapEvidence: 2 }); + assert.equal(summary.gaps.length, 1, "the recurring steer is one proposal-eligible gap cluster"); + assert.equal(summary.gaps[0].sessions, 2); + assert.equal(summary.totals.gapSightings, 2); + assert.equal(summary.directives.length, 2, "the steer keeps its per-session provenance rows"); +}); From 944908414b9b30e6bf5254f9bfda6e19ce62b66b Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 00:47:01 +0100 Subject: [PATCH 03/27] no-mistakes(review): keep directive section out of synthesis prompt --- src/fold.js | 8 ++++---- test/directives.test.js | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/fold.js b/src/fold.js index 47df423b..24bf89e3 100644 --- a/src/fold.js +++ b/src/fold.js @@ -729,14 +729,14 @@ function renderEvidence(summary, { includeReportOnly }) { } } - if (Array.isArray(summary.directives) && summary.directives.length) { + if (includeReportOnly && summary.directives?.length) { lines.push(""); lines.push("### Direct task/steering instructions cited (session authority, not memory content)"); lines.push( "These ids name one session's own task message (TASK-1) or later steering (STEER-N). " + - "They explain behaviour the memory file did not steer. They are not memory content: " + - "never rewrite them, and a steering theme recurring across sessions is a gap signal, " + - "not an edit to these rows. Counts are cites within that session, not cross-session relevance.", + "They explain behaviour the memory file did not steer. Diagnostic only: counts are cites " + + "within that session, not cross-session relevance, and a recurring steering theme reaches " + + "synthesis as a gap sighting instead.", ); for (const row of summary.directives) { lines.push( diff --git a/test/directives.test.js b/test/directives.test.js index 70bc9272..b8d8d05a 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -6,7 +6,7 @@ import { fileURLToPath } from "node:url"; import { distill } from "../src/distill.js"; import { sanitizeEvidence } from "../src/analyze.js"; -import { foldEvidence, renderEvidenceForPrompt } from "../src/fold.js"; +import { foldEvidence, renderEvidenceForPrompt, renderEvidenceReport } from "../src/fold.js"; import { parseMemoryUnits } from "../src/memory.js"; import { estimateTokens } from "../src/tokens.js"; import { @@ -161,10 +161,14 @@ test("the fold keeps directive cites out of memory-instruction rows", () => { 0, "no memory instruction drew a negative here", ); + const report = renderEvidenceReport(summary); + assert.match(report, /Direct task\/steering instructions cited/); + assert.match(report, /\[TASK-1\] task · turn 1/); const prompt = renderEvidenceForPrompt(summary); - assert.match(prompt, /Direct task\/steering instructions cited/); - assert.match(prompt, /\[TASK-1\] task · turn 1/); - assert.match(prompt, /never rewrite them/); + assert.ok( + !prompt.includes("Direct task/steering instructions cited") && !prompt.includes("[TASK-1]"), + "the directive section is diagnostic and never reaches the synthesis prompt", + ); }); test("the directive index costs a fraction of duplicating the turns it points at", () => { From 7f2c1fafa87e36e65952533e3156bfd2e8af1d88 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 00:55:47 +0100 Subject: [PATCH 04/27] no-mistakes(review): drop fold directive rows and their render section --- src/fold.js | 77 ++++------------------------------------- test/directives.test.js | 47 +++++-------------------- 2 files changed, 15 insertions(+), 109 deletions(-) diff --git a/src/fold.js b/src/fold.js index 24bf89e3..44c39b8c 100644 --- a/src/fold.js +++ b/src/fold.js @@ -132,39 +132,16 @@ export function foldEvidence( ]); const recordSources = issuedSources.slice(0, usable.length); const observationSources = issuedSources.slice(usable.length); - // Declared direct-instruction spans per session: only a TASK-/STEER- id this - // session's own analysis was issued counts as a directive citation. + // A direct task/steering cite is session authority, not memory content: the ids this + // session's own analysis was issued never become memory-instruction rows. The gap + // sightings carry the recurrence signal instead. const declaredDirectives = new Map(); for (const record of usable) { const sessionIdentity = record.transcript.identity || record.transcript.id; if (!Array.isArray(record.directives)) continue; - const byId = new Map(); - for (const span of record.directives) { - if (span && isDirectiveId(span.id)) byId.set(span.id, span); - } - if (byId.size) declaredDirectives.set(sessionIdentity, byId); + const ids = new Set(record.directives.filter((span) => span && isDirectiveId(span.id)).map((span) => span.id)); + if (ids.size) declaredDirectives.set(sessionIdentity, ids); } - // Session-scoped directive cites: one row per (session, span), never merged - // across sessions and never mixed into memory-instruction relevance. - const directiveBuckets = new Map(); - const touchDirective = (sessionIdentity, source, span) => { - const key = `${sessionIdentity} ${span.id}`; - if (!directiveBuckets.has(key)) { - directiveBuckets.set(key, { - id: span.id, - kind: span.kind ?? null, - turn: span.turn ?? null, - authority: span.authority ?? null, - lifetime: span.lifetime ?? null, - source, - sessionId: sessionIdentity, - positive: 0, - negative: 0, - quotes: [], - }); - } - return directiveBuckets.get(key); - }; for (const [index, record] of usable.entries()) { const sessionIdentity = record.transcript.identity || record.transcript.id; const corroborationIdentity = corroborationIdentityOf(record.transcript); @@ -177,22 +154,7 @@ export function foldEvidence( for (const polarity of ["positive", "negative"]) { for (const item of record[polarity] || []) { - const span = declared?.get(item.instruction); - if (span) { - const entry = touchDirective(sessionIdentity, source, span); - entry[polarity] += 1; - entry.quotes.push({ - polarity, - text: item.quote, - effect: item.effect, - moment: item.moment, - class: polarity === "negative" ? (item.class ?? null) : undefined, - source, - }); - if (polarity === "positive") positiveCount += 1; - else negativeCount += 1; - continue; - } + if (declared?.has(item.instruction)) continue; const entry = touch(item.instruction); entry[polarity] += 1; const category = classifyInteraction(record.transcript); @@ -405,9 +367,6 @@ export function foldEvidence( crossSurfaceDuplicates: duplicates.length, }, instructions: instructionRows, - directives: [...directiveBuckets.values()] - .map((entry) => ({ ...entry, quotes: entry.quotes.slice(0, 6) })) - .sort((a, b) => String(a.source).localeCompare(String(b.source)) || (a.turn ?? 0) - (b.turn ?? 0)), sources: [...sources], sourceProjects, sourceObservers, @@ -729,30 +688,6 @@ function renderEvidence(summary, { includeReportOnly }) { } } - if (includeReportOnly && summary.directives?.length) { - lines.push(""); - lines.push("### Direct task/steering instructions cited (session authority, not memory content)"); - lines.push( - "These ids name one session's own task message (TASK-1) or later steering (STEER-N). " + - "They explain behaviour the memory file did not steer. Diagnostic only: counts are cites " + - "within that session, not cross-session relevance, and a recurring steering theme reaches " + - "synthesis as a gap sighting instead.", - ); - for (const row of summary.directives) { - lines.push( - `- [${row.id}] ${row.kind ?? "steering"} · turn ${row.turn} · authority ${row.authority ?? "direct"} · ` + - `lifetime turns ${row.lifetime?.fromTurn ?? row.turn}+ ` + - `+${row.positive} -${row.negative} (from ${row.source})`, - ); - for (const quote of row.quotes.slice(0, 3)) { - const sign = quote.polarity === "negative" ? "-" : "+"; - const cls = quote.polarity === "negative" ? ` [${quote.class ?? "unclassified"}]` : ""; - const effect = quote.effect ? ` :: ${oneLine(quote.effect, 200)}` : ""; - lines.push(` ${sign}${cls} "${oneLine(quote.text)}"${effect} (${quote.source})`); - } - } - } - const representedOverlaps = new Set( summary.instructions.filter((row) => row.skillOverlap).map((row) => row.instruction), ); diff --git a/test/directives.test.js b/test/directives.test.js index b8d8d05a..31b67835 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -117,58 +117,30 @@ test("sanitizeEvidence accepts issued directive ids and drops the rest", () => { }); test("the fold keeps directive cites out of memory-instruction rows", () => { - const spans = extractDirectives(distill(fixtureEvents(), META).turns); - const meta = directiveMetadata(spans); - assert.ok( - JSON.stringify(meta).length < 800 && !JSON.stringify(meta).includes("Migrate"), - "persisted metadata carries ids and spans, never turn text", - ); + const meta = directiveMetadata(extractDirectives(distill(fixtureEvents(), META).turns)); + assert.ok(!JSON.stringify(meta).includes("Migrate"), "persisted metadata carries ids and spans, never turn text"); const memoryFile = { units: parseMemoryUnits("# T\n\n- First rule\n") }; - const base = (id) => ({ - status: "ok", - transcript: { id, harness: "claude", startedAt: Date.parse("2026-08-01T00:00:00Z") }, - positive: [], - negative: [], - gaps: [], - }); const summary = foldEvidence( [ { - ...base("s1"), + status: "ok", + transcript: { id: "s1", harness: "claude", startedAt: Date.parse("2026-08-01T00:00:00Z") }, directives: meta, positive: [{ instruction: "TASK-1", quote: "Migrate the auth module", effect: "followed it" }], negative: [{ instruction: "STEER-4", quote: "old refresh-token flow", effect: "dropped it", class: "harm" }], - }, - { - ...base("s2"), - directives: meta, - negative: [{ instruction: "STEER-4", quote: "old refresh-token flow", effect: "dropped it again" }], + gaps: [], }, ], { memoryFile }, ); - assert.equal(summary.directives.length, 3, "one row per session span, never merged across sessions"); - const task = summary.directives.find((row) => row.sessionId === "s1" && row.id === "TASK-1"); - assert.equal(task.positive, 1); - assert.equal(task.authority, "direct-task"); - assert.deepEqual(task.lifetime, { fromTurn: 1, toTurn: null }); assert.ok( !summary.instructions.some((row) => isDirectiveId(row.instruction)), "a directive cite never becomes a memory-instruction row", ); - assert.equal( - summary.totals.instructionsWithNegatives, - 0, - "no memory instruction drew a negative here", - ); - const report = renderEvidenceReport(summary); - assert.match(report, /Direct task\/steering instructions cited/); - assert.match(report, /\[TASK-1\] task · turn 1/); - const prompt = renderEvidenceForPrompt(summary); - assert.ok( - !prompt.includes("Direct task/steering instructions cited") && !prompt.includes("[TASK-1]"), - "the directive section is diagnostic and never reaches the synthesis prompt", - ); + assert.equal(summary.totals.instructionsWithNegatives, 0, "no memory instruction drew a negative here"); + for (const rendered of [renderEvidenceForPrompt(summary), renderEvidenceReport(summary)]) { + assert.ok(!rendered.includes("[TASK-1]") && !rendered.includes("[STEER-4]"), "no directive rows are rendered"); + } }); test("the directive index costs a fraction of duplicating the turns it points at", () => { @@ -244,5 +216,4 @@ test("the same steer ignored in two sessions still clusters into a proposal-elig assert.equal(summary.gaps.length, 1, "the recurring steer is one proposal-eligible gap cluster"); assert.equal(summary.gaps[0].sessions, 2); assert.equal(summary.totals.gapSightings, 2); - assert.equal(summary.directives.length, 2, "the steer keeps its per-session provenance rows"); }); From c516fd28e2159da921173207bcf8ceea8a305fa5 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 01:04:05 +0100 Subject: [PATCH 05/27] no-mistakes(review): index every non-empty envelope; guard directive ids directly --- src/directives.js | 15 ++++----------- src/fold.js | 15 +++------------ test/directives.test.js | 30 ++++++++++++++++++++++++------ 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/directives.js b/src/directives.js index c47295ee..fb0a96ec 100644 --- a/src/directives.js +++ b/src/directives.js @@ -8,8 +8,8 @@ * instruction" apart from "the agent followed the memory file". This module makes each * substantive user turn citable: * - * - the first user turn's authoritative envelope is the task span (`TASK-1`) - * - each later substantive user turn is a steering span (`STEER-`) + * - the first user turn with a non-empty envelope is the task span (`TASK-1`) + * - each later user turn with a non-empty envelope is a steering span (`STEER-`) * - every span carries source kind, turn, an authority tier distinct from project * memory (`direct-task` / `direct-steering`), and a lifetime running from its turn * onward @@ -34,9 +34,6 @@ export const TASK_ID = "TASK-1"; export const TASK_AUTHORITY = "direct-task"; export const STEERING_AUTHORITY = "direct-steering"; -/** Envelopes shorter than this carry no judgable instruction (acks, greetings). */ -export const MIN_ENVELOPE_WORDS = 3; - const FENCE = /^\s*(```|~~~)/; const QUOTE = /^\s*>/; @@ -73,12 +70,8 @@ export function carveEnvelope(text) { return kept.join("\n").trim(); } -function envelopeWords(envelope) { - return envelope.split(/\s+/).filter(Boolean).length; -} - /** - * Index the substantive user turns of one session. `turns` are the distilled message + * Index the user turns of one session that carry an authoritative envelope. `turns` are the distilled message * turns (`{ turn, role, text }`) in trace order, so ids stay stable across runs of the * same transcript and always point at text the model was actually sent. */ @@ -87,7 +80,7 @@ export function extractDirectives(turns) { let seenFirstUser = false; for (const entry of Array.isArray(turns) ? turns : []) { if (!entry || entry.role !== "user") continue; - if (envelopeWords(carveEnvelope(entry.text)) < MIN_ENVELOPE_WORDS) continue; + if (!carveEnvelope(entry.text)) continue; const span = !seenFirstUser ? { id: TASK_ID, kind: "task", turn: entry.turn, authority: TASK_AUTHORITY } : { id: steeringId(entry.turn), kind: "steering", turn: entry.turn, authority: STEERING_AUTHORITY }; diff --git a/src/fold.js b/src/fold.js index 44c39b8c..e86794e1 100644 --- a/src/fold.js +++ b/src/fold.js @@ -132,16 +132,6 @@ export function foldEvidence( ]); const recordSources = issuedSources.slice(0, usable.length); const observationSources = issuedSources.slice(usable.length); - // A direct task/steering cite is session authority, not memory content: the ids this - // session's own analysis was issued never become memory-instruction rows. The gap - // sightings carry the recurrence signal instead. - const declaredDirectives = new Map(); - for (const record of usable) { - const sessionIdentity = record.transcript.identity || record.transcript.id; - if (!Array.isArray(record.directives)) continue; - const ids = new Set(record.directives.filter((span) => span && isDirectiveId(span.id)).map((span) => span.id)); - if (ids.size) declaredDirectives.set(sessionIdentity, ids); - } for (const [index, record] of usable.entries()) { const sessionIdentity = record.transcript.identity || record.transcript.id; const corroborationIdentity = corroborationIdentityOf(record.transcript); @@ -150,11 +140,12 @@ export function foldEvidence( sources.add(source); if (record.transcript.project) sourceProjects[source] = record.transcript.project; const sessionIdentity = record.transcript.identity || record.transcript.id; - const declared = declaredDirectives.get(sessionIdentity); for (const polarity of ["positive", "negative"]) { for (const item of record[polarity] || []) { - if (declared?.has(item.instruction)) continue; + // A direct task/steering cite is session authority, not memory content, so it + // never becomes a memory-instruction row; the gap sightings carry recurrence. + if (isDirectiveId(item.instruction)) continue; const entry = touch(item.instruction); entry[polarity] += 1; const category = classifyInteraction(record.transcript); diff --git a/test/directives.test.js b/test/directives.test.js index 31b67835..cc37d5ff 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -45,27 +45,45 @@ test("an unclosed fence still withholds the paste", () => { assert.equal(carveEnvelope("Do the migration.\n```\nError: boom"), "Do the migration."); }); -test("the first substantive user turn is the task, later ones are steering", () => { +test("the first user turn is the task, later ones are steering; assistant and tool turns never index", () => { const { turns } = distill(fixtureEvents(), META); const spans = extractDirectives(turns); assert.deepEqual( spans.map((span) => [span.id, span.kind, span.turn, span.authority]), [ [TASK_ID, "task", 1, "direct-task"], + ["STEER-3", "steering", 3, "direct-steering"], ["STEER-4", "steering", 4, "direct-steering"], ], ); - // "thanks" is an ack, not an instruction; the assistant reply and tool result never index. - assert.ok(spans.every((span) => span.turn !== 2 && span.turn !== 3)); + assert.ok( + spans.every((span) => span.turn !== 2), + "the assistant reply and the tool result never index", + ); assert.ok(isDirectiveId("TASK-1") && isDirectiveId("STEER-4")); assert.ok(!isDirectiveId("AG-001") && !isDirectiveId("TASK-2") && !isDirectiveId("banana")); }); +test("a short imperative is addressable: length never decides which turn is the task", () => { + const spans = extractDirectives([ + { turn: 1, role: "user", text: "ship it" }, + { turn: 2, role: "user", text: "Add tests." }, + { turn: 3, role: "user", text: "Use pnpm, never npm." }, + ]); + assert.deepEqual( + spans.map((span) => [span.id, span.kind, span.turn]), + [ + [TASK_ID, "task", 1], + ["STEER-2", "steering", 2], + ["STEER-3", "steering", 3], + ], + ); +}); + test("every lifetime runs from its own turn onward; later steering never closes it", () => { const { turns } = distill(fixtureEvents(), META); - const [task, steering] = extractDirectives(turns); - assert.deepEqual(task.lifetime, { fromTurn: 1, toTurn: null }); - assert.deepEqual(steering.lifetime, { fromTurn: 4, toTurn: null }); + const spans = extractDirectives(turns); + for (const span of spans) assert.deepEqual(span.lifetime, { fromTurn: span.turn, toTurn: null }); }); test("a message that is only quotes and pastes yields no span", () => { From 6e1593c53c71cf3510c9d25a4be40562b7adcd7a Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 01:20:37 +0100 Subject: [PATCH 06/27] no-mistakes(review): close envelope fences per CommonMark marker matching --- src/directives.js | 28 +++++++++++++++++++++------- test/directives.test.js | 18 ++++++++++++++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/directives.js b/src/directives.js index fb0a96ec..b25c182e 100644 --- a/src/directives.js +++ b/src/directives.js @@ -34,7 +34,7 @@ export const TASK_ID = "TASK-1"; export const TASK_AUTHORITY = "direct-task"; export const STEERING_AUTHORITY = "direct-steering"; -const FENCE = /^\s*(```|~~~)/; +const FENCE = /^\s*(`{3,}|~{3,})([^\n]*)$/; const QUOTE = /^\s*>/; /** A span id is stable by construction: kind plus the distilled turn it points at. */ @@ -53,16 +53,30 @@ function steeringId(turn) { */ export function carveEnvelope(text) { const kept = []; - let inFence = false; + let opener = null; for (const line of String(text ?? "").split("\n")) { - if (FENCE.test(line)) { - inFence = !inFence; + const fence = FENCE.exec(line); + if (opener) { + // CommonMark: only a fence of the opener's character, at least as long, with + // nothing but whitespace after it, closes the block. A shorter or different + // inner fence is part of the paste. + if ( + fence && + fence[1][0] === opener.char && + fence[1].length >= opener.length && + fence[2].trim() === "" + ) { + opener = null; + } + // An unclosed fence still withholds the rest: a pasted log without a closing + // marker is still a paste, not an instruction. + continue; + } + if (fence) { + opener = { char: fence[1][0], length: fence[1].length }; continue; } - // An unclosed fence still withholds the rest: a pasted log without a closing - // marker is still a paste, not an instruction. - if (inFence) continue; if (QUOTE.test(line)) continue; kept.push(line); } diff --git a/test/directives.test.js b/test/directives.test.js index cc37d5ff..c01ecb6f 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -41,6 +41,24 @@ test("only the authoritative envelope becomes instruction text", () => { assert.ok(!envelope.includes("refreshSession"), "pasted tool output must not be authority"); }); +test("a nested inner fence stays inside the paste; the outer fence closes it", () => { + const envelope = carveEnvelope( + ["Update the README:", "````md", "# doc", "```bash", "rm -rf build", "```", "````", "Keep the examples short."].join( + "\n", + ), + ); + assert.ok(!envelope.includes("rm -rf build"), "pasted command must not be authority"); + assert.ok(!envelope.includes("# doc"), "pasted doc body must not be authority"); + assert.match(envelope, /Update the README:/); + assert.match(envelope, /Keep the examples short\./); +}); + +test("a tilde fence never closes a backtick block", () => { + const envelope = carveEnvelope(["Fix this:", "```", "~~~", "rm -rf build", "```", "Run the tests."].join("\n")); + assert.ok(!envelope.includes("rm -rf build"), "the tilde line does not close the backtick fence"); + assert.match(envelope, /Run the tests\./); +}); + test("an unclosed fence still withholds the paste", () => { assert.equal(carveEnvelope("Do the migration.\n```\nError: boom"), "Do the migration."); }); From 7df7bdcf4e03ae3fcd991a00577fb361ad5d0a8d Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 01:28:26 +0100 Subject: [PATCH 07/27] no-mistakes(review): reject backtick-in-info fence openers in envelope carving --- src/directives.js | 4 +++- test/directives.test.js | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/directives.js b/src/directives.js index b25c182e..e6f2aa51 100644 --- a/src/directives.js +++ b/src/directives.js @@ -73,7 +73,9 @@ export function carveEnvelope(text) { // marker is still a paste, not an instruction. continue; } - if (fence) { + // CommonMark: a backtick fence's info string may not contain a backtick, so a line + // of inline code is instruction text, not an opener. + if (fence && !(fence[1][0] === "`" && fence[2].includes("`"))) { opener = { char: fence[1][0], length: fence[1].length }; continue; } diff --git a/test/directives.test.js b/test/directives.test.js index c01ecb6f..465abf43 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -59,6 +59,12 @@ test("a tilde fence never closes a backtick block", () => { assert.match(envelope, /Run the tests\./); }); +test("a line of inline code is instruction text, not a fence opener", () => { + const envelope = carveEnvelope("```pnpm run check``` must pass before you push.\nAlso update the README."); + assert.match(envelope, /must pass before you push\./); + assert.match(envelope, /Also update the README\./); +}); + test("an unclosed fence still withholds the paste", () => { assert.equal(carveEnvelope("Do the migration.\n```\nError: boom"), "Do the migration."); }); From 7950b48ae0d40e45b0a30aa21d55972ddc6c616e Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 01:41:34 +0100 Subject: [PATCH 08/27] no-mistakes(document): document direct task/steering instruction sources; format directives --- README.md | 16 ++++++++++++++-- src/directives.js | 7 +------ test/directives.test.js | 18 +++++++++++------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 989c96c7..f1a05df0 100644 --- a/README.md +++ b/README.md @@ -369,8 +369,9 @@ weights still evolve as transcripts age, so the selected set can change over tim equally reproducible sample. Each distilled trace goes to a cheap model with the memory file, the project skill index, -and a rubric. It returns strict JSON: which instructions helped, which were violated, and -what mistakes no current instruction covers. Every negative carries a class - `harm` (following the instruction +an index of this session's direct instructions, and a rubric. It returns strict JSON: which +instructions helped, which were violated, and +what mistakes no memory instruction covers. Every negative carries a class - `harm` (following the instruction caused damage), `non-compliance` (the agent ignored it), or `irrelevant` - because those argue for opposite fates: harm argues against an instruction, non-compliance argues for reinforcing it. Every gap carries a domain - `orchestration` when the mistake was caused @@ -380,6 +381,17 @@ orchestrating tool) - and the analysis is shown the ledger's open gaps so it can cite an existing gap id instead of coining a paraphrase of it. +Much of what governs a session comes from the user directly rather than from a memory file, +so those instructions are addressable too. The user's own task message (`TASK-1`) and each +later steering turn (`STEER-`) become citable instruction sources with their own +authority, alongside the memory-file ids. Only the authoritative part of a user turn counts: +fenced blocks and quoted lines are pasted evidence, never instructions, and assistant or tool +text never becomes one. The index points at a turn already in the trace instead of repeating +its text, and only the ids and their metadata persist in the evidence record. Direct +instructions are session authority, not durable memory, so a mistake covered only by one is +still reported as a gap - the user had to say it in this session, and the next session starts +without it - and their citations do not score a memory-file instruction. + **Every claim must carry a verbatim quote, and the quote must be in the trace.** Quoteless items are discarded - the single most important defence against a model confabulating influence - and so are quotes that do not actually appear in the distilled trace they claim diff --git a/src/directives.js b/src/directives.js index e6f2aa51..697d4440 100644 --- a/src/directives.js +++ b/src/directives.js @@ -61,12 +61,7 @@ export function carveEnvelope(text) { // CommonMark: only a fence of the opener's character, at least as long, with // nothing but whitespace after it, closes the block. A shorter or different // inner fence is part of the paste. - if ( - fence && - fence[1][0] === opener.char && - fence[1].length >= opener.length && - fence[2].trim() === "" - ) { + if (fence && fence[1][0] === opener.char && fence[1].length >= opener.length && fence[2].trim() === "") { opener = null; } // An unclosed fence still withholds the rest: a pasted log without a closing diff --git a/test/directives.test.js b/test/directives.test.js index 465abf43..1701281b 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -43,9 +43,16 @@ test("only the authoritative envelope becomes instruction text", () => { test("a nested inner fence stays inside the paste; the outer fence closes it", () => { const envelope = carveEnvelope( - ["Update the README:", "````md", "# doc", "```bash", "rm -rf build", "```", "````", "Keep the examples short."].join( - "\n", - ), + [ + "Update the README:", + "````md", + "# doc", + "```bash", + "rm -rf build", + "```", + "````", + "Keep the examples short.", + ].join("\n"), ); assert.ok(!envelope.includes("rm -rf build"), "pasted command must not be authority"); assert.ok(!envelope.includes("# doc"), "pasted doc body must not be authority"); @@ -127,10 +134,7 @@ test("the index renders by reference and never repeats turn text", () => { assert.match(section, /see turn 1 in the trace/); assert.match(section, /lifetime turns 1\+/); for (const entry of turns.filter((candidate) => candidate.role === "user")) { - assert.ok( - !section.includes(carveEnvelope(entry.text)), - `turn ${entry.turn} text must not be duplicated`, - ); + assert.ok(!section.includes(carveEnvelope(entry.text)), `turn ${entry.turn} text must not be duplicated`); } assert.ok(trace.includes("Migrate the auth module"), "the trace itself still carries the text"); }); From 2dfb33f772df0ec7d5de76138e657a77e724a6ab Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 01:56:19 +0100 Subject: [PATCH 09/27] no-mistakes(document): reflow README direct-instruction paragraph wrap --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f1a05df0..a238b7f2 100644 --- a/README.md +++ b/README.md @@ -370,8 +370,8 @@ equally reproducible sample. Each distilled trace goes to a cheap model with the memory file, the project skill index, an index of this session's direct instructions, and a rubric. It returns strict JSON: which -instructions helped, which were violated, and -what mistakes no memory instruction covers. Every negative carries a class - `harm` (following the instruction +instructions helped, which were violated, and what mistakes no memory instruction covers. +Every negative carries a class - `harm` (following the instruction caused damage), `non-compliance` (the agent ignored it), or `irrelevant` - because those argue for opposite fates: harm argues against an instruction, non-compliance argues for reinforcing it. Every gap carries a domain - `orchestration` when the mistake was caused From c55bb8ccf6b430ae5717661c3c10d3994d80c6c1 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 02:31:16 +0100 Subject: [PATCH 10/27] no-mistakes(review): carve envelopes pre-clamp, bound fence indent, mark elided turns --- src/directives.js | 55 ++++++++++++------------ src/distill.js | 43 +++++++++++++++---- test/directives.test.js | 92 ++++++++++++++++++++++++++++++++++++----- 3 files changed, 144 insertions(+), 46 deletions(-) diff --git a/src/directives.js b/src/directives.js index 697d4440..2dc83c52 100644 --- a/src/directives.js +++ b/src/directives.js @@ -21,10 +21,12 @@ * untrusted evidence. Assistant messages and tool results never become spans. * * Spans are extracted from the distilled message turns, so turn numbers match the - * trace the analysis model reads. The prompt index renders them by reference (id plus - * a `see turn N` pointer), never by duplicating turn text: task and steering text is - * private by default, and only ids plus metadata (`directives` on the evidence - * record) ever persist - envelope text never leaves the in-memory prompt. + * trace the analysis model reads. Each turn carries the envelope `distill` carved from + * its full pre-clamp text, so a closing fence lost to message clamping cannot demote an + * instruction to a paste. The prompt index renders spans by reference (id plus a + * `see turn N` pointer, or a note that the turn was elided from the trace), never by + * duplicating turn text: task and steering text is private by default, and only ids plus + * metadata (`directives` on the evidence record) ever persist. * * A span is session authority, never durable project memory: a mistake only a direct * instruction covered is still a gap, because the next session starts without it. @@ -34,8 +36,9 @@ export const TASK_ID = "TASK-1"; export const TASK_AUTHORITY = "direct-task"; export const STEERING_AUTHORITY = "direct-steering"; -const FENCE = /^\s*(`{3,}|~{3,})([^\n]*)$/; -const QUOTE = /^\s*>/; +// CommonMark: four or more leading spaces is indented code, not a fence or a quote. +const FENCE = /^ {0,3}(`{3,}|~{3,})([^\n]*)$/; +const QUOTE = /^ {0,3}>/; /** A span id is stable by construction: kind plus the distilled turn it points at. */ export function isDirectiveId(id) { @@ -82,21 +85,22 @@ export function carveEnvelope(text) { } /** - * Index the user turns of one session that carry an authoritative envelope. `turns` are the distilled message - * turns (`{ turn, role, text }`) in trace order, so ids stay stable across runs of the - * same transcript and always point at text the model was actually sent. + * Index the user turns of one session that carry an authoritative envelope. `turns` are + * the distilled message turns (`{ turn, role, envelope, elided }`) in trace order, so ids + * stay stable across runs of the same transcript and each span knows whether its turn + * survived into the trace. */ export function extractDirectives(turns) { const spans = []; let seenFirstUser = false; for (const entry of Array.isArray(turns) ? turns : []) { if (!entry || entry.role !== "user") continue; - if (!carveEnvelope(entry.text)) continue; + if (!entry.envelope) continue; const span = !seenFirstUser ? { id: TASK_ID, kind: "task", turn: entry.turn, authority: TASK_AUTHORITY } : { id: steeringId(entry.turn), kind: "steering", turn: entry.turn, authority: STEERING_AUTHORITY }; seenFirstUser = true; - spans.push({ ...span, lifetime: { fromTurn: entry.turn, toTurn: null } }); + spans.push({ ...span, elided: entry.elided === true, lifetime: { fromTurn: entry.turn, toTurn: null } }); } return spans; } @@ -113,22 +117,21 @@ export function directiveMetadata(spans) { } /** - * The analysis-prompt index. By reference only: each entry names the turn whose text - * is already in the trace, so no user-turn text is duplicated into the prompt. + * The analysis-prompt index. By reference only: each entry names the turn whose text is + * already in the trace, or says the turn was elided from it, so no user-turn text is + * duplicated into the prompt. */ -export function renderDirectiveIndex(spans, { elided = false } = {}) { +export function renderDirectiveIndex(spans) { const list = Array.isArray(spans) ? spans : []; if (!list.length) return "(none - no substantive user instruction found in the trace)"; - const lines = list.map( - (span) => - `[${span.id}] ${span.kind} · turn ${span.turn} · authority ${span.authority} · ` + - `lifetime turns ${span.lifetime?.fromTurn ?? span.turn}+ - see turn ${span.turn} in the trace`, - ); - if (elided) { - lines.push( - "The trace middle was elided; a cited turn may sit in the elided span - " + - "open the raw transcript when a claim needs its text.", - ); - } - return lines.join("\n"); + return list + .map( + (span) => + `[${span.id}] ${span.kind} · turn ${span.turn} · authority ${span.authority} · ` + + `lifetime turns ${span.lifetime?.fromTurn ?? span.turn}+ - ` + + (span.elided + ? `turn ${span.turn} - elided from the trace, open the raw transcript for its text` + : `see turn ${span.turn} in the trace`), + ) + .join("\n"); } diff --git a/src/distill.js b/src/distill.js index 444d578f..9e0e15bc 100644 --- a/src/distill.js +++ b/src/distill.js @@ -1,3 +1,4 @@ +import { carveEnvelope } from "./directives.js"; import { redact } from "./redact.js"; import { estimateTokens } from "./tokens.js"; @@ -100,10 +101,18 @@ export function isBoilerplate(text) { export function distill(events, meta, options = {}) { const maxTraceTokens = options.maxTraceTokens ?? 12000; const lines = []; + let bodyChars = 0; + const push = (line) => { + lines.push(line); + bodyChars += line.length + 1; + }; // Message turns in trace order, with the same turn numbers the trace prints: the // single source of truth for anything that points at a turn (directive spans in - // `src/directives.js`). Text is the redacted form the model actually sees. + // `src/directives.js`). Text is the redacted form the model actually sees; the + // envelope is carved from the full pre-clamp text, so a closing fence that the clamp + // elides cannot turn an instruction into a paste. const turns = []; + const turnOffsets = []; let userTurns = 0; let assistantTurns = 0; let toolCalls = 0; @@ -112,22 +121,29 @@ export function distill(events, meta, options = {}) { for (const event of events) { if (!event) continue; if (event.kind === "message") { - const text = clampMessage(redact(event.text)); + const redacted = redact(event.text); + const text = clampMessage(redacted); if (!text || isBoilerplate(text)) continue; turn += 1; if (event.role === "user") userTurns += 1; else assistantTurns += 1; - turns.push({ turn, role: event.role, text }); - lines.push(`### turn ${turn} · ${event.role}`); - lines.push(text); - lines.push(""); + turns.push({ + turn, + role: event.role, + text, + envelope: event.role === "user" ? carveEnvelope(redacted) : "", + }); + turnOffsets.push(bodyChars); + push(`### turn ${turn} · ${event.role}`); + push(text); + push(""); } else if (event.kind === "tool") { toolCalls += 1; const input = redact(describeToolInput(event.input)); const result = redact(describeToolResult(event.result)); const status = event.status && event.status !== "completed" ? ` [${event.status}]` : ""; const arrow = result ? ` -> ${result}` : ""; - lines.push(`tool: ${event.name || "unknown"}${input ? ` ${JSON.stringify(input)}` : ""}${status}${arrow}`); + push(`tool: ${event.name || "unknown"}${input ? ` ${JSON.stringify(input)}` : ""}${status}${arrow}`); } } @@ -152,7 +168,12 @@ export function distill(events, meta, options = {}) { "transcript only if a specific claim needs the full text.", ].join("\n"); - const { body, elided } = capTrace(lines.join("\n").trim(), maxTraceTokens); + const rawBody = lines.join("\n").trim(); + const { body, elided, kept } = capTrace(rawBody, maxTraceTokens); + turns.forEach((entry, index) => { + const at = turnOffsets[index]; + entry.elided = !kept.some(([from, to]) => at >= from && at < to); + }); const trace = `${header}\n${body}\n${footer}\n`; return { @@ -174,7 +195,7 @@ export function distill(events, meta, options = {}) { * raw transcript path in the footer remains the escape hatch for anything in between. */ function capTrace(body, maxTraceTokens) { - if (estimateTokens(body) <= maxTraceTokens) return { body, elided: false }; + if (estimateTokens(body) <= maxTraceTokens) return { body, elided: false, kept: [[0, body.length]] }; const budgetChars = maxTraceTokens * 4; const headChars = Math.floor(budgetChars * 0.45); const tailChars = Math.floor(budgetChars * 0.45); @@ -184,5 +205,9 @@ function capTrace(body, maxTraceTokens) { return { body: `${head}\n\n[... middle of session elided: ~${droppedTokens} tokens. Open the raw transcript below if a claim needs it ...]\n\n${tail}`, elided: true, + kept: [ + [0, headChars], + [body.length - tailChars, body.length], + ], }; } diff --git a/test/directives.test.js b/test/directives.test.js index 1701281b..bedb6465 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -76,6 +76,44 @@ test("an unclosed fence still withholds the paste", () => { assert.equal(carveEnvelope("Do the migration.\n```\nError: boom"), "Do the migration."); }); +test("an indented fence line inside a paste neither opens nor closes", () => { + const envelope = carveEnvelope( + [ + "Fix the docs:", + "```md", + "Example:", + " ```bash", + " npm ci", + " ```", + "End.", + "```", + "Use pnpm, not npm.", + ].join("\n"), + ); + assert.ok(!envelope.includes("npm ci"), "the indented block stays inside the paste"); + assert.ok(!envelope.includes("End."), "the paste remainder never re-enters the envelope"); + assert.match(envelope, /Fix the docs:/); + assert.match(envelope, /Use pnpm, not npm\./); +}); + +test("an over-limit turn keeps its instruction when the closing fence falls in the clamped region", () => { + const log = "2026-01-01T00:00:00Z some CI log line\n".repeat(200); + const { turns } = distill( + [ + { kind: "message", role: "user", text: `\`\`\`\n${log}\`\`\`\n\nKeep the public API stable.` }, + { kind: "message", role: "assistant", text: "Will do." }, + ], + META, + ); + assert.ok(turns[0].text.includes("chars elided"), "the turn text is clamped"); + const spans = extractDirectives(turns); + assert.deepEqual( + spans.map((span) => [span.id, span.turn]), + [[TASK_ID, 1]], + ); + assert.equal(turns[0].envelope, "Keep the public API stable."); +}); + test("the first user turn is the task, later ones are steering; assistant and tool turns never index", () => { const { turns } = distill(fixtureEvents(), META); const spans = extractDirectives(turns); @@ -96,11 +134,15 @@ test("the first user turn is the task, later ones are steering; assistant and to }); test("a short imperative is addressable: length never decides which turn is the task", () => { - const spans = extractDirectives([ - { turn: 1, role: "user", text: "ship it" }, - { turn: 2, role: "user", text: "Add tests." }, - { turn: 3, role: "user", text: "Use pnpm, never npm." }, - ]); + const { turns } = distill( + [ + { kind: "message", role: "user", text: "ship it" }, + { kind: "message", role: "user", text: "Add tests." }, + { kind: "message", role: "user", text: "Use pnpm, never npm." }, + ], + META, + ); + const spans = extractDirectives(turns); assert.deepEqual( spans.map((span) => [span.id, span.kind, span.turn]), [ @@ -118,10 +160,14 @@ test("every lifetime runs from its own turn onward; later steering never closes }); test("a message that is only quotes and pastes yields no span", () => { - const spans = extractDirectives([ - { turn: 1, role: "user", text: "> quoted transcript line\n\n```\npasted output\n```" }, - { turn: 2, role: "assistant", text: "Noted." }, - ]); + const { turns } = distill( + [ + { kind: "message", role: "user", text: "> quoted transcript line\n\n```\npasted output\n```" }, + { kind: "message", role: "assistant", text: "Noted." }, + ], + META, + ); + const spans = extractDirectives(turns); assert.deepEqual(spans, []); assert.match(renderDirectiveIndex(spans), /\(none/); }); @@ -134,11 +180,35 @@ test("the index renders by reference and never repeats turn text", () => { assert.match(section, /see turn 1 in the trace/); assert.match(section, /lifetime turns 1\+/); for (const entry of turns.filter((candidate) => candidate.role === "user")) { - assert.ok(!section.includes(carveEnvelope(entry.text)), `turn ${entry.turn} text must not be duplicated`); + assert.ok(!section.includes(entry.envelope), `turn ${entry.turn} text must not be duplicated`); } assert.ok(trace.includes("Migrate the auth module"), "the trace itself still carries the text"); }); +test("an indexed turn the trace elided is marked elided, not pointed at", () => { + const filler = (n) => `${"pipeline detail ".repeat(400)} step ${n}`; + const events = [ + { kind: "message", role: "user", text: "Migrate the auth module." }, + ...Array.from({ length: 30 }, (_, i) => [ + { kind: "message", role: "assistant", text: filler(i) }, + { kind: "message", role: "user", text: `Also handle case ${i}.` }, + ]).flat(), + { kind: "message", role: "user", text: "Keep the public API stable." }, + ]; + const distilled = distill(events, META); + assert.equal(distilled.stats.elided, true); + const spans = extractDirectives(distilled.turns); + const elidedSpans = spans.filter((span) => span.elided); + assert.ok(elidedSpans.length, "the capped middle holds at least one indexed user turn"); + const section = renderDirectiveIndex(spans); + for (const span of elidedSpans) { + assert.ok(section.includes(`turn ${span.turn} - elided from the trace`), `span ${span.id} must be marked elided`); + assert.ok(!section.includes(`see turn ${span.turn} in the trace`)); + } + const kept = spans.find((span) => !span.elided); + assert.ok(section.includes(`see turn ${kept.turn} in the trace`), "a surviving turn still points at the trace"); +}); + test("sanitizeEvidence accepts issued directive ids and drops the rest", () => { const memoryFile = { units: parseMemoryUnits("# T\n\n- First rule\n") }; const clean = sanitizeEvidence( @@ -205,7 +275,7 @@ test("the directive index costs a fraction of duplicating the turns it points at harness: candidate.path.includes("pi-") ? "pi" : "claude", }); const spans = extractDirectives(distilled.turns); - const section = renderDirectiveIndex(spans, { elided: distilled.stats.elided }); + const section = renderDirectiveIndex(spans); const sectionTokens = estimateTokens(section); // By-reference cost scales with the span count, never the turn length: one // bounded entry per span, whatever the turn says. From 5b6d0cad561b33454d69e3b2fbd7ae3b5d03bd1e Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 02:40:24 +0100 Subject: [PATCH 11/27] no-mistakes(review): require whole-turn trace containment; relative fence indentation --- src/directives.js | 31 +++++++++++++++++++++++-------- src/distill.js | 9 +++++---- test/directives.test.js | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/src/directives.js b/src/directives.js index 2dc83c52..d379e321 100644 --- a/src/directives.js +++ b/src/directives.js @@ -36,9 +36,17 @@ export const TASK_ID = "TASK-1"; export const TASK_AUTHORITY = "direct-task"; export const STEERING_AUTHORITY = "direct-steering"; -// CommonMark: four or more leading spaces is indented code, not a fence or a quote. -const FENCE = /^ {0,3}(`{3,}|~{3,})([^\n]*)$/; -const QUOTE = /^ {0,3}>/; +const FENCE = /^([ \t]*)(`{3,}|~{3,})([^\n]*)$/; +const QUOTE = /^\s*>/; + +// CommonMark counts a tab as four columns, and fence indentation is relative: an opener +// may sit at any indentation (a list-item code block is still a fence), while its closer +// may be indented at most three columns further. +function indentColumns(prefix) { + let columns = 0; + for (const char of prefix) columns += char === "\t" ? 4 : 1; + return columns; +} /** A span id is stable by construction: kind plus the distilled turn it points at. */ export function isDirectiveId(id) { @@ -62,9 +70,16 @@ export function carveEnvelope(text) { const fence = FENCE.exec(line); if (opener) { // CommonMark: only a fence of the opener's character, at least as long, with - // nothing but whitespace after it, closes the block. A shorter or different - // inner fence is part of the paste. - if (fence && fence[1][0] === opener.char && fence[1].length >= opener.length && fence[2].trim() === "") { + // nothing but whitespace after it and indented no more than three columns past + // the opener, closes the block. A shorter, different or deeper inner fence is + // part of the paste. + if ( + fence && + fence[2][0] === opener.char && + fence[2].length >= opener.length && + fence[3].trim() === "" && + indentColumns(fence[1]) <= opener.indent + 3 + ) { opener = null; } // An unclosed fence still withholds the rest: a pasted log without a closing @@ -73,8 +88,8 @@ export function carveEnvelope(text) { } // CommonMark: a backtick fence's info string may not contain a backtick, so a line // of inline code is instruction text, not an opener. - if (fence && !(fence[1][0] === "`" && fence[2].includes("`"))) { - opener = { char: fence[1][0], length: fence[1].length }; + if (fence && !(fence[2][0] === "`" && fence[3].includes("`"))) { + opener = { char: fence[2][0], length: fence[2].length, indent: indentColumns(fence[1]) }; continue; } if (QUOTE.test(line)) continue; diff --git a/src/distill.js b/src/distill.js index 9e0e15bc..744a02c1 100644 --- a/src/distill.js +++ b/src/distill.js @@ -112,7 +112,7 @@ export function distill(events, meta, options = {}) { // envelope is carved from the full pre-clamp text, so a closing fence that the clamp // elides cannot turn an instruction into a paste. const turns = []; - const turnOffsets = []; + const turnSpans = []; let userTurns = 0; let assistantTurns = 0; let toolCalls = 0; @@ -133,9 +133,10 @@ export function distill(events, meta, options = {}) { text, envelope: event.role === "user" ? carveEnvelope(redacted) : "", }); - turnOffsets.push(bodyChars); + const from = bodyChars; push(`### turn ${turn} · ${event.role}`); push(text); + turnSpans.push([from, bodyChars - 1]); push(""); } else if (event.kind === "tool") { toolCalls += 1; @@ -171,8 +172,8 @@ export function distill(events, meta, options = {}) { const rawBody = lines.join("\n").trim(); const { body, elided, kept } = capTrace(rawBody, maxTraceTokens); turns.forEach((entry, index) => { - const at = turnOffsets[index]; - entry.elided = !kept.some(([from, to]) => at >= from && at < to); + const [from, to] = turnSpans[index]; + entry.elided = !kept.some(([keptFrom, keptTo]) => from >= keptFrom && to <= keptTo); }); const trace = `${header}\n${body}\n${footer}\n`; diff --git a/test/directives.test.js b/test/directives.test.js index bedb6465..123e9625 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -209,6 +209,41 @@ test("an indexed turn the trace elided is marked elided, not pointed at", () => assert.ok(section.includes(`see turn ${kept.turn} in the trace`), "a surviving turn still points at the trace"); }); +test("a fence indented inside a list item still opens a paste", () => { + const envelope = carveEnvelope( + ["Steps:", "10. Run:", "", " ```bash", " rm -rf dist", " ```", "", "Then deploy."].join("\n"), + ); + assert.ok(!envelope.includes("rm -rf dist"), "a list-item code block is still a paste"); + assert.match(envelope, /Steps:/); + assert.match(envelope, /Then deploy\./); +}); + +test("a turn the trace only half kept is marked elided, never pointed at", () => { + const events = Array.from({ length: 40 }, (_, i) => ({ + kind: "message", + role: i % 2 === 0 ? "user" : "assistant", + text: `${i % 2 === 0 ? "Also handle case" : "Handled case"} ${i}. ${"pipeline detail ".repeat(6)}marker-${i}`, + })); + const distilled = distill(events, META, { maxTraceTokens: 1000 }); + assert.equal(distilled.stats.elided, true); + const spans = extractDirectives(distilled.turns); + const section = renderDirectiveIndex(spans); + let halfKept = 0; + for (const span of spans) { + const entry = distilled.turns.find((candidate) => candidate.turn === span.turn); + const whole = distilled.trace.includes(entry.text); + const partial = !whole && distilled.trace.includes(entry.text.slice(0, 20)); + if (partial) halfKept += 1; + if (!span.elided) { + assert.ok(whole, `span ${span.id} is pointed at but turn ${span.turn} is not wholly in the trace`); + assert.ok(section.includes(`see turn ${span.turn} in the trace`)); + } else { + assert.ok(section.includes(`turn ${span.turn} - elided from the trace`)); + } + } + assert.ok(halfKept >= 1, "the head or tail cut must straddle at least one indexed turn"); +}); + test("sanitizeEvidence accepts issued directive ids and drops the rest", () => { const memoryFile = { units: parseMemoryUnits("# T\n\n- First rule\n") }; const clean = sanitizeEvidence( From 95536edce83690a3ed25a04aa439ad57091c3040 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 02:51:46 +0100 Subject: [PATCH 12/27] no-mistakes(document): document elided directive index entries; fix distill typecheck --- README.md | 3 ++- src/distill.js | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a238b7f2..4a42a736 100644 --- a/README.md +++ b/README.md @@ -387,7 +387,8 @@ later steering turn (`STEER-`) become citable instruction sources with the authority, alongside the memory-file ids. Only the authoritative part of a user turn counts: fenced blocks and quoted lines are pasted evidence, never instructions, and assistant or tool text never becomes one. The index points at a turn already in the trace instead of repeating -its text, and only the ids and their metadata persist in the evidence record. Direct +its text - or says that turn was elided from the trace, when the session was long enough to +be capped - and only the ids and their metadata persist in the evidence record. Direct instructions are session authority, not durable memory, so a mistake covered only by one is still reported as a gap - the user had to say it in this session, and the next session starts without it - and their citations do not score a memory-file instruction. diff --git a/src/distill.js b/src/distill.js index 744a02c1..45f13efb 100644 --- a/src/distill.js +++ b/src/distill.js @@ -132,6 +132,8 @@ export function distill(events, meta, options = {}) { role: event.role, text, envelope: event.role === "user" ? carveEnvelope(redacted) : "", + // Set once the trace cap is known, below. + elided: false, }); const from = bodyChars; push(`### turn ${turn} · ${event.role}`); From ef542bd6eae16578c49fb29c6abdce292d2504cc Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 25 Sep 2026 03:07:15 +0100 Subject: [PATCH 13/27] =?UTF-8?q?no-mistakes(ci):=20ci-5=20fixed.=20Invari?= =?UTF-8?q?ant:=20a=20directive=20index=20entry=20may=20say=20"see=20turn?= =?UTF-8?q?=20N=20in=20the=20trace"=20only=20when=20that=20turn's=20text?= =?UTF-8?q?=20actually=20reached=20the=20trace.=20Two=20ways=20it=20cannot?= =?UTF-8?q?:=20capTrace=20elision=20(already=20handled)=20and=20per-messag?= =?UTF-8?q?e=20clampMessage,=20which=20was=20unhandled=20because=20the=20e?= =?UTF-8?q?nvelope=20is=20now=20carved=20pre-clamp.=20Changes:=20-=20`src/?= =?UTF-8?q?distill.js`:=20each=20message=20turn=20records=20`clamped:=20te?= =?UTF-8?q?xt=20!=3D=3D=20redacted.trim()`=20=E2=80=94=20true=20exactly=20?= =?UTF-8?q?when=20clampMessage=20dropped=20a=20middle=20from=20the=20text?= =?UTF-8?q?=20the=20model=20reads.=20Whole-turn=20flag,=20matching=20the?= =?UTF-8?q?=20existing=20whole-turn=20`elided`=20flag=20(same=20conservati?= =?UTF-8?q?sm=20as=20the=20tail-boundary=20case).=20-=20`src/directives.js?= =?UTF-8?q?`:=20spans=20carry=20`clamped`;=20`renderDirectiveIndex`=20rend?= =?UTF-8?q?ers=20`turn=20N=20-=20clamped=20from=20the=20trace,=20open=20th?= =?UTF-8?q?e=20raw=20transcript=20for=20its=20text`=20(after=20the=20elide?= =?UTF-8?q?d=20branch,=20before=20the=20pointer=20branch).=20Header/JSDoc?= =?UTF-8?q?=20comments=20updated.=20No=20other=20consumer=20touched:=20`di?= =?UTF-8?q?rectiveMetadata`=20still=20persists=20only=20ids/turn/authority?= =?UTF-8?q?/lifetime.=20-=20`test/directives.test.js`:=20new=20test=20?= =?UTF-8?q?=E2=80=94=20a=20user=20message=20over=20the=206,000-char=20clam?= =?UTF-8?q?p=20whose=20only=20instruction=20sits=20in=20the=20clamped=20mi?= =?UTF-8?q?ddle=20yields=20`spans[0].clamped=20=3D=3D=3D=20true`,=20render?= =?UTF-8?q?s=20the=20clamped=20entry,=20and=20never=20renders=20"see=20tur?= =?UTF-8?q?n=201=20in=20the=20trace";=20also=20asserts=20the=20trace=20its?= =?UTF-8?q?elf=20lacks=20the=20instruction.=20ci-4=20out=20of=20scope=20pe?= =?UTF-8?q?r=20instruction.=20`pnpm=20run=20check`=20exits=200=20(711=20te?= =?UTF-8?q?sts=20pass,=20lint/format/typecheck=20clean)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/directives.js | 23 +++++++++++++++-------- src/distill.js | 3 +++ test/directives.test.js | 23 +++++++++++++++++++++++ 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/directives.js b/src/directives.js index d379e321..ba3e90df 100644 --- a/src/directives.js +++ b/src/directives.js @@ -24,7 +24,7 @@ * trace the analysis model reads. Each turn carries the envelope `distill` carved from * its full pre-clamp text, so a closing fence lost to message clamping cannot demote an * instruction to a paste. The prompt index renders spans by reference (id plus a - * `see turn N` pointer, or a note that the turn was elided from the trace), never by + * `see turn N` pointer, or a note that the turn was elided or clamped out of the trace), never by * duplicating turn text: task and steering text is private by default, and only ids plus * metadata (`directives` on the evidence record) ever persist. * @@ -101,9 +101,9 @@ export function carveEnvelope(text) { /** * Index the user turns of one session that carry an authoritative envelope. `turns` are - * the distilled message turns (`{ turn, role, envelope, elided }`) in trace order, so ids - * stay stable across runs of the same transcript and each span knows whether its turn - * survived into the trace. + * the distilled message turns (`{ turn, role, envelope, elided, clamped }`) in trace order, + * so ids stay stable across runs of the same transcript and each span knows whether its + * turn text survived into the trace. */ export function extractDirectives(turns) { const spans = []; @@ -115,7 +115,12 @@ export function extractDirectives(turns) { ? { id: TASK_ID, kind: "task", turn: entry.turn, authority: TASK_AUTHORITY } : { id: steeringId(entry.turn), kind: "steering", turn: entry.turn, authority: STEERING_AUTHORITY }; seenFirstUser = true; - spans.push({ ...span, elided: entry.elided === true, lifetime: { fromTurn: entry.turn, toTurn: null } }); + spans.push({ + ...span, + elided: entry.elided === true, + clamped: entry.clamped === true, + lifetime: { fromTurn: entry.turn, toTurn: null }, + }); } return spans; } @@ -133,8 +138,8 @@ export function directiveMetadata(spans) { /** * The analysis-prompt index. By reference only: each entry names the turn whose text is - * already in the trace, or says the turn was elided from it, so no user-turn text is - * duplicated into the prompt. + * already in the trace, or says the turn's text was elided or clamped out of it, so no + * user-turn text is duplicated into the prompt. */ export function renderDirectiveIndex(spans) { const list = Array.isArray(spans) ? spans : []; @@ -146,7 +151,9 @@ export function renderDirectiveIndex(spans) { `lifetime turns ${span.lifetime?.fromTurn ?? span.turn}+ - ` + (span.elided ? `turn ${span.turn} - elided from the trace, open the raw transcript for its text` - : `see turn ${span.turn} in the trace`), + : span.clamped + ? `turn ${span.turn} - clamped from the trace, open the raw transcript for its text` + : `see turn ${span.turn} in the trace`), ) .join("\n"); } diff --git a/src/distill.js b/src/distill.js index 45f13efb..d4618509 100644 --- a/src/distill.js +++ b/src/distill.js @@ -132,6 +132,9 @@ export function distill(events, meta, options = {}) { role: event.role, text, envelope: event.role === "user" ? carveEnvelope(redacted) : "", + // The envelope is carved pre-clamp, so a clamped turn's instruction may be absent + // from what the model reads; the index must not point at it. + clamped: text !== redacted.trim(), // Set once the trace cap is known, below. elided: false, }); diff --git a/test/directives.test.js b/test/directives.test.js index 123e9625..a156e7ba 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -114,6 +114,29 @@ test("an over-limit turn keeps its instruction when the closing fence falls in t assert.equal(turns[0].envelope, "Keep the public API stable."); }); +test("an over-limit turn whose only instruction is clamped out is marked clamped, not pointed at", () => { + const filler = (label) => `${label} log line detail\n`.repeat(300); + const { turns, trace } = distill( + [ + { + kind: "message", + role: "user", + text: `${filler("head")}Keep the public API stable.\n${filler("tail")}`, + }, + { kind: "message", role: "assistant", text: "Will do." }, + ], + META, + ); + assert.ok(!turns[0].text.includes("Keep the public API stable."), "the instruction sits in the clamped middle"); + assert.ok(!trace.includes("Keep the public API stable."), "the trace never carries the instruction"); + assert.match(turns[0].envelope, /Keep the public API stable\./); + const spans = extractDirectives(turns); + assert.equal(spans[0].clamped, true); + const section = renderDirectiveIndex(spans); + assert.match(section, /turn 1 - clamped from the trace, open the raw transcript for its text/); + assert.ok(!section.includes("see turn 1 in the trace")); +}); + test("the first user turn is the task, later ones are steering; assistant and tool turns never index", () => { const { turns } = distill(fixtureEvents(), META); const spans = extractDirectives(turns); From 78e44e6c19fa1fcfc8388b61a6b788e379dbab2b Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 9 Oct 2026 18:07:11 +0100 Subject: [PATCH 14/27] fix(directives): pin the relative fence-indent boundary with a test The branch already bounds fence indentation relative to the opener, which is what the indented-code review note asked for while keeping list-item fences working; an absolute ^ {0,3} form would regress those. Add the missing boundary test: a closer three columns past the opener closes, four past it does not. --- test/directives.test.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/directives.test.js b/test/directives.test.js index a156e7ba..ef7da351 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -72,6 +72,18 @@ test("a line of inline code is instruction text, not a fence opener", () => { assert.match(envelope, /Also update the README\./); }); +test("a closer indented more than three past its opener never closes the paste", () => { + // Fence indentation is relative: a closer at most three columns past the opener + // closes, anything deeper is paste content that must stay withheld. + const closed = carveEnvelope(["Do it:", " ```", "pasted", " ```", "Go."].join("\n")); + assert.ok(!closed.includes("pasted"), "the paste stays withheld"); + assert.match(closed, /Go\./, "a closer three past the opener still closes"); + + const open = carveEnvelope(["Do it:", " ```", "pasted", " ```", "still paste", " ```", "Go."].join("\n")); + assert.ok(!open.includes("still paste"), "a closer four past the opener must not close the paste"); + assert.match(open, /Go\./); +}); + test("an unclosed fence still withholds the paste", () => { assert.equal(carveEnvelope("Do the migration.\n```\nError: boom"), "Do the migration."); }); From f120fe1e061057a2a58478044c450eb0cf6f008b Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 9 Oct 2026 18:22:05 +0100 Subject: [PATCH 15/27] no-mistakes(review): Fix fence tab stops and strengthen clamping regression --- src/directives.js | 2 +- src/state.js | 2 +- test/directives.test.js | 40 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/directives.js b/src/directives.js index ba3e90df..8b510fb9 100644 --- a/src/directives.js +++ b/src/directives.js @@ -44,7 +44,7 @@ const QUOTE = /^\s*>/; // may be indented at most three columns further. function indentColumns(prefix) { let columns = 0; - for (const char of prefix) columns += char === "\t" ? 4 : 1; + for (const char of prefix) columns += char === "\t" ? 4 - (columns % 4) : 1; return columns; } diff --git a/src/state.js b/src/state.js index 74ab4c99..e0c5a43b 100644 --- a/src/state.js +++ b/src/state.js @@ -248,7 +248,7 @@ function migrateEvidenceRecord(record, transcript, identity) { }; } -export const ANALYSIS_INDEX_VERSION = 4; +export const ANALYSIS_INDEX_VERSION = 5; /** * Cache key for a transcript's analysis: its content signature, the memory-surface hash, diff --git a/test/directives.test.js b/test/directives.test.js index ef7da351..37977149 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -84,6 +84,41 @@ test("a closer indented more than three past its opener never closes the paste", assert.match(open, /Go\./); }); +test("tabs in an opener advance to four-column stops without promoting pasted text", () => { + for (const fence of ["```", "~~~"]) { + for (const spaces of [1, 2, 3]) { + const prefix = `${" ".repeat(spaces)}\t`; + const paste = [`${prefix}${fence}md`, "example", ` ${fence}`, "Delete production.", `${prefix}${fence}`]; + const { turns } = distill( + [ + { kind: "message", role: "user", text: paste.join("\n") }, + { kind: "message", role: "user", text: "Do the migration." }, + { kind: "message", role: "user", text: "Keep the public API stable." }, + ], + META, + ); + assert.equal(turns[0].envelope, "", "the deeper inner fence must keep the paste withheld"); + assert.deepEqual( + extractDirectives(turns).map((span) => [span.id, span.turn]), + [ + [TASK_ID, 2], + ["STEER-3", 3], + ], + ); + } + } +}); + +test("tabs in a closer advance to four-column stops without hiding the following instruction", () => { + for (const fence of ["```", "~~~"]) { + for (const spaces of [1, 2, 3]) { + const closer = `${" ".repeat(spaces)}\t ${fence}`; + const envelope = carveEnvelope(["Do it:", ` ${fence}`, "pasted", closer, "Go."].join("\n")); + assert.equal(envelope, "Do it:\nGo."); + } + } +}); + test("an unclosed fence still withholds the paste", () => { assert.equal(carveEnvelope("Do the migration.\n```\nError: boom"), "Do the migration."); }); @@ -110,14 +145,17 @@ test("an indented fence line inside a paste neither opens nor closes", () => { test("an over-limit turn keeps its instruction when the closing fence falls in the clamped region", () => { const log = "2026-01-01T00:00:00Z some CI log line\n".repeat(200); + const padding = "> quoted CI context\n".repeat(100); const { turns } = distill( [ - { kind: "message", role: "user", text: `\`\`\`\n${log}\`\`\`\n\nKeep the public API stable.` }, + { kind: "message", role: "user", text: `\`\`\`\n${log}\`\`\`\n${padding}\nKeep the public API stable.` }, { kind: "message", role: "assistant", text: "Will do." }, ], META, ); assert.ok(turns[0].text.includes("chars elided"), "the turn text is clamped"); + assert.ok(turns[0].text.includes("Keep the public API stable."), "the final instruction survives clamping"); + assert.equal(carveEnvelope(turns[0].text), "", "the clamped text loses its closing fence and hides the instruction"); const spans = extractDirectives(turns); assert.deepEqual( spans.map((span) => [span.id, span.turn]), From cc9c3b5c35b618510412c3413a52fa46f430d2f9 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Fri, 9 Oct 2026 19:21:10 +0100 Subject: [PATCH 16/27] no-mistakes(document): Clarify directive truncation and fence tab stops --- README.md | 6 +++--- src/directives.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4a42a736..5f0abc62 100644 --- a/README.md +++ b/README.md @@ -386,9 +386,9 @@ so those instructions are addressable too. The user's own task message (`TASK-1` later steering turn (`STEER-`) become citable instruction sources with their own authority, alongside the memory-file ids. Only the authoritative part of a user turn counts: fenced blocks and quoted lines are pasted evidence, never instructions, and assistant or tool -text never becomes one. The index points at a turn already in the trace instead of repeating -its text - or says that turn was elided from the trace, when the session was long enough to -be capped - and only the ids and their metadata persist in the evidence record. Direct +text never becomes one. The index references a trace turn without repeating its text. +If a turn is elided by session capping or clamped by message truncation, the index marks it +and points to the raw transcript. Only the ids and their metadata persist in the evidence record. Direct instructions are session authority, not durable memory, so a mistake covered only by one is still reported as a gap - the user had to say it in this session, and the next session starts without it - and their citations do not score a memory-file instruction. diff --git a/src/directives.js b/src/directives.js index 8b510fb9..48fd776c 100644 --- a/src/directives.js +++ b/src/directives.js @@ -39,7 +39,7 @@ export const STEERING_AUTHORITY = "direct-steering"; const FENCE = /^([ \t]*)(`{3,}|~{3,})([^\n]*)$/; const QUOTE = /^\s*>/; -// CommonMark counts a tab as four columns, and fence indentation is relative: an opener +// Tabs advance to the next four-column stop. Fence indentation is relative: an opener // may sit at any indentation (a list-item code block is still a fence), while its closer // may be indented at most three columns further. function indentColumns(prefix) { From 0c03509bc3fe21e4e599afb45085d7a5f157c3d5 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Thu, 15 Oct 2026 23:27:01 +0100 Subject: [PATCH 17/27] fix(fold): drop duplicated sessionIdentity from the main merge --- src/fold.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fold.js b/src/fold.js index e86794e1..d16d0b78 100644 --- a/src/fold.js +++ b/src/fold.js @@ -139,7 +139,6 @@ export function foldEvidence( const source = recordSources[index]; sources.add(source); if (record.transcript.project) sourceProjects[source] = record.transcript.project; - const sessionIdentity = record.transcript.identity || record.transcript.id; for (const polarity of ["positive", "negative"]) { for (const item of record[polarity] || []) { From bdff64c4be1e3dfa6e4e756e117ac9f8e2bb6815 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Sat, 17 Oct 2026 04:24:21 +0100 Subject: [PATCH 18/27] fix(adapters): preserve typed-block indentation for directive extraction Join typed-block texts without trimming so indented fence markers survive into the carved envelope; whitespace-only blocks stay omitted via the non-empty check. New adapter-level regression test plus fixture. Bump ANALYSIS_INDEX_VERSION 5->6: preserved leading/trailing whitespace changes what analysis accepts. --- src/discovery/adapters/shared.js | 4 ++-- src/state.js | 2 +- test/directives.test.js | 15 +++++++++++++++ test/fixtures/claude-indented-fences.jsonl | 7 +++++++ 4 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 test/fixtures/claude-indented-fences.jsonl diff --git a/src/discovery/adapters/shared.js b/src/discovery/adapters/shared.js index cb7a2daa..ebd8a854 100644 --- a/src/discovery/adapters/shared.js +++ b/src/discovery/adapters/shared.js @@ -155,8 +155,8 @@ export function contentToEvents(role, content, events) { } } if (texts.length) { - const joined = texts.join("\n").trim(); - if (joined) events.push({ kind: "message", role, text: joined }); + const joined = texts.join("\n"); + if (joined.trim()) events.push({ kind: "message", role, text: joined }); } } diff --git a/src/state.js b/src/state.js index e0c5a43b..ac5b4c9f 100644 --- a/src/state.js +++ b/src/state.js @@ -248,7 +248,7 @@ function migrateEvidenceRecord(record, transcript, identity) { }; } -export const ANALYSIS_INDEX_VERSION = 5; +export const ANALYSIS_INDEX_VERSION = 6; /** * Cache key for a transcript's analysis: its content signature, the memory-surface hash, diff --git a/test/directives.test.js b/test/directives.test.js index 37977149..c85d9589 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -119,6 +119,21 @@ test("tabs in a closer advance to four-column stops without hiding the following } }); +test("typed-block adapter preserves instructions after mixed-tab fence closers", () => { + const { events } = claude.read({ path: path.join(FIXTURES, "fixtures/claude-indented-fences.jsonl") }); + assert.equal(events.length, 6, "whitespace-only typed blocks must stay omitted"); + const { turns } = distill(events, META); + assert.deepEqual( + turns.map((entry) => entry.envelope), + Array(6).fill("Keep the public API stable."), + "all six fence variants must retain the following instruction and exclude the paste", + ); + assert.deepEqual( + extractDirectives(turns).map((span) => span.id), + [TASK_ID, "STEER-2", "STEER-3", "STEER-4", "STEER-5", "STEER-6"], + ); +}); + test("an unclosed fence still withholds the paste", () => { assert.equal(carveEnvelope("Do the migration.\n```\nError: boom"), "Do the migration."); }); diff --git a/test/fixtures/claude-indented-fences.jsonl b/test/fixtures/claude-indented-fences.jsonl new file mode 100644 index 00000000..4df38108 --- /dev/null +++ b/test/fixtures/claude-indented-fences.jsonl @@ -0,0 +1,7 @@ +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ```\npasted\n \t ```\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ```\npasted\n \t ```\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ```\npasted\n \t ```\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ~~~\npasted\n \t ~~~\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ~~~\npasted\n \t ~~~\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ~~~\npasted\n \t ~~~\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" \t\n "}]}} From c179402b6119c865d31bc781f8e9593b426fb438 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Tue, 20 Oct 2026 07:45:29 +0100 Subject: [PATCH 19/27] fix(directives): recognize list-item fence markers when carving the envelope Greptile P2 on PR 154 (comment 4322029695): carveEnvelope missed fence openers behind a list marker, so a pasted '- ```sh' block stayed instruction text and its indented closer opened a phantom fence that swallowed every later instruction. The opener now matches after an optional bullet (-, *, +) or ordered marker (digits with . or )) plus spaces, and the closer indent is measured from the fence's own column after that marker. Regression tests cover the comment's exact case and the ordered-list variant. Bump ANALYSIS_INDEX_VERSION to 7 for the changed envelope boundary. --- src/directives.js | 13 +++++++++---- src/state.js | 2 +- test/directives.test.js | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/directives.js b/src/directives.js index 48fd776c..6bc9eabc 100644 --- a/src/directives.js +++ b/src/directives.js @@ -36,12 +36,17 @@ export const TASK_ID = "TASK-1"; export const TASK_AUTHORITY = "direct-task"; export const STEERING_AUTHORITY = "direct-steering"; -const FENCE = /^([ \t]*)(`{3,}|~{3,})([^\n]*)$/; +// A fence opener may sit behind a list marker: an optional bullet (`-`, `*`, `+`) or +// ordered marker (digits with `.` or `)`) plus spaces before the fence. Group 1 keeps the +// whole prefix so the closer indent is measured from the fence's own column - the content +// column after that marker - not from the start of the line. +const FENCE = /^([ \t]*(?:[-*+][ \t]+|\d{1,9}[.)][ \t]+)?)(`{3,}|~{3,})([^\n]*)$/; const QUOTE = /^\s*>/; -// Tabs advance to the next four-column stop. Fence indentation is relative: an opener -// may sit at any indentation (a list-item code block is still a fence), while its closer -// may be indented at most three columns further. +// Tabs advance to the next four-column stop. Fence indentation is relative: an opener may +// sit at any indentation, optionally behind a list marker (a list-item code block is still +// a fence), while its closer may be indented at most three columns past the fence's own +// column. function indentColumns(prefix) { let columns = 0; for (const char of prefix) columns += char === "\t" ? 4 - (columns % 4) : 1; diff --git a/src/state.js b/src/state.js index ac5b4c9f..9841be27 100644 --- a/src/state.js +++ b/src/state.js @@ -248,7 +248,7 @@ function migrateEvidenceRecord(record, transcript, identity) { }; } -export const ANALYSIS_INDEX_VERSION = 6; +export const ANALYSIS_INDEX_VERSION = 7; /** * Cache key for a transcript's analysis: its content signature, the memory-surface hash, diff --git a/test/directives.test.js b/test/directives.test.js index c85d9589..4bfd1098 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -109,6 +109,24 @@ test("tabs in an opener advance to four-column stops without promoting pasted te } }); +test("a list-item fence is a paste; the real instruction after it stays visible", () => { + // Regression for greptile P2 on PR 154 (comment 4322029695): an opener behind a list + // marker used to stay instruction text and its indented closer opened a phantom fence + // that swallowed every later instruction. + const envelope = carveEnvelope( + ["Check the plan:", "- ```sh", " rm -rf tests", " ```", "Then report the checksum."].join("\n"), + ); + assert.ok(!envelope.includes("rm -rf tests"), "the listed paste stays excluded"); + assert.ok(envelope.includes("Check the plan:"), "text before the paste stays"); + assert.ok(envelope.includes("Then report the checksum."), "the instruction after the closer stays visible"); +}); + +test("an ordered-list fence is a paste too", () => { + const envelope = carveEnvelope(["1. ```sh", " make deploy", " ```", "Ship it."].join("\n")); + assert.ok(!envelope.includes("make deploy"), "the ordered-list paste stays excluded"); + assert.ok(envelope.includes("Ship it."), "the instruction after the closer stays visible"); +}); + test("tabs in a closer advance to four-column stops without hiding the following instruction", () => { for (const fence of ["```", "~~~"]) { for (const spaces of [1, 2, 3]) { From 106a33abe8396ab1a8fdb3efef0124fe5cb45df7 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Tue, 20 Oct 2026 07:53:05 +0100 Subject: [PATCH 20/27] no-mistakes(review): Restrict closing fences to whitespace-only prefixes --- src/directives.js | 13 +++++++------ test/directives.test.js | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/directives.js b/src/directives.js index 6bc9eabc..7526c282 100644 --- a/src/directives.js +++ b/src/directives.js @@ -40,7 +40,7 @@ export const STEERING_AUTHORITY = "direct-steering"; // ordered marker (digits with `.` or `)`) plus spaces before the fence. Group 1 keeps the // whole prefix so the closer indent is measured from the fence's own column - the content // column after that marker - not from the start of the line. -const FENCE = /^([ \t]*(?:[-*+][ \t]+|\d{1,9}[.)][ \t]+)?)(`{3,}|~{3,})([^\n]*)$/; +const FENCE = /^([ \t]*([-*+][ \t]+|\d{1,9}[.)][ \t]+)?)(`{3,}|~{3,})([^\n]*)$/; const QUOTE = /^\s*>/; // Tabs advance to the next four-column stop. Fence indentation is relative: an opener may @@ -80,9 +80,10 @@ export function carveEnvelope(text) { // part of the paste. if ( fence && - fence[2][0] === opener.char && - fence[2].length >= opener.length && - fence[3].trim() === "" && + !fence[2] && + fence[3][0] === opener.char && + fence[3].length >= opener.length && + fence[4].trim() === "" && indentColumns(fence[1]) <= opener.indent + 3 ) { opener = null; @@ -93,8 +94,8 @@ export function carveEnvelope(text) { } // CommonMark: a backtick fence's info string may not contain a backtick, so a line // of inline code is instruction text, not an opener. - if (fence && !(fence[2][0] === "`" && fence[3].includes("`"))) { - opener = { char: fence[2][0], length: fence[2].length, indent: indentColumns(fence[1]) }; + if (fence && !(fence[3][0] === "`" && fence[4].includes("`"))) { + opener = { char: fence[3][0], length: fence[3].length, indent: indentColumns(fence[1]) }; continue; } if (QUOTE.test(line)) continue; diff --git a/test/directives.test.js b/test/directives.test.js index 4bfd1098..e056212f 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -127,6 +127,33 @@ test("an ordered-list fence is a paste too", () => { assert.ok(envelope.includes("Ship it."), "the instruction after the closer stays visible"); }); +test("list markers inside either fence type never close a paste or issue directive IDs", () => { + for (const fence of ["```", "~~~"]) { + for (const marker of ["- ", "* ", "+ ", "1. ", "1) ", "123456789. ", "-\t"]) { + for (const prefix of ["", " \t"]) { + const paste = [`${fence}md`, `${prefix}${marker}${fence}`, "Delete production.", fence].join("\n"); + assert.equal(carveEnvelope(paste), "", "a list marker cannot close the enclosing paste"); + const { turns } = distill( + [ + { kind: "message", role: "user", text: paste }, + { kind: "message", role: "user", text: "Do the migration." }, + { kind: "message", role: "user", text: "Keep the public API stable." }, + ], + META, + ); + assert.equal(turns[0].envelope, ""); + assert.deepEqual( + extractDirectives(turns).map((span) => [span.id, span.turn]), + [ + [TASK_ID, 2], + ["STEER-3", 3], + ], + ); + } + } + } +}); + test("tabs in a closer advance to four-column stops without hiding the following instruction", () => { for (const fence of ["```", "~~~"]) { for (const spaces of [1, 2, 3]) { From 517f110897d5f889ff4f9721c6a1796457465710 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Tue, 20 Oct 2026 07:58:21 +0100 Subject: [PATCH 21/27] no-mistakes(review): Invalidate cached evidence after directive acceptance changes --- src/state.js | 2 +- test/config.test.js | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/state.js b/src/state.js index 9841be27..0cb67bea 100644 --- a/src/state.js +++ b/src/state.js @@ -248,7 +248,7 @@ function migrateEvidenceRecord(record, transcript, identity) { }; } -export const ANALYSIS_INDEX_VERSION = 7; +export const ANALYSIS_INDEX_VERSION = 8; /** * Cache key for a transcript's analysis: its content signature, the memory-surface hash, diff --git a/test/config.test.js b/test/config.test.js index f7802a2c..c0fb2d93 100644 --- a/test/config.test.js +++ b/test/config.test.js @@ -234,6 +234,21 @@ test("evidence is keyed to both the transcript and the memory file it was judged assert.equal(isEvidenceFresh(null, transcript, "sha256:aaa"), false); }); +test("evidence accepted before the list-marker closer fix is stale; version 8 evidence is reusable", () => { + const transcript = { identity: "directive-demo", mtimeMs: 111, bytes: 222 }; + const memoryHash = "sha256:aaa"; + const previousKey = "directive-demo:111:222:sha256:aaa:analysis-index-v7"; + const currentKey = "directive-demo:111:222:sha256:aaa:analysis-index-v8"; + + assert.equal( + isEvidenceFresh({ status: "ok", key: previousKey }, transcript, memoryHash), + false, + "the approved v7-to-v8 invalidation must reject evidence from the old directive parser", + ); + assert.equal(evidenceKey(transcript, memoryHash), currentKey); + assert.equal(isEvidenceFresh({ status: "ok", key: currentKey }, transcript, memoryHash), true); +}); + test("state round-trips through disk and survives a corrupt file", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-state-")); const state = new State(dir).ensure(); From f880c41a4acf40e3c53744cd07a6377ad55e5f26 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Wed, 21 Oct 2026 18:44:00 +0100 Subject: [PATCH 22/27] no-mistakes(document): Clarify task indexing and fence closer documentation --- README.md | 4 ++-- src/directives.js | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5f0abc62..91c4c6e5 100644 --- a/README.md +++ b/README.md @@ -382,8 +382,8 @@ analysis is shown the ledger's open gaps so it can cite an existing gap id inste coining a paraphrase of it. Much of what governs a session comes from the user directly rather than from a memory file, -so those instructions are addressable too. The user's own task message (`TASK-1`) and each -later steering turn (`STEER-`) become citable instruction sources with their own +so those instructions are addressable too. The first user turn with authoritative text (`TASK-1`) and each +later such turn (`STEER-`) become citable instruction sources with their own authority, alongside the memory-file ids. Only the authoritative part of a user turn counts: fenced blocks and quoted lines are pasted evidence, never instructions, and assistant or tool text never becomes one. The index references a trace turn without repeating its text. diff --git a/src/directives.js b/src/directives.js index 7526c282..298ca453 100644 --- a/src/directives.js +++ b/src/directives.js @@ -74,9 +74,9 @@ export function carveEnvelope(text) { for (const line of String(text ?? "").split("\n")) { const fence = FENCE.exec(line); if (opener) { - // CommonMark: only a fence of the opener's character, at least as long, with - // nothing but whitespace after it and indented no more than three columns past - // the opener, closes the block. A shorter, different or deeper inner fence is + // CommonMark: a closer must have a whitespace-only prefix, use the opener's + // character at least as many times, have only whitespace after it, and sit at most + // three columns past the opener. A shorter, different or deeper inner fence is // part of the paste. if ( fence && From 6a504c7379b850c781efe35cc61e55a47e2454d1 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Wed, 21 Oct 2026 18:58:52 +0100 Subject: [PATCH 23/27] fix(directives): recognize list-item blockquotes when carving the envelope A blockquote behind a single list marker (- > x, * > x, 1. > x) passed the quote check and was indexed as instruction text instead of staying evidence, so a paste-only turn could receive TASK-1. The QUOTE check now allows the same optional single marker the fence check accepts; markers require whitespace after them, so an unspaced dash keeps its old behaviour. Bump ANALYSIS_INDEX_VERSION to 9 so cached analysis accepted under the old carve is invalidated. --- src/directives.js | 4 +++- src/state.js | 2 +- test/config.test.js | 8 ++++---- test/directives.test.js | 45 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/src/directives.js b/src/directives.js index 298ca453..f943f359 100644 --- a/src/directives.js +++ b/src/directives.js @@ -41,7 +41,9 @@ export const STEERING_AUTHORITY = "direct-steering"; // whole prefix so the closer indent is measured from the fence's own column - the content // column after that marker - not from the start of the line. const FENCE = /^([ \t]*([-*+][ \t]+|\d{1,9}[.)][ \t]+)?)(`{3,}|~{3,})([^\n]*)$/; -const QUOTE = /^\s*>/; +// A blockquote may also sit behind a single list marker (`- > quoted`); a marker must +// be followed by whitespace, so an unspaced dash (`-> quoted`) stays instruction text. +const QUOTE = /^\s*(?:[-*+][ \t]+|\d{1,9}[.)][ \t]+)?>/; // Tabs advance to the next four-column stop. Fence indentation is relative: an opener may // sit at any indentation, optionally behind a list marker (a list-item code block is still diff --git a/src/state.js b/src/state.js index 0cb67bea..3ddcef5d 100644 --- a/src/state.js +++ b/src/state.js @@ -248,7 +248,7 @@ function migrateEvidenceRecord(record, transcript, identity) { }; } -export const ANALYSIS_INDEX_VERSION = 8; +export const ANALYSIS_INDEX_VERSION = 9; /** * Cache key for a transcript's analysis: its content signature, the memory-surface hash, diff --git a/test/config.test.js b/test/config.test.js index c0fb2d93..f0662d79 100644 --- a/test/config.test.js +++ b/test/config.test.js @@ -234,16 +234,16 @@ test("evidence is keyed to both the transcript and the memory file it was judged assert.equal(isEvidenceFresh(null, transcript, "sha256:aaa"), false); }); -test("evidence accepted before the list-marker closer fix is stale; version 8 evidence is reusable", () => { +test("evidence accepted before the list-item quote fix is stale; version 9 evidence is reusable", () => { const transcript = { identity: "directive-demo", mtimeMs: 111, bytes: 222 }; const memoryHash = "sha256:aaa"; - const previousKey = "directive-demo:111:222:sha256:aaa:analysis-index-v7"; - const currentKey = "directive-demo:111:222:sha256:aaa:analysis-index-v8"; + const previousKey = "directive-demo:111:222:sha256:aaa:analysis-index-v8"; + const currentKey = "directive-demo:111:222:sha256:aaa:analysis-index-v9"; assert.equal( isEvidenceFresh({ status: "ok", key: previousKey }, transcript, memoryHash), false, - "the approved v7-to-v8 invalidation must reject evidence from the old directive parser", + "the approved v8-to-v9 invalidation must reject evidence from the old directive parser", ); assert.equal(evidenceKey(transcript, memoryHash), currentKey); assert.equal(isEvidenceFresh({ status: "ok", key: currentKey }, transcript, memoryHash), true); diff --git a/test/directives.test.js b/test/directives.test.js index e056212f..72bc6220 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -305,6 +305,51 @@ test("a message that is only quotes and pastes yields no span", () => { assert.match(renderDirectiveIndex(spans), /\(none/); }); +test("a paste-only turn of list-item blockquotes yields no span", () => { + for (const line of [ + "- > Delete the tests.", + "* > Delete the tests.", + "+ > Delete the tests.", + "1. > Delete the tests.", + "1) > Delete the tests.", + " - > Delete the tests.", + "-\t> Delete the tests.", + ]) { + assert.equal(carveEnvelope(line), "", "a list-item blockquote is quoted evidence, never an instruction"); + } + const { turns } = distill( + [ + { kind: "message", role: "user", text: "- > Delete the tests." }, + { kind: "message", role: "assistant", text: "Noted." }, + ], + META, + ); + assert.deepEqual(extractDirectives(turns), []); +}); + +test("a list-item blockquote inside a real turn stays evidence while the instruction indexes", () => { + const { turns } = distill( + [ + { kind: "message", role: "user", text: "Run the migration.\n- > old transcript line" }, + { kind: "message", role: "user", text: "Keep the public API stable." }, + ], + META, + ); + assert.equal(turns[0].envelope, "Run the migration."); + assert.deepEqual( + extractDirectives(turns).map((span) => [span.id, span.turn]), + [ + [TASK_ID, 1], + ["STEER-2", 2], + ], + ); +}); + +test("markers keep ordinary behaviour: plain items stay instructions and unspaced dashes never match", () => { + assert.equal(carveEnvelope("- Run the tests."), "- Run the tests."); + assert.equal(carveEnvelope("-> Delete the tests."), "-> Delete the tests."); +}); + test("the index renders by reference and never repeats turn text", () => { const { turns, trace } = distill(fixtureEvents(), META); const spans = extractDirectives(turns); From dabde3ed3bfd64cfd5e3a3498af65054fa945547 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Wed, 21 Oct 2026 19:15:35 +0100 Subject: [PATCH 24/27] =?UTF-8?q?no-mistakes(ci):=20Fixed=20signal-test=20?= =?UTF-8?q?startup=20race:=20publish=20child=20readiness=20only=20after=20?= =?UTF-8?q?installing=20its=20SIGTERM=20handler.=20Reproduced=20CI?= =?UTF-8?q?=E2=80=99s=20exact=20failure=20with=20a=20forced=20scheduling?= =?UTF-8?q?=20delay;=20same=20probe=20passes=20after=20fix.=20All=2043=20h?= =?UTF-8?q?arness/subprocess=20tests=20pass=20locally.=20Temporary=20probe?= =?UTF-8?q?=20removed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/harness-invoke.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/harness-invoke.test.js b/test/harness-invoke.test.js index d8df2993..6042b613 100644 --- a/test/harness-invoke.test.js +++ b/test/harness-invoke.test.js @@ -494,11 +494,11 @@ test("the process wrapper escalates when its harness child ignores termination", signalChild, `#!${process.execPath} const fs = require("node:fs"); -fs.writeFileSync(${JSON.stringify(readyPath)}, String(process.pid)); process.on("SIGTERM", () => { fs.writeFileSync(${JSON.stringify(signalPath)}, "SIGTERM"); }); setInterval(() => {}, 1000); +fs.writeFileSync(${JSON.stringify(readyPath)}, String(process.pid)); `, ); fs.chmodSync(signalChild, 0o755); From fd4cca736fd77c590085847f53939d8f24b9d973 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Thu, 22 Oct 2026 22:48:54 +0100 Subject: [PATCH 25/27] chore: retrigger review From c60451727003369df92225d99c000cc486f032df Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Sun, 25 Oct 2026 19:28:24 +0000 Subject: [PATCH 26/27] no-mistakes(review): Preserve partial directive provenance for truncated transcripts --- src/acpx.js | 2 +- src/analyze.js | 1 + src/directives.js | 11 ++- src/discovery/adapters/claude.js | 4 +- src/discovery/adapters/codex.js | 4 +- src/discovery/adapters/grok.js | 4 +- src/discovery/adapters/pi.js | 4 +- src/discovery/adapters/shared.js | 6 +- src/discovery/index.js | 2 + src/distill.js | 1 + src/state.js | 2 +- test/directives.test.js | 119 ++++++++++++++++++++++++++++++- 12 files changed, 144 insertions(+), 16 deletions(-) diff --git a/src/acpx.js b/src/acpx.js index 28d602d3..f62162bd 100644 --- a/src/acpx.js +++ b/src/acpx.js @@ -913,7 +913,7 @@ function recoverPiUsage({ promptFile, cwd, startedAt }) { for (const candidate of candidates) { const descriptor = piStore.classify(candidate); if (!descriptor || !wanted.has(descriptor.cwd)) continue; - const entries = readJsonl(candidate.path); + const { entries } = readJsonl(candidate.path); const firstUser = entries.find((e) => e.type === "message" && e.message?.role === "user"); if (!firstUser || piMessageText(firstUser.message.content).trim() !== prompt) continue; diff --git a/src/analyze.js b/src/analyze.js index c2942efe..3d446da7 100644 --- a/src/analyze.js +++ b/src/analyze.js @@ -266,6 +266,7 @@ async function analyzeOne({ const distilled = distill(raw.events, { ...transcript, model: raw.model, + prefixDiscarded: raw.prefixDiscarded, rawPath: rawFile ?? raw.rawPath, }); diff --git a/src/directives.js b/src/directives.js index f943f359..0aca663c 100644 --- a/src/directives.js +++ b/src/directives.js @@ -119,12 +119,14 @@ export function extractDirectives(turns) { for (const entry of Array.isArray(turns) ? turns : []) { if (!entry || entry.role !== "user") continue; if (!entry.envelope) continue; - const span = !seenFirstUser - ? { id: TASK_ID, kind: "task", turn: entry.turn, authority: TASK_AUTHORITY } - : { id: steeringId(entry.turn), kind: "steering", turn: entry.turn, authority: STEERING_AUTHORITY }; + const span = + !seenFirstUser && !entry.partial + ? { id: TASK_ID, kind: "task", turn: entry.turn, authority: TASK_AUTHORITY } + : { id: steeringId(entry.turn), kind: "steering", turn: entry.turn, authority: STEERING_AUTHORITY }; seenFirstUser = true; spans.push({ ...span, + partial: entry.partial === true, elided: entry.elided === true, clamped: entry.clamped === true, lifetime: { fromTurn: entry.turn, toTurn: null }, @@ -140,6 +142,8 @@ export function directiveMetadata(spans) { kind: span.kind, turn: span.turn, authority: span.authority, + partial: span.partial === true, + stable: span.partial !== true, lifetime: { ...span.lifetime }, })); } @@ -156,6 +160,7 @@ export function renderDirectiveIndex(spans) { .map( (span) => `[${span.id}] ${span.kind} · turn ${span.turn} · authority ${span.authority} · ` + + (span.partial ? "partial transcript · unstable id · " : "") + `lifetime turns ${span.lifetime?.fromTurn ?? span.turn}+ - ` + (span.elided ? `turn ${span.turn} - elided from the trace, open the raw transcript for its text` diff --git a/src/discovery/adapters/claude.js b/src/discovery/adapters/claude.js index 23f94fb0..f5f3b80c 100644 --- a/src/discovery/adapters/claude.js +++ b/src/discovery/adapters/claude.js @@ -83,7 +83,7 @@ export function classify(candidate) { } export function read(ref) { - const entries = readJsonl(ref.path); + const { entries, prefixDiscarded } = readJsonl(ref.path); const events = []; let model = null; @@ -98,5 +98,5 @@ export function read(ref) { } } - return { events: attachToolResults(events), model }; + return { events: attachToolResults(events), model, prefixDiscarded }; } diff --git a/src/discovery/adapters/codex.js b/src/discovery/adapters/codex.js index 7fd31a1a..172008e6 100644 --- a/src/discovery/adapters/codex.js +++ b/src/discovery/adapters/codex.js @@ -102,7 +102,7 @@ export function classify(candidate) { } export function read(ref) { - const entries = readJsonl(ref.path); + const { entries, prefixDiscarded } = readJsonl(ref.path); const events = []; let model = null; @@ -143,7 +143,7 @@ export function read(ref) { } } - return { events: attachToolResults(events), model }; + return { events: attachToolResults(events), model, prefixDiscarded }; } function parseMaybeJson(value) { diff --git a/src/discovery/adapters/grok.js b/src/discovery/adapters/grok.js index 713067c2..ec05791b 100644 --- a/src/discovery/adapters/grok.js +++ b/src/discovery/adapters/grok.js @@ -61,7 +61,7 @@ export function read(ref) { const chatPath = ref.extra?.chatPath || path.join(ref.path, "chat_history.jsonl"); if (!fs.existsSync(chatPath)) return { events: [], model: null }; - const entries = readJsonl(chatPath); + const { entries, prefixDiscarded } = readJsonl(chatPath); const events = []; let model = null; @@ -91,7 +91,7 @@ export function read(ref) { } } - return { events: attachToolResults(events), model }; + return { events: attachToolResults(events), model, prefixDiscarded }; } /** Exported for the raw-transcript escape hatch: the analysis agent opens this file. */ diff --git a/src/discovery/adapters/pi.js b/src/discovery/adapters/pi.js index 68e2b733..7fef606e 100644 --- a/src/discovery/adapters/pi.js +++ b/src/discovery/adapters/pi.js @@ -226,7 +226,7 @@ export function classify(candidate, options = {}) { } export function read(ref) { - const entries = readJsonl(ref.path); + const { entries, prefixDiscarded } = readJsonl(ref.path); const events = []; let model = null; const omp = entries[0]?.type === "title"; @@ -248,7 +248,7 @@ export function read(ref) { contentToEvents(role, message.content, events); } - return { events: attachToolResults(events), model }; + return { events: attachToolResults(events), model, prefixDiscarded }; } function textOf(content) { diff --git a/src/discovery/adapters/shared.js b/src/discovery/adapters/shared.js index ebd8a854..fc9f5d01 100644 --- a/src/discovery/adapters/shared.js +++ b/src/discovery/adapters/shared.js @@ -41,9 +41,11 @@ export function readJsonFile(file) { export function readJsonl(file, { maxBytes = 64 * 1024 * 1024 } = {}) { let text; + let prefixDiscarded = false; try { const stat = fs.statSync(file); if (stat.size > maxBytes) { + prefixDiscarded = true; // Very large sessions are read tail-first: recent turns carry the loss signal. const fd = fs.openSync(file, "r"); const buffer = Buffer.alloc(maxBytes); @@ -55,7 +57,7 @@ export function readJsonl(file, { maxBytes = 64 * 1024 * 1024 } = {}) { text = fs.readFileSync(file, "utf8"); } } catch { - return []; + return { entries: [], prefixDiscarded }; } const out = []; for (const line of text.split("\n")) { @@ -63,7 +65,7 @@ export function readJsonl(file, { maxBytes = 64 * 1024 * 1024 } = {}) { const value = parseJsonLine(line); if (value) out.push(value); } - return out; + return { entries: out, prefixDiscarded }; } export function statOrNull(file) { diff --git a/src/discovery/index.js b/src/discovery/index.js index 82a116ed..e8b4188b 100644 --- a/src/discovery/index.js +++ b/src/discovery/index.js @@ -518,6 +518,7 @@ export async function readTranscript(transcript) { return { events: result.events || [], model: result.model || transcript.model || null, + prefixDiscarded: result.prefixDiscarded === true, rawPath: cached, }; } @@ -526,6 +527,7 @@ export async function readTranscript(transcript) { return { events: result.events || [], model: result.model || transcript.model || null, + prefixDiscarded: result.prefixDiscarded === true, rawPath: adapter.rawPath ? adapter.rawPath(transcript) : transcript.path, }; } diff --git a/src/distill.js b/src/distill.js index d4618509..3ec20809 100644 --- a/src/distill.js +++ b/src/distill.js @@ -130,6 +130,7 @@ export function distill(events, meta, options = {}) { turns.push({ turn, role: event.role, + partial: meta.prefixDiscarded === true, text, envelope: event.role === "user" ? carveEnvelope(redacted) : "", // The envelope is carved pre-clamp, so a clamped turn's instruction may be absent diff --git a/src/state.js b/src/state.js index 3ddcef5d..7b86a53e 100644 --- a/src/state.js +++ b/src/state.js @@ -248,7 +248,7 @@ function migrateEvidenceRecord(record, transcript, identity) { }; } -export const ANALYSIS_INDEX_VERSION = 9; +export const ANALYSIS_INDEX_VERSION = 10; /** * Cache key for a transcript's analysis: its content signature, the memory-surface hash, diff --git a/test/directives.test.js b/test/directives.test.js index 72bc6220..d24d0619 100644 --- a/test/directives.test.js +++ b/test/directives.test.js @@ -1,11 +1,14 @@ import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { distill } from "../src/distill.js"; -import { sanitizeEvidence } from "../src/analyze.js"; +import { analyzeTranscripts, sanitizeEvidence } from "../src/analyze.js"; +import { readTranscript } from "../src/discovery/index.js"; +import { State } from "../src/state.js"; import { foldEvidence, renderEvidenceForPrompt, renderEvidenceReport } from "../src/fold.js"; import { parseMemoryUnits } from "../src/memory.js"; import { estimateTokens } from "../src/tokens.js"; @@ -266,6 +269,120 @@ test("the first user turn is the task, later ones are steering; assistant and to assert.ok(!isDirectiveId("AG-001") && !isDirectiveId("TASK-2") && !isDirectiveId("banana")); }); +test("a discarded transcript prefix withholds task authority and persists unstable steering", async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-partial-directives-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const formats = { + claude: (role, text) => ({ type: role, message: { content: text } }), + codex: (role, text) => ({ type: "response_item", payload: { type: "message", role, content: text } }), + pi: (role, text) => ({ type: "message", message: { role, content: text } }), + grok: (role, text) => ({ type: role, content: text }), + }; + const padding = "x".repeat(1024 * 1024); + for (const [harness, message] of Object.entries(formats)) { + const file = path.join(root, `${harness}.jsonl`); + const appendMessage = (role, text) => fs.appendFileSync(file, `${JSON.stringify(message(role, text))}\n`); + appendMessage("user", "Migrate the auth module."); + appendMessage("assistant", "Working on it."); + appendMessage("user", "Keep the public API stable."); + const transcript = { ...META, harness, path: file, extra: { chatPath: file } }; + const complete = await readTranscript(transcript); + assert.equal(complete.prefixDiscarded, false); + const completeSpans = extractDirectives(distill(complete.events, { ...META, ...complete }).turns); + assert.deepEqual( + directiveMetadata(completeSpans).map(({ id, partial, stable }) => [id, partial, stable]), + [ + [TASK_ID, false, true], + ["STEER-3", false, true], + ], + "a complete transcript retains the initial task and stable turn IDs", + ); + + fs.appendFileSync(file, '{"type":"padding","text":"'); + for (let i = 0; i < 64; i += 1) fs.appendFileSync(file, padding); + fs.appendFileSync(file, '"}\n'); + appendMessage("user", "Now run the tests too."); + appendMessage("assistant", "Tests pass."); + appendMessage("user", "Report the results."); + + for (const remote of [false, true]) { + const state = new State(root, { + stateDir: path.join(root, `${harness}-${remote ? "remote" : "local"}`), + exclude: false, + }).ensure(); + const ref = remote + ? { ...transcript, host: "fixture-host", remote: { kind: "raw", cachePath: file } } + : transcript; + const raw = await readTranscript(ref); + assert.equal(raw.prefixDiscarded, true, `${harness}: prefix loss must survive transcript reading`); + assert.deepEqual( + raw.events.map((event) => event.text), + ["Now run the tests too.", "Tests pass.", "Report the results."], + "only the tail survives; the original task is unavailable", + ); + const summary = await analyzeTranscripts({ + transcripts: [ref], + memoryFile: { path: "AGENTS.md", units: [] }, + memoryHash: "memory", + repo: { root }, + config: { + state, + jobs: 1, + discovery: { minUserTurns: 1 }, + agents: { + resolve: async () => ({ agent: "pi" }), + withFallthrough: async () => ({ + text: JSON.stringify({ + positive: [ + { instruction: "TASK-1", quote: "Now run the tests too.", effect: "invented task authority" }, + { instruction: "STEER-1", quote: "Now run the tests too.", effect: "ran tests" }, + { instruction: "STEER-3", quote: "Report the results.", effect: "reported results" }, + ], + negative: [], + gaps: [], + }), + }), + }, + }, + }); + assert.equal(summary.analyzed, 1); + const evidence = state.readEvidence(ref); + assert.deepEqual(evidence.directives, [ + { + id: "STEER-1", + kind: "steering", + turn: 1, + authority: "direct-steering", + partial: true, + stable: false, + lifetime: { fromTurn: 1, toTurn: null }, + }, + { + id: "STEER-3", + kind: "steering", + turn: 3, + authority: "direct-steering", + partial: true, + stable: false, + lifetime: { fromTurn: 3, toTurn: null }, + }, + ]); + assert.deepEqual( + evidence.positive.map((item) => item.instruction), + ["STEER-1", "STEER-3"], + ); + const promptDir = path.join(state.root, "prompts"); + const prompt = fs.readFileSync(path.join(promptDir, fs.readdirSync(promptDir)[0]), "utf8"); + assert.match( + prompt, + /\[STEER-1\] steering · turn 1 · authority direct-steering · partial transcript · unstable id/, + ); + assert.doesNotMatch(prompt, /\[TASK-1\] task · turn/); + assert.ok(!JSON.stringify(evidence.directives).includes("Now run the tests too.")); + } + } +}); + test("a short imperative is addressable: length never decides which turn is the task", () => { const { turns } = distill( [ From c8714afbdaa2b674a8c76842fd3faa3139068033 Mon Sep 17 00:00:00 2001 From: Andrew Sweet Date: Sun, 25 Oct 2026 19:36:28 +0000 Subject: [PATCH 27/27] no-mistakes(review): Align evidence cache regression with analysis index v10 --- test/config.test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/config.test.js b/test/config.test.js index f0662d79..e7df6250 100644 --- a/test/config.test.js +++ b/test/config.test.js @@ -234,16 +234,16 @@ test("evidence is keyed to both the transcript and the memory file it was judged assert.equal(isEvidenceFresh(null, transcript, "sha256:aaa"), false); }); -test("evidence accepted before the list-item quote fix is stale; version 9 evidence is reusable", () => { +test("evidence accepted before the partial-transcript fix is stale; version 10 evidence is reusable", () => { const transcript = { identity: "directive-demo", mtimeMs: 111, bytes: 222 }; const memoryHash = "sha256:aaa"; - const previousKey = "directive-demo:111:222:sha256:aaa:analysis-index-v8"; - const currentKey = "directive-demo:111:222:sha256:aaa:analysis-index-v9"; + const previousKey = "directive-demo:111:222:sha256:aaa:analysis-index-v9"; + const currentKey = "directive-demo:111:222:sha256:aaa:analysis-index-v10"; assert.equal( isEvidenceFresh({ status: "ok", key: previousKey }, transcript, memoryHash), false, - "the approved v8-to-v9 invalidation must reject evidence from the old directive parser", + "the approved v9-to-v10 invalidation must reject evidence without partial-transcript provenance", ); assert.equal(evidenceKey(transcript, memoryHash), currentKey); assert.equal(isEvidenceFresh({ status: "ok", key: currentKey }, transcript, memoryHash), true);