diff --git a/AGENTS.md b/AGENTS.md index 22c14a5a..fb47e235 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,11 @@ Each owning file's comments and tests hold the detail; read them before touching - **Quotes must be found in the trace they cite** (`sanitizeEvidence` in `src/analyze.js`), so fake agents in tests must quote real session text. Bump `ANALYSIS_INDEX_VERSION` (`src/state.js`) for any change to what analysis accepts. +- **Direct task/steering instructions are addressable** (`src/directives.js`): the first substantive user turn is + `TASK-1`, later ones `STEER-`, each with `direct-task`/`direct-steering` authority and a lifetime from + its turn onward. Only the carved envelope indexes (fenced blocks and `>` quotes excluded); assistant/tool text + never indexes. Spans render by turn reference, never duplicated text; only ids plus metadata persist. The fold + has no directive rows: cites stay session authority, and recurrence reaches synthesis as ordinary gaps. - **Sampling is deterministic and sticky** (`src/sample.js`): a per-transcript hash of `transcriptIdentity` and `config.seed`, never `Math.random()` or an index/position-derived draw. - **Corpus mix is interactive vs non-interactive, never an unknown bucket** (`src/interaction.js`). diff --git a/README.md b/README.md index 989c96c7..91c4c6e5 100644 --- a/README.md +++ b/README.md @@ -369,8 +369,9 @@ weights still evolve as transcripts age, so the selected set can change over tim equally reproducible sample. Each distilled trace goes to a cheap model with the memory file, the project skill index, -and a rubric. It returns strict JSON: which instructions helped, which were violated, and -what mistakes no current instruction covers. Every negative carries a class - `harm` (following the instruction +an index of this session's direct instructions, and a rubric. It returns strict JSON: which +instructions helped, which were violated, and what mistakes no memory instruction covers. +Every negative carries a class - `harm` (following the instruction caused damage), `non-compliance` (the agent ignored it), or `irrelevant` - because those argue for opposite fates: harm argues against an instruction, non-compliance argues for reinforcing it. Every gap carries a domain - `orchestration` when the mistake was caused @@ -380,6 +381,18 @@ orchestrating tool) - and the analysis is shown the ledger's open gaps so it can cite an existing gap id instead of coining a paraphrase of it. +Much of what governs a session comes from the user directly rather than from a memory file, +so those instructions are addressable too. The first user turn with authoritative text (`TASK-1`) and each +later such turn (`STEER-`) become citable instruction sources with their own +authority, alongside the memory-file ids. Only the authoritative part of a user turn counts: +fenced blocks and quoted lines are pasted evidence, never instructions, and assistant or tool +text never becomes one. The index references a trace turn without repeating its text. +If a turn is elided by session capping or clamped by message truncation, the index marks it +and points to the raw transcript. Only the ids and their metadata persist in the evidence record. Direct +instructions are session authority, not durable memory, so a mistake covered only by one is +still reported as a gap - the user had to say it in this session, and the next session starts +without it - and their citations do not score a memory-file instruction. + **Every claim must carry a verbatim quote, and the quote must be in the trace.** Quoteless items are discarded - the single most important defence against a model confabulating influence - and so are quotes that do not actually appear in the distilled trace they claim diff --git a/src/acpx.js b/src/acpx.js index 28d602d3..f62162bd 100644 --- a/src/acpx.js +++ b/src/acpx.js @@ -913,7 +913,7 @@ function recoverPiUsage({ promptFile, cwd, startedAt }) { for (const candidate of candidates) { const descriptor = piStore.classify(candidate); if (!descriptor || !wanted.has(descriptor.cwd)) continue; - const entries = readJsonl(candidate.path); + const { entries } = readJsonl(candidate.path); const firstUser = entries.find((e) => e.type === "message" && e.message?.role === "user"); if (!firstUser || piMessageText(firstUser.message.content).trim() !== prompt) continue; diff --git a/src/analyze.js b/src/analyze.js index 0b532ed2..3d446da7 100644 --- a/src/analyze.js +++ b/src/analyze.js @@ -3,6 +3,7 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; import { extractJson, runModelCall, usageRecord } from "./acpx.js"; +import { directiveMetadata, extractDirectives, renderDirectiveIndex } from "./directives.js"; import { distill } from "./distill.js"; import { classifyInteraction } from "./interaction.js"; import { getAdapter, readTranscript } from "./discovery/index.js"; @@ -149,7 +150,7 @@ function foldSpace(text) { * `quotesNotInTrace` counts what the trace check rejected, so a run whose analysis model * paraphrases everything reads as that rather than as a clean repo. */ -export function sanitizeEvidence(parsed, memoryFile = null, trace = null) { +export function sanitizeEvidence(parsed, memoryFile = null, trace = null, directiveIds = null) { const clean = { positive: [], negative: [], @@ -160,6 +161,11 @@ export function sanitizeEvidence(parsed, memoryFile = null, trace = null) { if (!parsed || typeof parsed !== "object") return clean; const validInstructions = memoryFile ? new Set(instructionUnits(memoryFile).map((unit) => unit.id)) : null; + // Direct task/steering spans are session-scoped instruction sources alongside the + // memory index; a citation of one is addressable evidence, not a stale reference. + if (validInstructions && Array.isArray(directiveIds)) { + for (const id of directiveIds) validInstructions.add(id); + } const foldedTrace = typeof trace === "string" && !clean.usedRawTranscript ? foldSpace(trace) : null; const hasQuote = (item) => { if (typeof item?.quote !== "string" || item.quote.trim().length < 8) return false; @@ -260,6 +266,7 @@ async function analyzeOne({ const distilled = distill(raw.events, { ...transcript, model: raw.model, + prefixDiscarded: raw.prefixDiscarded, rawPath: rawFile ?? raw.rawPath, }); @@ -299,11 +306,13 @@ async function analyzeOne({ fs.writeFileSync(rawFile, `${lines.join("\n")}\n`, { mode: 0o600, flag: "wx" }); } + const directives = extractDirectives(distilled.turns); const prompt = renderPrompt("analysis", { MEMORY_PATH: memoryFile.path, INSTRUCTION_INDEX: renderInstructionIndex(memoryFile), ALSO_LOADED: alsoLoaded, SKILLS: skillIndex, + DIRECTIVES: renderDirectiveIndex(directives), OPEN_GAPS: openGapIndex, TRACE: distilled.trace, }); @@ -336,9 +345,18 @@ async function analyzeOne({ throw new Error("analysis returned no parseable JSON"); } + const evidence = sanitizeEvidence( + parsed, + memoryFile, + distilled.trace, + directives.map((span) => span.id), + ); return { status: "ok", - evidence: sanitizeEvidence(parsed, memoryFile, distilled.trace), + // The record carries this session's directive span metadata - ids, turn, authority + // and lifetime, never envelope text - so a cited TASK-1 / STEER-N id stays + // resolvable in persisted evidence (see `src/directives.js`). + evidence: { ...evidence, directives: directiveMetadata(directives) }, usage: usageRecord(ranWith, result), distilled, }; diff --git a/src/directives.js b/src/directives.js new file mode 100644 index 00000000..0aca663c --- /dev/null +++ b/src/directives.js @@ -0,0 +1,172 @@ +/** + * Direct task and steering instructions as addressable instruction sources. + * + * Backpass indexes project memory files as instruction sources, but in real sessions + * much of what governs agent behaviour comes straight from the user: the initial task + * message and later steering turns. Without ids those read only as undifferentiated + * transcript evidence, so analysis cannot say "the agent followed the user's direct + * instruction" apart from "the agent followed the memory file". This module makes each + * substantive user turn citable: + * + * - the first user turn with a non-empty envelope is the task span (`TASK-1`) + * - each later user turn with a non-empty envelope is a steering span (`STEER-`) + * - every span carries source kind, turn, an authority tier distinct from project + * memory (`direct-task` / `direct-steering`), and a lifetime running from its turn + * onward + * + * Only the authoritative envelope of a user message becomes a span. Quoted transcripts + * (`>` blockquotes), pasted tool output and code (fenced blocks), retry reasons and + * planned actions embedded as quotes or pastes are structurally excluded: indexing + * them as authority would invert instructions that say to treat that text as + * untrusted evidence. Assistant messages and tool results never become spans. + * + * Spans are extracted from the distilled message turns, so turn numbers match the + * trace the analysis model reads. Each turn carries the envelope `distill` carved from + * its full pre-clamp text, so a closing fence lost to message clamping cannot demote an + * instruction to a paste. The prompt index renders spans by reference (id plus a + * `see turn N` pointer, or a note that the turn was elided or clamped out of the trace), never by + * duplicating turn text: task and steering text is private by default, and only ids plus + * metadata (`directives` on the evidence record) ever persist. + * + * A span is session authority, never durable project memory: a mistake only a direct + * instruction covered is still a gap, because the next session starts without it. + */ + +export const TASK_ID = "TASK-1"; +export const TASK_AUTHORITY = "direct-task"; +export const STEERING_AUTHORITY = "direct-steering"; + +// A fence opener may sit behind a list marker: an optional bullet (`-`, `*`, `+`) or +// ordered marker (digits with `.` or `)`) plus spaces before the fence. Group 1 keeps the +// whole prefix so the closer indent is measured from the fence's own column - the content +// column after that marker - not from the start of the line. +const FENCE = /^([ \t]*([-*+][ \t]+|\d{1,9}[.)][ \t]+)?)(`{3,}|~{3,})([^\n]*)$/; +// A blockquote may also sit behind a single list marker (`- > quoted`); a marker must +// be followed by whitespace, so an unspaced dash (`-> quoted`) stays instruction text. +const QUOTE = /^\s*(?:[-*+][ \t]+|\d{1,9}[.)][ \t]+)?>/; + +// Tabs advance to the next four-column stop. Fence indentation is relative: an opener may +// sit at any indentation, optionally behind a list marker (a list-item code block is still +// a fence), while its closer may be indented at most three columns past the fence's own +// column. +function indentColumns(prefix) { + let columns = 0; + for (const char of prefix) columns += char === "\t" ? 4 - (columns % 4) : 1; + return columns; +} + +/** A span id is stable by construction: kind plus the distilled turn it points at. */ +export function isDirectiveId(id) { + return id === TASK_ID || /^STEER-\d+$/.test(String(id || "")); +} + +function steeringId(turn) { + return `STEER-${turn}`; +} + +/** + * The authoritative envelope of a user message. Fenced code blocks (pasted tool output, + * code, retry logs) and blockquote lines (quoted transcripts, pasted replies, planned + * actions quoted back) are evidence-only, never authority, so they are stripped. + */ +export function carveEnvelope(text) { + const kept = []; + let opener = null; + + for (const line of String(text ?? "").split("\n")) { + const fence = FENCE.exec(line); + if (opener) { + // CommonMark: a closer must have a whitespace-only prefix, use the opener's + // character at least as many times, have only whitespace after it, and sit at most + // three columns past the opener. A shorter, different or deeper inner fence is + // part of the paste. + if ( + fence && + !fence[2] && + fence[3][0] === opener.char && + fence[3].length >= opener.length && + fence[4].trim() === "" && + indentColumns(fence[1]) <= opener.indent + 3 + ) { + opener = null; + } + // An unclosed fence still withholds the rest: a pasted log without a closing + // marker is still a paste, not an instruction. + continue; + } + // CommonMark: a backtick fence's info string may not contain a backtick, so a line + // of inline code is instruction text, not an opener. + if (fence && !(fence[3][0] === "`" && fence[4].includes("`"))) { + opener = { char: fence[3][0], length: fence[3].length, indent: indentColumns(fence[1]) }; + continue; + } + if (QUOTE.test(line)) continue; + kept.push(line); + } + + return kept.join("\n").trim(); +} + +/** + * Index the user turns of one session that carry an authoritative envelope. `turns` are + * the distilled message turns (`{ turn, role, envelope, elided, clamped }`) in trace order, + * so ids stay stable across runs of the same transcript and each span knows whether its + * turn text survived into the trace. + */ +export function extractDirectives(turns) { + const spans = []; + let seenFirstUser = false; + for (const entry of Array.isArray(turns) ? turns : []) { + if (!entry || entry.role !== "user") continue; + if (!entry.envelope) continue; + const span = + !seenFirstUser && !entry.partial + ? { id: TASK_ID, kind: "task", turn: entry.turn, authority: TASK_AUTHORITY } + : { id: steeringId(entry.turn), kind: "steering", turn: entry.turn, authority: STEERING_AUTHORITY }; + seenFirstUser = true; + spans.push({ + ...span, + partial: entry.partial === true, + elided: entry.elided === true, + clamped: entry.clamped === true, + lifetime: { fromTurn: entry.turn, toTurn: null }, + }); + } + return spans; +} + +/** Metadata that may persist on an evidence record: ids and spans, never turn text. */ +export function directiveMetadata(spans) { + return (Array.isArray(spans) ? spans : []).map((span) => ({ + id: span.id, + kind: span.kind, + turn: span.turn, + authority: span.authority, + partial: span.partial === true, + stable: span.partial !== true, + lifetime: { ...span.lifetime }, + })); +} + +/** + * The analysis-prompt index. By reference only: each entry names the turn whose text is + * already in the trace, or says the turn's text was elided or clamped out of it, so no + * user-turn text is duplicated into the prompt. + */ +export function renderDirectiveIndex(spans) { + const list = Array.isArray(spans) ? spans : []; + if (!list.length) return "(none - no substantive user instruction found in the trace)"; + return list + .map( + (span) => + `[${span.id}] ${span.kind} · turn ${span.turn} · authority ${span.authority} · ` + + (span.partial ? "partial transcript · unstable id · " : "") + + `lifetime turns ${span.lifetime?.fromTurn ?? span.turn}+ - ` + + (span.elided + ? `turn ${span.turn} - elided from the trace, open the raw transcript for its text` + : span.clamped + ? `turn ${span.turn} - clamped from the trace, open the raw transcript for its text` + : `see turn ${span.turn} in the trace`), + ) + .join("\n"); +} diff --git a/src/discovery/adapters/claude.js b/src/discovery/adapters/claude.js index 23f94fb0..f5f3b80c 100644 --- a/src/discovery/adapters/claude.js +++ b/src/discovery/adapters/claude.js @@ -83,7 +83,7 @@ export function classify(candidate) { } export function read(ref) { - const entries = readJsonl(ref.path); + const { entries, prefixDiscarded } = readJsonl(ref.path); const events = []; let model = null; @@ -98,5 +98,5 @@ export function read(ref) { } } - return { events: attachToolResults(events), model }; + return { events: attachToolResults(events), model, prefixDiscarded }; } diff --git a/src/discovery/adapters/codex.js b/src/discovery/adapters/codex.js index 7fd31a1a..172008e6 100644 --- a/src/discovery/adapters/codex.js +++ b/src/discovery/adapters/codex.js @@ -102,7 +102,7 @@ export function classify(candidate) { } export function read(ref) { - const entries = readJsonl(ref.path); + const { entries, prefixDiscarded } = readJsonl(ref.path); const events = []; let model = null; @@ -143,7 +143,7 @@ export function read(ref) { } } - return { events: attachToolResults(events), model }; + return { events: attachToolResults(events), model, prefixDiscarded }; } function parseMaybeJson(value) { diff --git a/src/discovery/adapters/grok.js b/src/discovery/adapters/grok.js index 713067c2..ec05791b 100644 --- a/src/discovery/adapters/grok.js +++ b/src/discovery/adapters/grok.js @@ -61,7 +61,7 @@ export function read(ref) { const chatPath = ref.extra?.chatPath || path.join(ref.path, "chat_history.jsonl"); if (!fs.existsSync(chatPath)) return { events: [], model: null }; - const entries = readJsonl(chatPath); + const { entries, prefixDiscarded } = readJsonl(chatPath); const events = []; let model = null; @@ -91,7 +91,7 @@ export function read(ref) { } } - return { events: attachToolResults(events), model }; + return { events: attachToolResults(events), model, prefixDiscarded }; } /** Exported for the raw-transcript escape hatch: the analysis agent opens this file. */ diff --git a/src/discovery/adapters/pi.js b/src/discovery/adapters/pi.js index 68e2b733..7fef606e 100644 --- a/src/discovery/adapters/pi.js +++ b/src/discovery/adapters/pi.js @@ -226,7 +226,7 @@ export function classify(candidate, options = {}) { } export function read(ref) { - const entries = readJsonl(ref.path); + const { entries, prefixDiscarded } = readJsonl(ref.path); const events = []; let model = null; const omp = entries[0]?.type === "title"; @@ -248,7 +248,7 @@ export function read(ref) { contentToEvents(role, message.content, events); } - return { events: attachToolResults(events), model }; + return { events: attachToolResults(events), model, prefixDiscarded }; } function textOf(content) { diff --git a/src/discovery/adapters/shared.js b/src/discovery/adapters/shared.js index cb7a2daa..fc9f5d01 100644 --- a/src/discovery/adapters/shared.js +++ b/src/discovery/adapters/shared.js @@ -41,9 +41,11 @@ export function readJsonFile(file) { export function readJsonl(file, { maxBytes = 64 * 1024 * 1024 } = {}) { let text; + let prefixDiscarded = false; try { const stat = fs.statSync(file); if (stat.size > maxBytes) { + prefixDiscarded = true; // Very large sessions are read tail-first: recent turns carry the loss signal. const fd = fs.openSync(file, "r"); const buffer = Buffer.alloc(maxBytes); @@ -55,7 +57,7 @@ export function readJsonl(file, { maxBytes = 64 * 1024 * 1024 } = {}) { text = fs.readFileSync(file, "utf8"); } } catch { - return []; + return { entries: [], prefixDiscarded }; } const out = []; for (const line of text.split("\n")) { @@ -63,7 +65,7 @@ export function readJsonl(file, { maxBytes = 64 * 1024 * 1024 } = {}) { const value = parseJsonLine(line); if (value) out.push(value); } - return out; + return { entries: out, prefixDiscarded }; } export function statOrNull(file) { @@ -155,8 +157,8 @@ export function contentToEvents(role, content, events) { } } if (texts.length) { - const joined = texts.join("\n").trim(); - if (joined) events.push({ kind: "message", role, text: joined }); + const joined = texts.join("\n"); + if (joined.trim()) events.push({ kind: "message", role, text: joined }); } } diff --git a/src/discovery/index.js b/src/discovery/index.js index 82a116ed..e8b4188b 100644 --- a/src/discovery/index.js +++ b/src/discovery/index.js @@ -518,6 +518,7 @@ export async function readTranscript(transcript) { return { events: result.events || [], model: result.model || transcript.model || null, + prefixDiscarded: result.prefixDiscarded === true, rawPath: cached, }; } @@ -526,6 +527,7 @@ export async function readTranscript(transcript) { return { events: result.events || [], model: result.model || transcript.model || null, + prefixDiscarded: result.prefixDiscarded === true, rawPath: adapter.rawPath ? adapter.rawPath(transcript) : transcript.path, }; } diff --git a/src/distill.js b/src/distill.js index b890d478..3ec20809 100644 --- a/src/distill.js +++ b/src/distill.js @@ -1,3 +1,4 @@ +import { carveEnvelope } from "./directives.js"; import { redact } from "./redact.js"; import { estimateTokens } from "./tokens.js"; @@ -100,6 +101,18 @@ export function isBoilerplate(text) { export function distill(events, meta, options = {}) { const maxTraceTokens = options.maxTraceTokens ?? 12000; const lines = []; + let bodyChars = 0; + const push = (line) => { + lines.push(line); + bodyChars += line.length + 1; + }; + // Message turns in trace order, with the same turn numbers the trace prints: the + // single source of truth for anything that points at a turn (directive spans in + // `src/directives.js`). Text is the redacted form the model actually sees; the + // envelope is carved from the full pre-clamp text, so a closing fence that the clamp + // elides cannot turn an instruction into a paste. + const turns = []; + const turnSpans = []; let userTurns = 0; let assistantTurns = 0; let toolCalls = 0; @@ -108,21 +121,36 @@ export function distill(events, meta, options = {}) { for (const event of events) { if (!event) continue; if (event.kind === "message") { - const text = clampMessage(redact(event.text)); + const redacted = redact(event.text); + const text = clampMessage(redacted); if (!text || isBoilerplate(text)) continue; turn += 1; if (event.role === "user") userTurns += 1; else assistantTurns += 1; - lines.push(`### turn ${turn} · ${event.role}`); - lines.push(text); - lines.push(""); + turns.push({ + turn, + role: event.role, + partial: meta.prefixDiscarded === true, + text, + envelope: event.role === "user" ? carveEnvelope(redacted) : "", + // The envelope is carved pre-clamp, so a clamped turn's instruction may be absent + // from what the model reads; the index must not point at it. + clamped: text !== redacted.trim(), + // Set once the trace cap is known, below. + elided: false, + }); + const from = bodyChars; + push(`### turn ${turn} · ${event.role}`); + push(text); + turnSpans.push([from, bodyChars - 1]); + push(""); } else if (event.kind === "tool") { toolCalls += 1; const input = redact(describeToolInput(event.input)); const result = redact(describeToolResult(event.result)); const status = event.status && event.status !== "completed" ? ` [${event.status}]` : ""; const arrow = result ? ` -> ${result}` : ""; - lines.push(`tool: ${event.name || "unknown"}${input ? ` ${JSON.stringify(input)}` : ""}${status}${arrow}`); + push(`tool: ${event.name || "unknown"}${input ? ` ${JSON.stringify(input)}` : ""}${status}${arrow}`); } } @@ -147,11 +175,17 @@ export function distill(events, meta, options = {}) { "transcript only if a specific claim needs the full text.", ].join("\n"); - const { body, elided } = capTrace(lines.join("\n").trim(), maxTraceTokens); + const rawBody = lines.join("\n").trim(); + const { body, elided, kept } = capTrace(rawBody, maxTraceTokens); + turns.forEach((entry, index) => { + const [from, to] = turnSpans[index]; + entry.elided = !kept.some(([keptFrom, keptTo]) => from >= keptFrom && to <= keptTo); + }); const trace = `${header}\n${body}\n${footer}\n`; return { trace, + turns, stats: { userTurns, assistantTurns, @@ -168,7 +202,7 @@ export function distill(events, meta, options = {}) { * raw transcript path in the footer remains the escape hatch for anything in between. */ function capTrace(body, maxTraceTokens) { - if (estimateTokens(body) <= maxTraceTokens) return { body, elided: false }; + if (estimateTokens(body) <= maxTraceTokens) return { body, elided: false, kept: [[0, body.length]] }; const budgetChars = maxTraceTokens * 4; const headChars = Math.floor(budgetChars * 0.45); const tailChars = Math.floor(budgetChars * 0.45); @@ -178,5 +212,9 @@ function capTrace(body, maxTraceTokens) { return { body: `${head}\n\n[... middle of session elided: ~${droppedTokens} tokens. Open the raw transcript below if a claim needs it ...]\n\n${tail}`, elided: true, + kept: [ + [0, headChars], + [body.length - tailChars, body.length], + ], }; } diff --git a/src/fold.js b/src/fold.js index e03b783e..d16d0b78 100644 --- a/src/fold.js +++ b/src/fold.js @@ -2,6 +2,7 @@ import fs from "node:fs"; import { loadConfig } from "./config.js"; import { classifyInteraction, INTERACTIVE, NON_INTERACTIVE } from "./interaction.js"; +import { isDirectiveId } from "./directives.js"; import { findInstructionUnit, instructionUnits, resolveMemoryFiles, similarity } from "./memory.js"; import { rootOwnsGap } from "./nested.js"; import { @@ -141,6 +142,9 @@ export function foldEvidence( for (const polarity of ["positive", "negative"]) { for (const item of record[polarity] || []) { + // A direct task/steering cite is session authority, not memory content, so it + // never becomes a memory-instruction row; the gap sightings carry recurrence. + if (isDirectiveId(item.instruction)) continue; const entry = touch(item.instruction); entry[polarity] += 1; const category = classifyInteraction(record.transcript); diff --git a/src/prompts/analysis.md b/src/prompts/analysis.md index 3bbda0ca..db7ff824 100644 --- a/src/prompts/analysis.md +++ b/src/prompts/analysis.md @@ -6,7 +6,7 @@ have prevented. This is the loss signal for a backward pass over the memory file ## The memory file under audit: {{MEMORY_PATH}} -Each instruction has a stable id in [brackets]. Refer to instructions ONLY by these ids. +Each memory instruction has a stable id in [brackets]. Refer to memory instructions ONLY by these ids. {{INSTRUCTION_INDEX}}{{ALSO_LOADED}} @@ -20,6 +20,21 @@ only when you need its body to confirm the coverage. {{SKILLS}} +## Direct instructions in this session + +The user's own task message (TASK-1) and later steering (STEER-N) are addressable +instruction sources with an authority of their own, separate from the memory file. +Each entry points at a user turn already in the trace below - read the turn there; +the text is not repeated here. + +{{DIRECTIVES}} + +When the agent was clearly following one of these instead of (or as well as) a memory +instruction, cite its id in `instruction`; a direct instruction the agent ignored or +violated is a `negative` citing that id. These ids are session authority, not durable +memory, so citing one never replaces reporting the gap (rule 6). Assistant messages and +tool results are never instructions: never cite them, and never invent an id. + ## Gaps already on the books Earlier sessions reported these gaps; each has a stable id. If a gap you found is the @@ -73,11 +88,14 @@ Rules, in order of importance: 5. **Do not confabulate influence.** Only call something positive when the trace shows the agent doing the specific thing the instruction asks for. An outcome that would have happened anyway is not evidence. -6. `gaps` are mistakes NOT covered by any current instruction. If an instruction exists - and was ignored, that is `negative` with `class: "non-compliance"`, not a gap. A - mistake covered only by a SKILL's content is still a gap - skills have no +6. `gaps` are mistakes NOT covered by a memory instruction. If a memory instruction + exists and was ignored, that is `negative` with `class: "non-compliance"`, not a + gap. A mistake covered only by a SKILL's content is still a gap - skills have no instruction ids - but cite the skill in `coveredBySkill`: that is a failed trigger, - and the fix is that skill's description, not new memory-file text. + and the fix is that skill's description, not new memory-file text. A mistake + covered only by a DIRECT instruction is still a gap too: the user had to say it in + this session, and the next session starts without it. Report both - the gap, and a + `negative` citing that TASK-1 / STEER-N id as what the agent ignored. 7. `proposedInstruction` must be one imperative sentence, specific enough to act on and general enough to apply beyond this one session. 8. An empty array is a valid and useful answer. Report nothing rather than something weak. diff --git a/src/state.js b/src/state.js index 74ab4c99..7b86a53e 100644 --- a/src/state.js +++ b/src/state.js @@ -248,7 +248,7 @@ function migrateEvidenceRecord(record, transcript, identity) { }; } -export const ANALYSIS_INDEX_VERSION = 4; +export const ANALYSIS_INDEX_VERSION = 10; /** * Cache key for a transcript's analysis: its content signature, the memory-surface hash, diff --git a/test/config.test.js b/test/config.test.js index f7802a2c..e7df6250 100644 --- a/test/config.test.js +++ b/test/config.test.js @@ -234,6 +234,21 @@ test("evidence is keyed to both the transcript and the memory file it was judged assert.equal(isEvidenceFresh(null, transcript, "sha256:aaa"), false); }); +test("evidence accepted before the partial-transcript fix is stale; version 10 evidence is reusable", () => { + const transcript = { identity: "directive-demo", mtimeMs: 111, bytes: 222 }; + const memoryHash = "sha256:aaa"; + const previousKey = "directive-demo:111:222:sha256:aaa:analysis-index-v9"; + const currentKey = "directive-demo:111:222:sha256:aaa:analysis-index-v10"; + + assert.equal( + isEvidenceFresh({ status: "ok", key: previousKey }, transcript, memoryHash), + false, + "the approved v9-to-v10 invalidation must reject evidence without partial-transcript provenance", + ); + assert.equal(evidenceKey(transcript, memoryHash), currentKey); + assert.equal(isEvidenceFresh({ status: "ok", key: currentKey }, transcript, memoryHash), true); +}); + test("state round-trips through disk and survives a corrupt file", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-state-")); const state = new State(dir).ensure(); diff --git a/test/directives.test.js b/test/directives.test.js new file mode 100644 index 00000000..d24d0619 --- /dev/null +++ b/test/directives.test.js @@ -0,0 +1,665 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { distill } from "../src/distill.js"; +import { analyzeTranscripts, sanitizeEvidence } from "../src/analyze.js"; +import { readTranscript } from "../src/discovery/index.js"; +import { State } from "../src/state.js"; +import { foldEvidence, renderEvidenceForPrompt, renderEvidenceReport } from "../src/fold.js"; +import { parseMemoryUnits } from "../src/memory.js"; +import { estimateTokens } from "../src/tokens.js"; +import { + TASK_ID, + carveEnvelope, + directiveMetadata, + extractDirectives, + isDirectiveId, + renderDirectiveIndex, +} from "../src/directives.js"; +import * as claude from "../src/discovery/adapters/claude.js"; +import * as pi from "../src/discovery/adapters/pi.js"; + +const FIXTURES = path.dirname(fileURLToPath(import.meta.url)); +const META = { + id: "directive-demo", + harness: "claude", + association: { tier: 1, confidence: "exact" }, + rawPath: "/tmp/directive-spans.jsonl", +}; + +function fixtureEvents() { + return JSON.parse(fs.readFileSync(path.join(FIXTURES, "fixtures/directive-spans.json"), "utf8")); +} + +test("only the authoritative envelope becomes instruction text", () => { + const envelope = carveEnvelope(fixtureEvents()[0].text); + assert.match(envelope, /Migrate the auth module/); + assert.match(envelope, /Keep the public function names unchanged/); + assert.ok(!envelope.includes("previous assistant said"), "quoted transcript must not be authority"); + assert.ok(!envelope.includes("401 Unauthorized"), "quoted tool output must not be authority"); + assert.ok(!envelope.includes("refreshSession"), "pasted tool output must not be authority"); +}); + +test("a nested inner fence stays inside the paste; the outer fence closes it", () => { + const envelope = carveEnvelope( + [ + "Update the README:", + "````md", + "# doc", + "```bash", + "rm -rf build", + "```", + "````", + "Keep the examples short.", + ].join("\n"), + ); + assert.ok(!envelope.includes("rm -rf build"), "pasted command must not be authority"); + assert.ok(!envelope.includes("# doc"), "pasted doc body must not be authority"); + assert.match(envelope, /Update the README:/); + assert.match(envelope, /Keep the examples short\./); +}); + +test("a tilde fence never closes a backtick block", () => { + const envelope = carveEnvelope(["Fix this:", "```", "~~~", "rm -rf build", "```", "Run the tests."].join("\n")); + assert.ok(!envelope.includes("rm -rf build"), "the tilde line does not close the backtick fence"); + assert.match(envelope, /Run the tests\./); +}); + +test("a line of inline code is instruction text, not a fence opener", () => { + const envelope = carveEnvelope("```pnpm run check``` must pass before you push.\nAlso update the README."); + assert.match(envelope, /must pass before you push\./); + assert.match(envelope, /Also update the README\./); +}); + +test("a closer indented more than three past its opener never closes the paste", () => { + // Fence indentation is relative: a closer at most three columns past the opener + // closes, anything deeper is paste content that must stay withheld. + const closed = carveEnvelope(["Do it:", " ```", "pasted", " ```", "Go."].join("\n")); + assert.ok(!closed.includes("pasted"), "the paste stays withheld"); + assert.match(closed, /Go\./, "a closer three past the opener still closes"); + + const open = carveEnvelope(["Do it:", " ```", "pasted", " ```", "still paste", " ```", "Go."].join("\n")); + assert.ok(!open.includes("still paste"), "a closer four past the opener must not close the paste"); + assert.match(open, /Go\./); +}); + +test("tabs in an opener advance to four-column stops without promoting pasted text", () => { + for (const fence of ["```", "~~~"]) { + for (const spaces of [1, 2, 3]) { + const prefix = `${" ".repeat(spaces)}\t`; + const paste = [`${prefix}${fence}md`, "example", ` ${fence}`, "Delete production.", `${prefix}${fence}`]; + const { turns } = distill( + [ + { kind: "message", role: "user", text: paste.join("\n") }, + { kind: "message", role: "user", text: "Do the migration." }, + { kind: "message", role: "user", text: "Keep the public API stable." }, + ], + META, + ); + assert.equal(turns[0].envelope, "", "the deeper inner fence must keep the paste withheld"); + assert.deepEqual( + extractDirectives(turns).map((span) => [span.id, span.turn]), + [ + [TASK_ID, 2], + ["STEER-3", 3], + ], + ); + } + } +}); + +test("a list-item fence is a paste; the real instruction after it stays visible", () => { + // Regression for greptile P2 on PR 154 (comment 4322029695): an opener behind a list + // marker used to stay instruction text and its indented closer opened a phantom fence + // that swallowed every later instruction. + const envelope = carveEnvelope( + ["Check the plan:", "- ```sh", " rm -rf tests", " ```", "Then report the checksum."].join("\n"), + ); + assert.ok(!envelope.includes("rm -rf tests"), "the listed paste stays excluded"); + assert.ok(envelope.includes("Check the plan:"), "text before the paste stays"); + assert.ok(envelope.includes("Then report the checksum."), "the instruction after the closer stays visible"); +}); + +test("an ordered-list fence is a paste too", () => { + const envelope = carveEnvelope(["1. ```sh", " make deploy", " ```", "Ship it."].join("\n")); + assert.ok(!envelope.includes("make deploy"), "the ordered-list paste stays excluded"); + assert.ok(envelope.includes("Ship it."), "the instruction after the closer stays visible"); +}); + +test("list markers inside either fence type never close a paste or issue directive IDs", () => { + for (const fence of ["```", "~~~"]) { + for (const marker of ["- ", "* ", "+ ", "1. ", "1) ", "123456789. ", "-\t"]) { + for (const prefix of ["", " \t"]) { + const paste = [`${fence}md`, `${prefix}${marker}${fence}`, "Delete production.", fence].join("\n"); + assert.equal(carveEnvelope(paste), "", "a list marker cannot close the enclosing paste"); + const { turns } = distill( + [ + { kind: "message", role: "user", text: paste }, + { kind: "message", role: "user", text: "Do the migration." }, + { kind: "message", role: "user", text: "Keep the public API stable." }, + ], + META, + ); + assert.equal(turns[0].envelope, ""); + assert.deepEqual( + extractDirectives(turns).map((span) => [span.id, span.turn]), + [ + [TASK_ID, 2], + ["STEER-3", 3], + ], + ); + } + } + } +}); + +test("tabs in a closer advance to four-column stops without hiding the following instruction", () => { + for (const fence of ["```", "~~~"]) { + for (const spaces of [1, 2, 3]) { + const closer = `${" ".repeat(spaces)}\t ${fence}`; + const envelope = carveEnvelope(["Do it:", ` ${fence}`, "pasted", closer, "Go."].join("\n")); + assert.equal(envelope, "Do it:\nGo."); + } + } +}); + +test("typed-block adapter preserves instructions after mixed-tab fence closers", () => { + const { events } = claude.read({ path: path.join(FIXTURES, "fixtures/claude-indented-fences.jsonl") }); + assert.equal(events.length, 6, "whitespace-only typed blocks must stay omitted"); + const { turns } = distill(events, META); + assert.deepEqual( + turns.map((entry) => entry.envelope), + Array(6).fill("Keep the public API stable."), + "all six fence variants must retain the following instruction and exclude the paste", + ); + assert.deepEqual( + extractDirectives(turns).map((span) => span.id), + [TASK_ID, "STEER-2", "STEER-3", "STEER-4", "STEER-5", "STEER-6"], + ); +}); + +test("an unclosed fence still withholds the paste", () => { + assert.equal(carveEnvelope("Do the migration.\n```\nError: boom"), "Do the migration."); +}); + +test("an indented fence line inside a paste neither opens nor closes", () => { + const envelope = carveEnvelope( + [ + "Fix the docs:", + "```md", + "Example:", + " ```bash", + " npm ci", + " ```", + "End.", + "```", + "Use pnpm, not npm.", + ].join("\n"), + ); + assert.ok(!envelope.includes("npm ci"), "the indented block stays inside the paste"); + assert.ok(!envelope.includes("End."), "the paste remainder never re-enters the envelope"); + assert.match(envelope, /Fix the docs:/); + assert.match(envelope, /Use pnpm, not npm\./); +}); + +test("an over-limit turn keeps its instruction when the closing fence falls in the clamped region", () => { + const log = "2026-01-01T00:00:00Z some CI log line\n".repeat(200); + const padding = "> quoted CI context\n".repeat(100); + const { turns } = distill( + [ + { kind: "message", role: "user", text: `\`\`\`\n${log}\`\`\`\n${padding}\nKeep the public API stable.` }, + { kind: "message", role: "assistant", text: "Will do." }, + ], + META, + ); + assert.ok(turns[0].text.includes("chars elided"), "the turn text is clamped"); + assert.ok(turns[0].text.includes("Keep the public API stable."), "the final instruction survives clamping"); + assert.equal(carveEnvelope(turns[0].text), "", "the clamped text loses its closing fence and hides the instruction"); + const spans = extractDirectives(turns); + assert.deepEqual( + spans.map((span) => [span.id, span.turn]), + [[TASK_ID, 1]], + ); + assert.equal(turns[0].envelope, "Keep the public API stable."); +}); + +test("an over-limit turn whose only instruction is clamped out is marked clamped, not pointed at", () => { + const filler = (label) => `${label} log line detail\n`.repeat(300); + const { turns, trace } = distill( + [ + { + kind: "message", + role: "user", + text: `${filler("head")}Keep the public API stable.\n${filler("tail")}`, + }, + { kind: "message", role: "assistant", text: "Will do." }, + ], + META, + ); + assert.ok(!turns[0].text.includes("Keep the public API stable."), "the instruction sits in the clamped middle"); + assert.ok(!trace.includes("Keep the public API stable."), "the trace never carries the instruction"); + assert.match(turns[0].envelope, /Keep the public API stable\./); + const spans = extractDirectives(turns); + assert.equal(spans[0].clamped, true); + const section = renderDirectiveIndex(spans); + assert.match(section, /turn 1 - clamped from the trace, open the raw transcript for its text/); + assert.ok(!section.includes("see turn 1 in the trace")); +}); + +test("the first user turn is the task, later ones are steering; assistant and tool turns never index", () => { + const { turns } = distill(fixtureEvents(), META); + const spans = extractDirectives(turns); + assert.deepEqual( + spans.map((span) => [span.id, span.kind, span.turn, span.authority]), + [ + [TASK_ID, "task", 1, "direct-task"], + ["STEER-3", "steering", 3, "direct-steering"], + ["STEER-4", "steering", 4, "direct-steering"], + ], + ); + assert.ok( + spans.every((span) => span.turn !== 2), + "the assistant reply and the tool result never index", + ); + assert.ok(isDirectiveId("TASK-1") && isDirectiveId("STEER-4")); + assert.ok(!isDirectiveId("AG-001") && !isDirectiveId("TASK-2") && !isDirectiveId("banana")); +}); + +test("a discarded transcript prefix withholds task authority and persists unstable steering", async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-partial-directives-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const formats = { + claude: (role, text) => ({ type: role, message: { content: text } }), + codex: (role, text) => ({ type: "response_item", payload: { type: "message", role, content: text } }), + pi: (role, text) => ({ type: "message", message: { role, content: text } }), + grok: (role, text) => ({ type: role, content: text }), + }; + const padding = "x".repeat(1024 * 1024); + for (const [harness, message] of Object.entries(formats)) { + const file = path.join(root, `${harness}.jsonl`); + const appendMessage = (role, text) => fs.appendFileSync(file, `${JSON.stringify(message(role, text))}\n`); + appendMessage("user", "Migrate the auth module."); + appendMessage("assistant", "Working on it."); + appendMessage("user", "Keep the public API stable."); + const transcript = { ...META, harness, path: file, extra: { chatPath: file } }; + const complete = await readTranscript(transcript); + assert.equal(complete.prefixDiscarded, false); + const completeSpans = extractDirectives(distill(complete.events, { ...META, ...complete }).turns); + assert.deepEqual( + directiveMetadata(completeSpans).map(({ id, partial, stable }) => [id, partial, stable]), + [ + [TASK_ID, false, true], + ["STEER-3", false, true], + ], + "a complete transcript retains the initial task and stable turn IDs", + ); + + fs.appendFileSync(file, '{"type":"padding","text":"'); + for (let i = 0; i < 64; i += 1) fs.appendFileSync(file, padding); + fs.appendFileSync(file, '"}\n'); + appendMessage("user", "Now run the tests too."); + appendMessage("assistant", "Tests pass."); + appendMessage("user", "Report the results."); + + for (const remote of [false, true]) { + const state = new State(root, { + stateDir: path.join(root, `${harness}-${remote ? "remote" : "local"}`), + exclude: false, + }).ensure(); + const ref = remote + ? { ...transcript, host: "fixture-host", remote: { kind: "raw", cachePath: file } } + : transcript; + const raw = await readTranscript(ref); + assert.equal(raw.prefixDiscarded, true, `${harness}: prefix loss must survive transcript reading`); + assert.deepEqual( + raw.events.map((event) => event.text), + ["Now run the tests too.", "Tests pass.", "Report the results."], + "only the tail survives; the original task is unavailable", + ); + const summary = await analyzeTranscripts({ + transcripts: [ref], + memoryFile: { path: "AGENTS.md", units: [] }, + memoryHash: "memory", + repo: { root }, + config: { + state, + jobs: 1, + discovery: { minUserTurns: 1 }, + agents: { + resolve: async () => ({ agent: "pi" }), + withFallthrough: async () => ({ + text: JSON.stringify({ + positive: [ + { instruction: "TASK-1", quote: "Now run the tests too.", effect: "invented task authority" }, + { instruction: "STEER-1", quote: "Now run the tests too.", effect: "ran tests" }, + { instruction: "STEER-3", quote: "Report the results.", effect: "reported results" }, + ], + negative: [], + gaps: [], + }), + }), + }, + }, + }); + assert.equal(summary.analyzed, 1); + const evidence = state.readEvidence(ref); + assert.deepEqual(evidence.directives, [ + { + id: "STEER-1", + kind: "steering", + turn: 1, + authority: "direct-steering", + partial: true, + stable: false, + lifetime: { fromTurn: 1, toTurn: null }, + }, + { + id: "STEER-3", + kind: "steering", + turn: 3, + authority: "direct-steering", + partial: true, + stable: false, + lifetime: { fromTurn: 3, toTurn: null }, + }, + ]); + assert.deepEqual( + evidence.positive.map((item) => item.instruction), + ["STEER-1", "STEER-3"], + ); + const promptDir = path.join(state.root, "prompts"); + const prompt = fs.readFileSync(path.join(promptDir, fs.readdirSync(promptDir)[0]), "utf8"); + assert.match( + prompt, + /\[STEER-1\] steering · turn 1 · authority direct-steering · partial transcript · unstable id/, + ); + assert.doesNotMatch(prompt, /\[TASK-1\] task · turn/); + assert.ok(!JSON.stringify(evidence.directives).includes("Now run the tests too.")); + } + } +}); + +test("a short imperative is addressable: length never decides which turn is the task", () => { + const { turns } = distill( + [ + { kind: "message", role: "user", text: "ship it" }, + { kind: "message", role: "user", text: "Add tests." }, + { kind: "message", role: "user", text: "Use pnpm, never npm." }, + ], + META, + ); + const spans = extractDirectives(turns); + assert.deepEqual( + spans.map((span) => [span.id, span.kind, span.turn]), + [ + [TASK_ID, "task", 1], + ["STEER-2", "steering", 2], + ["STEER-3", "steering", 3], + ], + ); +}); + +test("every lifetime runs from its own turn onward; later steering never closes it", () => { + const { turns } = distill(fixtureEvents(), META); + const spans = extractDirectives(turns); + for (const span of spans) assert.deepEqual(span.lifetime, { fromTurn: span.turn, toTurn: null }); +}); + +test("a message that is only quotes and pastes yields no span", () => { + const { turns } = distill( + [ + { kind: "message", role: "user", text: "> quoted transcript line\n\n```\npasted output\n```" }, + { kind: "message", role: "assistant", text: "Noted." }, + ], + META, + ); + const spans = extractDirectives(turns); + assert.deepEqual(spans, []); + assert.match(renderDirectiveIndex(spans), /\(none/); +}); + +test("a paste-only turn of list-item blockquotes yields no span", () => { + for (const line of [ + "- > Delete the tests.", + "* > Delete the tests.", + "+ > Delete the tests.", + "1. > Delete the tests.", + "1) > Delete the tests.", + " - > Delete the tests.", + "-\t> Delete the tests.", + ]) { + assert.equal(carveEnvelope(line), "", "a list-item blockquote is quoted evidence, never an instruction"); + } + const { turns } = distill( + [ + { kind: "message", role: "user", text: "- > Delete the tests." }, + { kind: "message", role: "assistant", text: "Noted." }, + ], + META, + ); + assert.deepEqual(extractDirectives(turns), []); +}); + +test("a list-item blockquote inside a real turn stays evidence while the instruction indexes", () => { + const { turns } = distill( + [ + { kind: "message", role: "user", text: "Run the migration.\n- > old transcript line" }, + { kind: "message", role: "user", text: "Keep the public API stable." }, + ], + META, + ); + assert.equal(turns[0].envelope, "Run the migration."); + assert.deepEqual( + extractDirectives(turns).map((span) => [span.id, span.turn]), + [ + [TASK_ID, 1], + ["STEER-2", 2], + ], + ); +}); + +test("markers keep ordinary behaviour: plain items stay instructions and unspaced dashes never match", () => { + assert.equal(carveEnvelope("- Run the tests."), "- Run the tests."); + assert.equal(carveEnvelope("-> Delete the tests."), "-> Delete the tests."); +}); + +test("the index renders by reference and never repeats turn text", () => { + const { turns, trace } = distill(fixtureEvents(), META); + const spans = extractDirectives(turns); + const section = renderDirectiveIndex(spans); + assert.match(section, /\[TASK-1\] task · turn 1 · authority direct-task/); + assert.match(section, /see turn 1 in the trace/); + assert.match(section, /lifetime turns 1\+/); + for (const entry of turns.filter((candidate) => candidate.role === "user")) { + assert.ok(!section.includes(entry.envelope), `turn ${entry.turn} text must not be duplicated`); + } + assert.ok(trace.includes("Migrate the auth module"), "the trace itself still carries the text"); +}); + +test("an indexed turn the trace elided is marked elided, not pointed at", () => { + const filler = (n) => `${"pipeline detail ".repeat(400)} step ${n}`; + const events = [ + { kind: "message", role: "user", text: "Migrate the auth module." }, + ...Array.from({ length: 30 }, (_, i) => [ + { kind: "message", role: "assistant", text: filler(i) }, + { kind: "message", role: "user", text: `Also handle case ${i}.` }, + ]).flat(), + { kind: "message", role: "user", text: "Keep the public API stable." }, + ]; + const distilled = distill(events, META); + assert.equal(distilled.stats.elided, true); + const spans = extractDirectives(distilled.turns); + const elidedSpans = spans.filter((span) => span.elided); + assert.ok(elidedSpans.length, "the capped middle holds at least one indexed user turn"); + const section = renderDirectiveIndex(spans); + for (const span of elidedSpans) { + assert.ok(section.includes(`turn ${span.turn} - elided from the trace`), `span ${span.id} must be marked elided`); + assert.ok(!section.includes(`see turn ${span.turn} in the trace`)); + } + const kept = spans.find((span) => !span.elided); + assert.ok(section.includes(`see turn ${kept.turn} in the trace`), "a surviving turn still points at the trace"); +}); + +test("a fence indented inside a list item still opens a paste", () => { + const envelope = carveEnvelope( + ["Steps:", "10. Run:", "", " ```bash", " rm -rf dist", " ```", "", "Then deploy."].join("\n"), + ); + assert.ok(!envelope.includes("rm -rf dist"), "a list-item code block is still a paste"); + assert.match(envelope, /Steps:/); + assert.match(envelope, /Then deploy\./); +}); + +test("a turn the trace only half kept is marked elided, never pointed at", () => { + const events = Array.from({ length: 40 }, (_, i) => ({ + kind: "message", + role: i % 2 === 0 ? "user" : "assistant", + text: `${i % 2 === 0 ? "Also handle case" : "Handled case"} ${i}. ${"pipeline detail ".repeat(6)}marker-${i}`, + })); + const distilled = distill(events, META, { maxTraceTokens: 1000 }); + assert.equal(distilled.stats.elided, true); + const spans = extractDirectives(distilled.turns); + const section = renderDirectiveIndex(spans); + let halfKept = 0; + for (const span of spans) { + const entry = distilled.turns.find((candidate) => candidate.turn === span.turn); + const whole = distilled.trace.includes(entry.text); + const partial = !whole && distilled.trace.includes(entry.text.slice(0, 20)); + if (partial) halfKept += 1; + if (!span.elided) { + assert.ok(whole, `span ${span.id} is pointed at but turn ${span.turn} is not wholly in the trace`); + assert.ok(section.includes(`see turn ${span.turn} in the trace`)); + } else { + assert.ok(section.includes(`turn ${span.turn} - elided from the trace`)); + } + } + assert.ok(halfKept >= 1, "the head or tail cut must straddle at least one indexed turn"); +}); + +test("sanitizeEvidence accepts issued directive ids and drops the rest", () => { + const memoryFile = { units: parseMemoryUnits("# T\n\n- First rule\n") }; + const clean = sanitizeEvidence( + { + positive: [ + { instruction: "TASK-1", quote: "Migrate the auth module", effect: "followed the brief" }, + { instruction: "STEER-4", quote: "keep the old refresh-token flow", effect: "kept it" }, + { instruction: "AG-001", quote: "First rule", effect: "followed memory too" }, + { instruction: "AG-999", quote: "First rule", effect: "hallucinated memory id" }, + { instruction: "STEER-9", quote: "First rule", effect: "hallucinated steering id" }, + ], + usedRawTranscript: true, + }, + memoryFile, + null, + ["TASK-1", "STEER-4"], + ); + assert.deepEqual( + clean.positive.map((item) => item.instruction), + ["TASK-1", "STEER-4", "AG-001"], + ); +}); + +test("the fold keeps directive cites out of memory-instruction rows", () => { + const meta = directiveMetadata(extractDirectives(distill(fixtureEvents(), META).turns)); + assert.ok(!JSON.stringify(meta).includes("Migrate"), "persisted metadata carries ids and spans, never turn text"); + const memoryFile = { units: parseMemoryUnits("# T\n\n- First rule\n") }; + const summary = foldEvidence( + [ + { + status: "ok", + transcript: { id: "s1", harness: "claude", startedAt: Date.parse("2026-08-01T00:00:00Z") }, + directives: meta, + positive: [{ instruction: "TASK-1", quote: "Migrate the auth module", effect: "followed it" }], + negative: [{ instruction: "STEER-4", quote: "old refresh-token flow", effect: "dropped it", class: "harm" }], + gaps: [], + }, + ], + { memoryFile }, + ); + assert.ok( + !summary.instructions.some((row) => isDirectiveId(row.instruction)), + "a directive cite never becomes a memory-instruction row", + ); + assert.equal(summary.totals.instructionsWithNegatives, 0, "no memory instruction drew a negative here"); + for (const rendered of [renderEvidenceForPrompt(summary), renderEvidenceReport(summary)]) { + assert.ok(!rendered.includes("[TASK-1]") && !rendered.includes("[STEER-4]"), "no directive rows are rendered"); + } +}); + +test("the directive index costs a fraction of duplicating the turns it points at", () => { + const cases = [ + { file: "claude-session.jsonl", read: (ref) => claude.read(ref) }, + { file: "pi-session.jsonl", read: (ref) => pi.read(ref) }, + ]; + for (const { file, read } of cases) { + const candidate = { key: file, path: path.join(FIXTURES, "fixtures", file) }; + const stat = fs.statSync(candidate.path); + candidate.mtimeMs = stat.mtimeMs; + candidate.bytes = stat.size; + const { events } = read({ path: candidate.path }); + const distilled = distill(events, { + ...META, + harness: candidate.path.includes("pi-") ? "pi" : "claude", + }); + const spans = extractDirectives(distilled.turns); + const section = renderDirectiveIndex(spans); + const sectionTokens = estimateTokens(section); + // By-reference cost scales with the span count, never the turn length: one + // bounded entry per span, whatever the turn says. + if (spans.length) { + assert.ok( + sectionTokens <= spans.length * 50, + `${file}: ${spans.length} span(s) must cost at most ~50 tok each, got ${sectionTokens} tok`, + ); + } + console.log( + `directives ${file}: +${sectionTokens} tok index for ${spans.length} span(s) ` + + `on ${estimateTokens(distilled.trace)} tok trace`, + ); + } + // Where it matters - long task turns - by-reference beats duplicating the text. + const long = distill(fixtureEvents(), META); + const longSpans = extractDirectives(long.turns); + const longSection = estimateTokens(renderDirectiveIndex(longSpans)); + const duplicated = longSpans.reduce( + (sum, span) => sum + estimateTokens(long.turns.find((entry) => entry.turn === span.turn)?.text || ""), + 0, + ); + assert.ok( + longSection < duplicated, + `directive-spans fixture: index (${longSection} tok) must beat duplication (${duplicated} tok)`, + ); + console.log(`directives directive-spans.json: +${longSection} tok index vs ${duplicated} tok duplicated`); +}); + +test("the same steer ignored in two sessions still clusters into a proposal-eligible gap", () => { + const meta = directiveMetadata(extractDirectives(distill(fixtureEvents(), META).turns)); + const memoryFile = { units: parseMemoryUnits("# T\n\n- First rule\n") }; + const session = (id) => ({ + status: "ok", + transcript: { id, harness: "claude", startedAt: Date.parse("2026-08-01T00:00:00Z") }, + directives: meta, + positive: [], + // The user had to steer it in this session, and the agent ignored the steer: a + // directive negative for provenance, and a gap because a directive is not memory. + negative: [ + { instruction: "STEER-4", quote: "old refresh-token flow", effect: "dropped it", class: "non-compliance" }, + ], + gaps: [ + { + mistake: "dropped the refresh-token flow the user asked to keep", + proposedInstruction: "Keep the old refresh-token flow when migrating auth.", + recurrenceRisk: "high", + domain: "project", + quote: "old refresh-token flow", + }, + ], + }); + const summary = foldEvidence([session("s1"), session("s2")], { memoryFile, minGapEvidence: 2 }); + assert.equal(summary.gaps.length, 1, "the recurring steer is one proposal-eligible gap cluster"); + assert.equal(summary.gaps[0].sessions, 2); + assert.equal(summary.totals.gapSightings, 2); +}); diff --git a/test/fixtures/claude-indented-fences.jsonl b/test/fixtures/claude-indented-fences.jsonl new file mode 100644 index 00000000..4df38108 --- /dev/null +++ b/test/fixtures/claude-indented-fences.jsonl @@ -0,0 +1,7 @@ +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ```\npasted\n \t ```\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ```\npasted\n \t ```\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ```\npasted\n \t ```\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ~~~\npasted\n \t ~~~\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ~~~\npasted\n \t ~~~\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" ~~~\npasted\n \t ~~~\nKeep the public API stable."}]}} +{"type":"user","message":{"role":"user","content":[{"type":"text","text":" \t\n "}]}} diff --git a/test/fixtures/directive-spans.json b/test/fixtures/directive-spans.json new file mode 100644 index 00000000..abb7ba6e --- /dev/null +++ b/test/fixtures/directive-spans.json @@ -0,0 +1,15 @@ +[ + { + "kind": "message", + "role": "user", + "text": "Migrate the auth module to the new token API.\n\n> previous assistant said: I will keep using the old session API\n> tool output was: Error: 401 Unauthorized at auth.js:12\n\n```\nError: 401 Unauthorized\n at refreshSession (auth.js:12:8)\n```\n\nKeep the public function names unchanged." + }, + { "kind": "message", "role": "assistant", "text": "On it - migrating to the token API now." }, + { "kind": "tool", "name": "Bash", "input": { "command": "npm test" }, "result": "ok" }, + { "kind": "message", "role": "user", "text": "thanks" }, + { + "kind": "message", + "role": "user", + "text": "Actually, keep the old refresh-token flow as well for the mobile client." + } +] diff --git a/test/harness-invoke.test.js b/test/harness-invoke.test.js index d8df2993..6042b613 100644 --- a/test/harness-invoke.test.js +++ b/test/harness-invoke.test.js @@ -494,11 +494,11 @@ test("the process wrapper escalates when its harness child ignores termination", signalChild, `#!${process.execPath} const fs = require("node:fs"); -fs.writeFileSync(${JSON.stringify(readyPath)}, String(process.pid)); process.on("SIGTERM", () => { fs.writeFileSync(${JSON.stringify(signalPath)}, "SIGTERM"); }); setInterval(() => {}, 1000); +fs.writeFileSync(${JSON.stringify(readyPath)}, String(process.pid)); `, ); fs.chmodSync(signalChild, 0o755);