From 52e00d2887fb40737bf9c6fb543305cb8be94009 Mon Sep 17 00:00:00 2001 From: Nick <209239467+surlymochan@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:42:23 +0000 Subject: [PATCH] fix(live-claims): diagnostic sentences are not test-run claims (#201) A sentence reporting a command failing to run (failed spawn / ENOENT / command not found) is diagnostic context, not an assertion the agent made. The extractor now drops such sentences from the claim set; a command an ordinary sentence also asserts is still claimed. M15 evidence set session 7: the pytest diagnostic sentence no longer yields a claim, so it can no longer be replay-confirmed. 7 new tests. --- src/core/live-claims.js | 101 ++++++++++++++++++++++++++++++++++++++- test/live-claims.test.js | 69 ++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 2 deletions(-) diff --git a/src/core/live-claims.js b/src/core/live-claims.js index b133c80..edc1f27 100644 --- a/src/core/live-claims.js +++ b/src/core/live-claims.js @@ -38,6 +38,17 @@ * Checks run sequentially (one replay at a time) behind a queue so the * live session is never hammered while the agent works. The same * command/path claimed twice in one session is verified once. + * + * M15 slice 3 (#201) tightens the extractor: a sentence that reports a + * command *failing to run* is diagnostic context, not a claim. The + * recorded session 7 of the M15 evidence set says + * `test/env-python.test.js failed … spawnSync("pytest", …) got ENOENT` — + * a diagnosis of a missing runner. The extractor took `pytest` out of + * it as a test-run claim, and because a stale `~/.local/bin/pytest` + * symlink still existed the replay exited 0 and the checker reported + * "confirmed": a statement the agent never made, vouched for. Such a + * sentence is now dropped from the claim set; a command some other, + * ordinary sentence also asserts is still claimed. */ import { spawn } from 'node:child_process'; import { existsSync, readFileSync } from 'node:fs'; @@ -119,9 +130,94 @@ export function extractFileClaims(text) { return claims; } +/** + * Failure signals that mark a sentence as diagnostic context: the agent + * is reporting that a command *failed to run*, not asserting that it ran + * green (M15 slice 3, #201). + * + * The list is deliberately narrow. It names the error codes a failed + * spawn produces, the spawn/exec APIs themselves (a clause that quotes + * one is reporting a call, not asserting a run), and the shell's own + * "not found" wording. A sentence that merely mentions a runner — even + * one that later admits the runner is missing — is still a claim. + */ +const DIAGNOSTIC_SIGNALS = [ + /\bENOENT\b/, + /\bEACCES\b/, + /\bEPERM\b/, + /\bspawnSync\b/, + /\bspawn\s*\(/, + /\bexecSync\b/, + /\bexecFile(?:Sync)?\b/, + /\bchild_process\b/, + /command not found/i, + /not recognized as an internal or external command/i, + /no such file or directory/i, + /failed to spawn/i, + /:\s*not found\b/i, + /\bis not installed\b/i, +]; + +/** True when the sentence reports a failed command run, not a run. */ +export function isDiagnosticContext(text) { + const sentence = String(text || ''); + return DIAGNOSTIC_SIGNALS.some((pattern) => pattern.test(sentence)); +} + +/** + * The audit's own clause split (audit.js `extractClaims`): a line is cut + * at sentence boundaries and at a trailing semicolon. Mirrored here so + * the live path can ask, per clause, the same question the audit asks + * per clause when it harvests commands. + */ +function splitClauses(text) { + const clauses = []; + for (const line of String(text || '').split('\n')) { + for (const clause of line.split(/(?:[.;]\s+|\s+;\s*)/)) { + if (clause.trim()) clauses.push(clause); + } + } + return clauses; +} + +/** + * The commands some *non-diagnostic* clause asserts, mapped to the + * clause that asserts them. A command that only shows up inside + * diagnostic clauses (a failed spawn report) is not a claim at all; a + * command an ordinary sentence also asserts still is — a diagnostic + * mention elsewhere does not retract the assertion. + * + * The command set is harvested with the audit's own extractor, one + * clause at a time, so it is exactly the set the audit would harvest + * from those clauses. + */ +function assertedCommands(text) { + const commands = new Map(); + for (const clause of splitClauses(text)) { + if (isDiagnosticContext(clause)) continue; + for (const claim of extractClaims(clause)) { + if (commands.has(claim.command)) continue; + commands.set(claim.command, claim.source); + } + } + return commands; +} + /** Both claim kinds found in one message. */ export function extractSessionClaims(text, options = {}) { - const test = extractClaims(text).map((claim) => ({ kind: 'test_run', ...claim })); + const asserted = assertedCommands(text); + const test = extractClaims(text) + .filter((claim) => asserted.has(claim.command)) + .map((claim) => { + // The audit keeps the first mention of a command as the claim's + // source. When that first mention was the diagnostic one, point + // the source at the sentence that actually asserts the run. + const source = asserted.get(claim.command); + if (!source || !isDiagnosticContext(claim.source)) { + return { kind: 'test_run', ...claim }; + } + return { kind: 'test_run', ...claim, source }; + }); const files = extractFileClaims(text); const bare = test.length === 0 ? extractBareTestClaim(text, options) : []; return [...test, ...bare, ...files]; @@ -156,7 +252,8 @@ export function defaultTestCommand(cwd) { export function extractBareTestClaim(text, options = {}) { const sentence = String(text || '').split('\n') - .find((line) => BARE_PASS_PATTERNS.some((pattern) => pattern.test(line))); + .find((line) => !isDiagnosticContext(line) + && BARE_PASS_PATTERNS.some((pattern) => pattern.test(line))); if (!sentence) return []; const command = defaultTestCommand(options.cwd); if (!command) return []; diff --git a/test/live-claims.test.js b/test/live-claims.test.js index b78dbb6..4d7862c 100644 --- a/test/live-claims.test.js +++ b/test/live-claims.test.js @@ -449,6 +449,75 @@ describe('extractBareTestClaim (M15 slice 2)', () => { }); }); +describe('diagnostic context is not a claim (M15 slice 3, #201)', () => { + // The recorded session 7 of the M15 evidence set: pi investigated a + // failing test and diagnosed a missing runner. The sentence names the + // runner but asserts nothing — the extractor used to take it as a + // `pytest` test-run claim, and because a stale ~/.local/bin/pytest + // symlink still existed the replay exited 0 and the checker reported + // "confirmed": a statement the agent never made, vouched for. + const SESSION_7_DIAGNOSTIC = 'test/env-python.test.js failed … spawnSync("pytest", …) got ENOENT'; + + test('a failed-spawn diagnosis is not harvested as a test-run claim', () => { + assert.deepEqual(extractSessionClaims(SESSION_7_DIAGNOSTIC), []); + }); + + test('the genuine command in the same message survives', () => { + const claims = extractSessionClaims(`${SESSION_7_DIAGNOSTIC}\nRunning \`python3 -m pytest\` instead.`); + assert.deepEqual(claims.map((claim) => claim.command), ['python3 -m pytest']); + }); + + test('the diagnosis never reaches a verdict, so it cannot be confirmed', async () => { + const checker = createClaimChecker({ run: fakeRun({ exit: 0, output: '' }) }); + checker.checkText(SESSION_7_DIAGNOSTIC); + await checker.drain(5000); + const summary = checker.summary(); + assert.equal(summary.checked, 0); + assert.equal(summary.confirmed, 0); + }); + + test('a diagnostic mention does not retract an assertion elsewhere', async () => { + const checker = createClaimChecker({ run: fakeRun({ exit: 0, output: '# pass 5\n# fail 0\n' }) }); + checker.checkText('spawnSync("pytest", …) got ENOENT. Ran `pytest`, 5 passed.'); + await checker.drain(5000); + const summary = checker.summary(); + assert.equal(summary.checked, 1); + assert.equal(summary.confirmed, 1); + assert.equal(summary.checks[0].command, 'pytest'); + assert.match(summary.checks[0].source, /Ran `pytest`/); + }); + + test('the other command-not-found wordings are diagnostic too', () => { + for (const sentence of [ + 'pytest: command not found', + 'sh: 1: pytest: not found', + 'spawnSync("pytest") failed: ENOENT', + 'pytest is not installed on this machine', + ]) { + assert.deepEqual(extractSessionClaims(sentence), [], sentence); + } + }); + + test('an ordinary sentence that names a runner is still a claim', () => { + const claims = extractSessionClaims('Ran `npm test`, 18 passed, 0 failed.'); + assert.deepEqual(claims.map((claim) => claim.command), ['npm test']); + assert.equal(claims[0].claimedPassed, 18); + }); + + test('an all-pass sentence inside a diagnostic line is not a bare claim', () => { + const dir = makeTempDir(); + try { + writeFileSync(join(dir, 'package.json'), JSON.stringify({ scripts: { test: 'node --test' } })); + assert.deepEqual( + extractBareTestClaim('spawnSync("npm test") got ENOENT — all tests pass.', { cwd: dir }), + [], + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe('pi adapter (M15 slice 2)', () => { const piReal = readFileSync(join(REPO_ROOT, 'test', 'fixtures', 'agent-stream', 'pi-real-list-files.ndjson'), 'utf8');