Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 99 additions & 2 deletions src/core/live-claims.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,17 @@
* Checks run sequentially (one replay at a time) behind a queue so the
* live session is never hammered while the agent works. The same
* command/path claimed twice in one session is verified once.
*
* M15 slice 3 (#201) tightens the extractor: a sentence that reports a
* command *failing to run* is diagnostic context, not a claim. The
* recorded session 7 of the M15 evidence set says
* `test/env-python.test.js failed … spawnSync("pytest", …) got ENOENT` —
* a diagnosis of a missing runner. The extractor took `pytest` out of
* it as a test-run claim, and because a stale `~/.local/bin/pytest`
* symlink still existed the replay exited 0 and the checker reported
* "confirmed": a statement the agent never made, vouched for. Such a
* sentence is now dropped from the claim set; a command some other,
* ordinary sentence also asserts is still claimed.
*/
import { spawn } from 'node:child_process';
import { existsSync, readFileSync } from 'node:fs';
Expand Down Expand Up @@ -119,9 +130,94 @@ export function extractFileClaims(text) {
return claims;
}

/**
* Failure signals that mark a sentence as diagnostic context: the agent
* is reporting that a command *failed to run*, not asserting that it ran
* green (M15 slice 3, #201).
*
* The list is deliberately narrow. It names the error codes a failed
* spawn produces, the spawn/exec APIs themselves (a clause that quotes
* one is reporting a call, not asserting a run), and the shell's own
* "not found" wording. A sentence that merely mentions a runner — even
* one that later admits the runner is missing — is still a claim.
*/
const DIAGNOSTIC_SIGNALS = [
/\bENOENT\b/,
/\bEACCES\b/,
/\bEPERM\b/,
/\bspawnSync\b/,
/\bspawn\s*\(/,
/\bexecSync\b/,
/\bexecFile(?:Sync)?\b/,
/\bchild_process\b/,
/command not found/i,
/not recognized as an internal or external command/i,
/no such file or directory/i,
/failed to spawn/i,
/:\s*not found\b/i,
/\bis not installed\b/i,
];

/** True when the sentence reports a failed command run, not a run. */
export function isDiagnosticContext(text) {
const sentence = String(text || '');
return DIAGNOSTIC_SIGNALS.some((pattern) => pattern.test(sentence));
}

/**
* The audit's own clause split (audit.js `extractClaims`): a line is cut
* at sentence boundaries and at a trailing semicolon. Mirrored here so
* the live path can ask, per clause, the same question the audit asks
* per clause when it harvests commands.
*/
function splitClauses(text) {
const clauses = [];
for (const line of String(text || '').split('\n')) {
for (const clause of line.split(/(?:[.;]\s+|\s+;\s*)/)) {
if (clause.trim()) clauses.push(clause);
}
}
return clauses;
}

/**
* The commands some *non-diagnostic* clause asserts, mapped to the
* clause that asserts them. A command that only shows up inside
* diagnostic clauses (a failed spawn report) is not a claim at all; a
* command an ordinary sentence also asserts still is — a diagnostic
* mention elsewhere does not retract the assertion.
*
* The command set is harvested with the audit's own extractor, one
* clause at a time, so it is exactly the set the audit would harvest
* from those clauses.
*/
function assertedCommands(text) {
const commands = new Map();
for (const clause of splitClauses(text)) {
if (isDiagnosticContext(clause)) continue;
for (const claim of extractClaims(clause)) {
if (commands.has(claim.command)) continue;
commands.set(claim.command, claim.source);
}
}
return commands;
}

/** Both claim kinds found in one message. */
export function extractSessionClaims(text, options = {}) {
const test = extractClaims(text).map((claim) => ({ kind: 'test_run', ...claim }));
const asserted = assertedCommands(text);
const test = extractClaims(text)
.filter((claim) => asserted.has(claim.command))
.map((claim) => {
// The audit keeps the first mention of a command as the claim's
// source. When that first mention was the diagnostic one, point
// the source at the sentence that actually asserts the run.
const source = asserted.get(claim.command);
if (!source || !isDiagnosticContext(claim.source)) {
return { kind: 'test_run', ...claim };
}
return { kind: 'test_run', ...claim, source };
});
const files = extractFileClaims(text);
const bare = test.length === 0 ? extractBareTestClaim(text, options) : [];
return [...test, ...bare, ...files];
Expand Down Expand Up @@ -156,7 +252,8 @@ export function defaultTestCommand(cwd) {

export function extractBareTestClaim(text, options = {}) {
const sentence = String(text || '').split('\n')
.find((line) => BARE_PASS_PATTERNS.some((pattern) => pattern.test(line)));
.find((line) => !isDiagnosticContext(line)
&& BARE_PASS_PATTERNS.some((pattern) => pattern.test(line)));
if (!sentence) return [];
const command = defaultTestCommand(options.cwd);
if (!command) return [];
Expand Down
69 changes: 69 additions & 0 deletions test/live-claims.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,75 @@ describe('extractBareTestClaim (M15 slice 2)', () => {
});
});

describe('diagnostic context is not a claim (M15 slice 3, #201)', () => {
// The recorded session 7 of the M15 evidence set: pi investigated a
// failing test and diagnosed a missing runner. The sentence names the
// runner but asserts nothing — the extractor used to take it as a
// `pytest` test-run claim, and because a stale ~/.local/bin/pytest
// symlink still existed the replay exited 0 and the checker reported
// "confirmed": a statement the agent never made, vouched for.
const SESSION_7_DIAGNOSTIC = 'test/env-python.test.js failed … spawnSync("pytest", …) got ENOENT';

test('a failed-spawn diagnosis is not harvested as a test-run claim', () => {
assert.deepEqual(extractSessionClaims(SESSION_7_DIAGNOSTIC), []);
});

test('the genuine command in the same message survives', () => {
const claims = extractSessionClaims(`${SESSION_7_DIAGNOSTIC}\nRunning \`python3 -m pytest\` instead.`);
assert.deepEqual(claims.map((claim) => claim.command), ['python3 -m pytest']);
});

test('the diagnosis never reaches a verdict, so it cannot be confirmed', async () => {
const checker = createClaimChecker({ run: fakeRun({ exit: 0, output: '' }) });
checker.checkText(SESSION_7_DIAGNOSTIC);
await checker.drain(5000);
const summary = checker.summary();
assert.equal(summary.checked, 0);
assert.equal(summary.confirmed, 0);
});

test('a diagnostic mention does not retract an assertion elsewhere', async () => {
const checker = createClaimChecker({ run: fakeRun({ exit: 0, output: '# pass 5\n# fail 0\n' }) });
checker.checkText('spawnSync("pytest", …) got ENOENT. Ran `pytest`, 5 passed.');
await checker.drain(5000);
const summary = checker.summary();
assert.equal(summary.checked, 1);
assert.equal(summary.confirmed, 1);
assert.equal(summary.checks[0].command, 'pytest');
assert.match(summary.checks[0].source, /Ran `pytest`/);
});

test('the other command-not-found wordings are diagnostic too', () => {
for (const sentence of [
'pytest: command not found',
'sh: 1: pytest: not found',
'spawnSync("pytest") failed: ENOENT',
'pytest is not installed on this machine',
]) {
assert.deepEqual(extractSessionClaims(sentence), [], sentence);
}
});

test('an ordinary sentence that names a runner is still a claim', () => {
const claims = extractSessionClaims('Ran `npm test`, 18 passed, 0 failed.');
assert.deepEqual(claims.map((claim) => claim.command), ['npm test']);
assert.equal(claims[0].claimedPassed, 18);
});

test('an all-pass sentence inside a diagnostic line is not a bare claim', () => {
const dir = makeTempDir();
try {
writeFileSync(join(dir, 'package.json'), JSON.stringify({ scripts: { test: 'node --test' } }));
assert.deepEqual(
extractBareTestClaim('spawnSync("npm test") got ENOENT — all tests pass.', { cwd: dir }),
[],
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});

describe('pi adapter (M15 slice 2)', () => {
const piReal = readFileSync(join(REPO_ROOT, 'test', 'fixtures', 'agent-stream', 'pi-real-list-files.ndjson'), 'utf8');

Expand Down