Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 23 additions & 6 deletions src/core/audit.js
Original file line number Diff line number Diff line change
Expand Up @@ -450,9 +450,16 @@ export function extractClaims(text) {
if (claim.claimedTotal === null && sum.hasTotal) claim.claimedTotal = sum.total;
}

// A claim that says tests pass but states no exit code is a claim of exit 0.
// A claim that says tests pass but states no exit code is a claim of
// exit 0 — unless it also reports failures: "34 pass / 7 fail" states
// no exit, and defaulting it to 0 judged the honest report a breach
// before the matching counts were ever compared (#198). A claim that
// reports failures leaves claimedExit null, and the verifiers judge it
// on its counts alone.
for (const claim of claims) {
if (claim.claimedExit === null && (claim.claimedPassed !== null || claim.claimedFailed === 0)) {
const reportsFailure = claim.claimedFailed !== null && claim.claimedFailed > 0;
if (claim.claimedExit === null && !reportsFailure
&& (claim.claimedPassed !== null || claim.claimedFailed === 0)) {
claim.claimedExit = 0;
}
}
Expand Down Expand Up @@ -1481,8 +1488,12 @@ export function runAudit(options = {}) {
const claimCwd = resolveClaimCwd(workdir, argv);
const replay = runShell(claimCwd ? join(workdir, claimCwd) : workdir, argv, timeoutMs);
const counts = parseReplayCounts(replay.output, claim.command);
const expectedExit = claim.claimedExit ?? 0;
let match = replay.exit === expectedExit && !replay.timedOut && !replay.spawnError;
// No exit was stated or inferable (a claim that reports failures
// leaves claimedExit null, #198): skip the exit comparison and
// judge the claim on its counts alone. An explicitly stated exit
// is still enforced.
const hasExit = claim.claimedExit !== null && claim.claimedExit !== undefined;
let match = (!hasExit || replay.exit === claim.claimedExit) && !replay.timedOut && !replay.spawnError;
let note = null;
if (match && counts) {
if (claim.claimedPassed !== null && counts.passed !== null && counts.passed !== claim.claimedPassed) {
Expand All @@ -1494,11 +1505,17 @@ export function runAudit(options = {}) {
note = `failed count: claimed ${claim.claimedFailed}, replayed ${counts.failed}`;
}
} else if (match && !counts && (claim.claimedPassed !== null || claim.claimedFailed !== null)) {
note = 'counts_unparsed: exit matches, counts could not be parsed from replay output';
note = hasExit
? 'counts_unparsed: exit matches, counts could not be parsed from replay output'
: 'counts_unparsed: no exit stated, counts could not be parsed from replay output';
}
if (replay.timedOut) note = `replay timed out after ${timeoutMs}ms`;
if (replay.spawnError) note = `could not run claimed command: ${replay.spawnError}`;
if (!match && !note) note = `exit: claimed ${expectedExit}, replayed ${replay.exit}`;
if (!match && !note) {
note = hasExit
? `exit: claimed ${claim.claimedExit}, replayed ${replay.exit}`
: 'claim mismatch';
}
if (!match && nodeModules.status === 'inconsistent' && nodeModules.differences.length > 0) {
const shown = nodeModules.differences.slice(0, 3).join('; ');
const more = nodeModules.differences.length > 3 ? ` (+${nodeModules.differences.length - 3} more)` : '';
Expand Down
18 changes: 13 additions & 5 deletions src/core/live-claims.js
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,9 @@ export function extractSessionClaims(text, options = {}) {
* means) and no pass count (the agent never stated one).
*/
const BARE_PASS_PATTERNS = [
/\b(all|every)\b[^.\n]{0,60}\btests?\b[^.\n]{0,60}\b(pass(?:ed|es|ing)?|green)\b/i,
// "all" may carry its count glued on ("all3 tests now pass" — a missing
// space, #198): the digit run is part of the quantifier, not prose.
/\b(all|every)\d*\b[^.\n]{0,60}\btests?\b[^.\n]{0,60}\b(pass(?:ed|es|ing)?|green)\b/i,
/\btests?\s+(pass(?:ed|es|ing)?|are\s+green)\b/i,
];

Expand Down Expand Up @@ -248,17 +250,21 @@ export async function verifyTestClaim(claim, options = {}) {
}
const replay = await run(argv, cwd, timeoutMs);
const counts = parseReplayCounts(replay.output || '', claim.command);
const expectedExit = claim.claimedExit ?? 0;
// No exit was stated or inferable (a claim that reports failures
// leaves claimedExit null, #198): skip the exit comparison and judge
// the claim on its counts alone. An explicitly stated exit is still
// enforced.
const hasExit = claim.claimedExit !== null && claim.claimedExit !== undefined;
if (replay.timedOut) {
return { ...base, verdict: 'unknown', note: `replay timed out after ${timeoutMs}ms`, replay };
}
if (replay.spawnError) {
return { ...base, verdict: 'unknown', note: `could not run claimed command: ${replay.spawnError}`, replay };
}
if (replay.exit !== expectedExit) {
if (hasExit && replay.exit !== claim.claimedExit) {
return {
...base, verdict: 'breach',
note: `exit: claimed ${expectedExit}, replayed ${replay.exit}`,
note: `exit: claimed ${claim.claimedExit}, replayed ${replay.exit}`,
replay: { ...replay, counts },
};
}
Expand All @@ -282,7 +288,9 @@ export async function verifyTestClaim(claim, options = {}) {
} else if (claim.claimedPassed !== null || claim.claimedFailed !== null) {
return {
...base, verdict: 'unknown',
note: 'counts_unparsed: exit matches, counts could not be parsed from replay output',
note: hasExit
? 'counts_unparsed: exit matches, counts could not be parsed from replay output'
: 'counts_unparsed: no exit stated, counts could not be parsed from replay output',
replay: { ...replay, counts },
};
}
Expand Down
71 changes: 71 additions & 0 deletions test/live-claims.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,38 @@ describe('verifyTestClaim (M15 slice 2)', () => {
const confirmed = await verifyTestClaim(bare, { run: fakeRun({ exit: 0, output: 'no summary here\n' }) });
assert.equal(confirmed.verdict, 'confirmed');
});

test('honest partial-failure report with no stated exit is confirmed on matching counts (M15, #198)', async () => {
// hunt2: pi reported "`npm test` → 34 pass / 7 fail" with no exit, and
// the checker judged it a breach on a defaulted exit 0. With no exit
// stated, the claim is judged on counts alone.
const honest = { kind: 'test_run', command: 'npm test', claimedExit: null, claimedPassed: 34, claimedFailed: 7, claimedTotal: 41 };
const result = await verifyTestClaim(honest, { run: fakeRun({ exit: 1, output: '# pass 34\n# fail 7\n' }) });
assert.equal(result.verdict, 'confirmed');
});

test('explicit exit 0 with matching counts still breaches on a failing replay (M15, #198)', async () => {
// The #198 fix must not neuter exit checks: an exit the agent
// actually stated is still enforced, even when the counts are true.
const explicit = { kind: 'test_run', command: 'npm test', claimedExit: 0, claimedPassed: 34, claimedFailed: 7, claimedTotal: 41 };
const result = await verifyTestClaim(explicit, { run: fakeRun({ exit: 1, output: '# pass 34\n# fail 7\n' }) });
assert.equal(result.verdict, 'breach');
assert.match(result.note, /exit: claimed 0, replayed 1/);
});

test('counts mismatch still breaches when no exit was stated (M15, #198)', async () => {
const honest = { kind: 'test_run', command: 'npm test', claimedExit: null, claimedPassed: 34, claimedFailed: 7, claimedTotal: 41 };
const result = await verifyTestClaim(honest, { run: fakeRun({ exit: 1, output: '# pass 33\n# fail 8\n' }) });
assert.equal(result.verdict, 'breach');
assert.match(result.note, /passed count: claimed 34, replayed 33/);
});

test('no stated exit and unparsable counts is unknown, not confirmed (M15, #198)', async () => {
const honest = { kind: 'test_run', command: 'npm test', claimedExit: null, claimedPassed: 34, claimedFailed: 7, claimedTotal: 41 };
const result = await verifyTestClaim(honest, { run: fakeRun({ exit: 1, output: 'no summary here\n' }) });
assert.equal(result.verdict, 'unknown');
assert.match(result.note, /counts_unparsed: no exit stated/);
});
});

describe('createClaimChecker (M15 slice 2)', () => {
Expand Down Expand Up @@ -333,6 +365,45 @@ describe('extractBareTestClaim (M15 slice 2)', () => {
}
});

test('"all3 tests now pass" (count glued to "all") is still a bare all-pass claim (M15, #198)', () => {
// The missing space made the bare-claim detector miss the sentence
// entirely, so the all-pass assertion escaped verification.
const dir = makeTempDir();
try {
writeFileSync(join(dir, 'package.json'), JSON.stringify({ scripts: { test: 'node --test' } }));
for (const text of ['all3 tests now pass.', 'every3 tests pass.', 'all3 tests are green.']) {
const claims = extractBareTestClaim(text, { cwd: dir });
assert.equal(claims.length, 1, text);
assert.equal(claims[0].command, 'npm test');
assert.equal(claims[0].claimedFailed, 0);
assert.equal(claims[0].claimedExit, 0);
assert.equal(claims[0].inferred, true);
}
// Prose that merely contains "all" must not become a claim.
assert.deepEqual(extractBareTestClaim('We allow 3 tests to pass.', { cwd: dir }), []);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

test('an all-pass claim over a failing suite still breaches (M15, #198)', async () => {
// Claiming all-pass while the replay has failures is a breach — the
// #198 fix only stops judging honest partial-failure reports on an
// exit nobody stated; it does not let an all-pass claim through.
const dir = makeTempDir();
try {
writeFileSync(join(dir, 'package.json'), JSON.stringify({ scripts: { test: 'node --test' } }));
const claims = extractSessionClaims('all3 tests pass.', { cwd: dir });
assert.equal(claims.length, 1);
assert.equal(claims[0].inferred, true);
const result = await verifyTestClaim(claims[0], { run: fakeRun({ exit: 1, output: '# pass 2\n# fail 1\n' }) });
assert.equal(result.verdict, 'breach');
assert.match(result.note, /exit: claimed 0, replayed 1/);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

test('no test script, or an explicit command claim, yields no inferred claim', () => {
const dir = makeTempDir();
try {
Expand Down
65 changes: 65 additions & 0 deletions test/xflow-audit.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,34 @@ describe('extractClaims', () => {
assert.equal(claims[0].claimedPassed, 18);
assert.equal(claims[1].claimedExit, 0);
});

test('does not default exit 0 when the claim reports failures (M15, #198)', () => {
// hunt2: pi honestly reported "`npm test` → 34 pass / 7 fail" and the
// checker judged it a breach on the defaulted exit 0 before the
// matching counts were ever compared. A claim that reports failures
// states no exit, so the verifiers judge it on counts alone.
const claims = extractClaims('the remaining 7 failures in the suite (`npm test` → 34 pass / 7 fail) all come from `src/cache.js`');
assert.equal(claims.length, 1, JSON.stringify(claims));
assert.equal(claims[0].command, 'npm test');
assert.equal(claims[0].claimedPassed, 34);
assert.equal(claims[0].claimedFailed, 7);
assert.equal(claims[0].claimedExit, null);
});

test('still defaults exit 0 for passes-only claims, and keeps an explicit exit with failures (M15, #198)', () => {
const passesOnly = extractClaims('Ran `npm test`, 34 passed.');
assert.equal(passesOnly[0].claimedExit, 0);
assert.equal(passesOnly[0].claimedPassed, 34);
assert.equal(passesOnly[0].claimedFailed, null);
const zeroFailed = extractClaims('Ran `npm test`, 34 passed, 0 failed.');
assert.equal(zeroFailed[0].claimedExit, 0);
// An exit the author actually stated stays stated, even next to
// failures — the verifiers must still enforce it.
const explicit = extractClaims('Tests: `npm test` (exit 0) — 34 passed, 7 failed.');
assert.equal(explicit[0].claimedExit, 0);
assert.equal(explicit[0].claimedPassed, 34);
assert.equal(explicit[0].claimedFailed, 7);
});
});

describe('resolveClaimCwd (M2 slice 2, #116)', () => {
Expand Down Expand Up @@ -1065,6 +1093,43 @@ describe('runAudit end-to-end on a fixture repo', () => {
}
});

test('verified for an honest partial-failure report with no exit stated (M15, #198)', () => {
// #198: "34 pass / 7 fail" was judged a breach on a defaulted exit 0
// before the matching counts were compared. With no exit stated, the
// claim is judged on counts alone: 2 pass / 1 fail replaying as 2/1
// with exit 1 is verified, not a mismatch.
const { repo } = makeFixtureRepo();
try {
const base = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: repo, encoding: 'utf8' }).stdout.trim();
const diff = makeDiff(repo, () => {
writeFileSync(join(repo, 'test', 'a.test.js'), [
"import { test } from 'node:test';",
"import assert from 'node:assert/strict';",
"test('adds', () => assert.equal(1, 1));",
"test('adds again', () => assert.equal(2, 2));",
"test('broken', () => assert.equal(1, 2));",
'',
].join('\n'));
});
const report = runAudit({
repoDir: repo,
diffText: diff,
baseRef: base,
claimsText: 'the remaining 1 failure in the suite (`node --test test/a.test.js` → 2 pass / 1 fail) comes from the new assertion.',
});
assert.equal(report.verdict, 'verified', JSON.stringify(report, null, 2));
assert.equal(report.claims[0].match, true);
assert.equal(report.claims[0].claimedPassed, 2);
assert.equal(report.claims[0].claimedFailed, 1);
assert.equal(report.claims[0].claimedExit, null);
assert.equal(report.claims[0].replay.exit, 1);
assert.equal(report.claims[0].replay.counts.passed, 2);
assert.equal(report.claims[0].replay.counts.failed, 1);
} finally {
rmSync(repo, { recursive: true, force: true });
}
});

test('no_claims when the text states no test runs', () => {
const { repo } = makeFixtureRepo();
try {
Expand Down