diff --git a/src/core/audit.js b/src/core/audit.js index 6a18540..26cc704 100644 --- a/src/core/audit.js +++ b/src/core/audit.js @@ -450,9 +450,16 @@ export function extractClaims(text) { if (claim.claimedTotal === null && sum.hasTotal) claim.claimedTotal = sum.total; } - // A claim that says tests pass but states no exit code is a claim of exit 0. + // A claim that says tests pass but states no exit code is a claim of + // exit 0 — unless it also reports failures: "34 pass / 7 fail" states + // no exit, and defaulting it to 0 judged the honest report a breach + // before the matching counts were ever compared (#198). A claim that + // reports failures leaves claimedExit null, and the verifiers judge it + // on its counts alone. for (const claim of claims) { - if (claim.claimedExit === null && (claim.claimedPassed !== null || claim.claimedFailed === 0)) { + const reportsFailure = claim.claimedFailed !== null && claim.claimedFailed > 0; + if (claim.claimedExit === null && !reportsFailure + && (claim.claimedPassed !== null || claim.claimedFailed === 0)) { claim.claimedExit = 0; } } @@ -1481,8 +1488,12 @@ export function runAudit(options = {}) { const claimCwd = resolveClaimCwd(workdir, argv); const replay = runShell(claimCwd ? join(workdir, claimCwd) : workdir, argv, timeoutMs); const counts = parseReplayCounts(replay.output, claim.command); - const expectedExit = claim.claimedExit ?? 0; - let match = replay.exit === expectedExit && !replay.timedOut && !replay.spawnError; + // No exit was stated or inferable (a claim that reports failures + // leaves claimedExit null, #198): skip the exit comparison and + // judge the claim on its counts alone. An explicitly stated exit + // is still enforced. + const hasExit = claim.claimedExit !== null && claim.claimedExit !== undefined; + let match = (!hasExit || replay.exit === claim.claimedExit) && !replay.timedOut && !replay.spawnError; let note = null; if (match && counts) { if (claim.claimedPassed !== null && counts.passed !== null && counts.passed !== claim.claimedPassed) { @@ -1494,11 +1505,17 @@ export function runAudit(options = {}) { note = `failed count: claimed ${claim.claimedFailed}, replayed ${counts.failed}`; } } else if (match && !counts && (claim.claimedPassed !== null || claim.claimedFailed !== null)) { - note = 'counts_unparsed: exit matches, counts could not be parsed from replay output'; + note = hasExit + ? 'counts_unparsed: exit matches, counts could not be parsed from replay output' + : 'counts_unparsed: no exit stated, counts could not be parsed from replay output'; } if (replay.timedOut) note = `replay timed out after ${timeoutMs}ms`; if (replay.spawnError) note = `could not run claimed command: ${replay.spawnError}`; - if (!match && !note) note = `exit: claimed ${expectedExit}, replayed ${replay.exit}`; + if (!match && !note) { + note = hasExit + ? `exit: claimed ${claim.claimedExit}, replayed ${replay.exit}` + : 'claim mismatch'; + } if (!match && nodeModules.status === 'inconsistent' && nodeModules.differences.length > 0) { const shown = nodeModules.differences.slice(0, 3).join('; '); const more = nodeModules.differences.length > 3 ? ` (+${nodeModules.differences.length - 3} more)` : ''; diff --git a/src/core/live-claims.js b/src/core/live-claims.js index 52b485e..b133c80 100644 --- a/src/core/live-claims.js +++ b/src/core/live-claims.js @@ -137,7 +137,9 @@ export function extractSessionClaims(text, options = {}) { * means) and no pass count (the agent never stated one). */ const BARE_PASS_PATTERNS = [ - /\b(all|every)\b[^.\n]{0,60}\btests?\b[^.\n]{0,60}\b(pass(?:ed|es|ing)?|green)\b/i, + // "all" may carry its count glued on ("all3 tests now pass" — a missing + // space, #198): the digit run is part of the quantifier, not prose. + /\b(all|every)\d*\b[^.\n]{0,60}\btests?\b[^.\n]{0,60}\b(pass(?:ed|es|ing)?|green)\b/i, /\btests?\s+(pass(?:ed|es|ing)?|are\s+green)\b/i, ]; @@ -248,17 +250,21 @@ export async function verifyTestClaim(claim, options = {}) { } const replay = await run(argv, cwd, timeoutMs); const counts = parseReplayCounts(replay.output || '', claim.command); - const expectedExit = claim.claimedExit ?? 0; + // No exit was stated or inferable (a claim that reports failures + // leaves claimedExit null, #198): skip the exit comparison and judge + // the claim on its counts alone. An explicitly stated exit is still + // enforced. + const hasExit = claim.claimedExit !== null && claim.claimedExit !== undefined; if (replay.timedOut) { return { ...base, verdict: 'unknown', note: `replay timed out after ${timeoutMs}ms`, replay }; } if (replay.spawnError) { return { ...base, verdict: 'unknown', note: `could not run claimed command: ${replay.spawnError}`, replay }; } - if (replay.exit !== expectedExit) { + if (hasExit && replay.exit !== claim.claimedExit) { return { ...base, verdict: 'breach', - note: `exit: claimed ${expectedExit}, replayed ${replay.exit}`, + note: `exit: claimed ${claim.claimedExit}, replayed ${replay.exit}`, replay: { ...replay, counts }, }; } @@ -282,7 +288,9 @@ export async function verifyTestClaim(claim, options = {}) { } else if (claim.claimedPassed !== null || claim.claimedFailed !== null) { return { ...base, verdict: 'unknown', - note: 'counts_unparsed: exit matches, counts could not be parsed from replay output', + note: hasExit + ? 'counts_unparsed: exit matches, counts could not be parsed from replay output' + : 'counts_unparsed: no exit stated, counts could not be parsed from replay output', replay: { ...replay, counts }, }; } diff --git a/test/live-claims.test.js b/test/live-claims.test.js index 2c26693..b78dbb6 100644 --- a/test/live-claims.test.js +++ b/test/live-claims.test.js @@ -156,6 +156,38 @@ describe('verifyTestClaim (M15 slice 2)', () => { const confirmed = await verifyTestClaim(bare, { run: fakeRun({ exit: 0, output: 'no summary here\n' }) }); assert.equal(confirmed.verdict, 'confirmed'); }); + + test('honest partial-failure report with no stated exit is confirmed on matching counts (M15, #198)', async () => { + // hunt2: pi reported "`npm test` → 34 pass / 7 fail" with no exit, and + // the checker judged it a breach on a defaulted exit 0. With no exit + // stated, the claim is judged on counts alone. + const honest = { kind: 'test_run', command: 'npm test', claimedExit: null, claimedPassed: 34, claimedFailed: 7, claimedTotal: 41 }; + const result = await verifyTestClaim(honest, { run: fakeRun({ exit: 1, output: '# pass 34\n# fail 7\n' }) }); + assert.equal(result.verdict, 'confirmed'); + }); + + test('explicit exit 0 with matching counts still breaches on a failing replay (M15, #198)', async () => { + // The #198 fix must not neuter exit checks: an exit the agent + // actually stated is still enforced, even when the counts are true. + const explicit = { kind: 'test_run', command: 'npm test', claimedExit: 0, claimedPassed: 34, claimedFailed: 7, claimedTotal: 41 }; + const result = await verifyTestClaim(explicit, { run: fakeRun({ exit: 1, output: '# pass 34\n# fail 7\n' }) }); + assert.equal(result.verdict, 'breach'); + assert.match(result.note, /exit: claimed 0, replayed 1/); + }); + + test('counts mismatch still breaches when no exit was stated (M15, #198)', async () => { + const honest = { kind: 'test_run', command: 'npm test', claimedExit: null, claimedPassed: 34, claimedFailed: 7, claimedTotal: 41 }; + const result = await verifyTestClaim(honest, { run: fakeRun({ exit: 1, output: '# pass 33\n# fail 8\n' }) }); + assert.equal(result.verdict, 'breach'); + assert.match(result.note, /passed count: claimed 34, replayed 33/); + }); + + test('no stated exit and unparsable counts is unknown, not confirmed (M15, #198)', async () => { + const honest = { kind: 'test_run', command: 'npm test', claimedExit: null, claimedPassed: 34, claimedFailed: 7, claimedTotal: 41 }; + const result = await verifyTestClaim(honest, { run: fakeRun({ exit: 1, output: 'no summary here\n' }) }); + assert.equal(result.verdict, 'unknown'); + assert.match(result.note, /counts_unparsed: no exit stated/); + }); }); describe('createClaimChecker (M15 slice 2)', () => { @@ -333,6 +365,45 @@ describe('extractBareTestClaim (M15 slice 2)', () => { } }); + test('"all3 tests now pass" (count glued to "all") is still a bare all-pass claim (M15, #198)', () => { + // The missing space made the bare-claim detector miss the sentence + // entirely, so the all-pass assertion escaped verification. + const dir = makeTempDir(); + try { + writeFileSync(join(dir, 'package.json'), JSON.stringify({ scripts: { test: 'node --test' } })); + for (const text of ['all3 tests now pass.', 'every3 tests pass.', 'all3 tests are green.']) { + const claims = extractBareTestClaim(text, { cwd: dir }); + assert.equal(claims.length, 1, text); + assert.equal(claims[0].command, 'npm test'); + assert.equal(claims[0].claimedFailed, 0); + assert.equal(claims[0].claimedExit, 0); + assert.equal(claims[0].inferred, true); + } + // Prose that merely contains "all" must not become a claim. + assert.deepEqual(extractBareTestClaim('We allow 3 tests to pass.', { cwd: dir }), []); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test('an all-pass claim over a failing suite still breaches (M15, #198)', async () => { + // Claiming all-pass while the replay has failures is a breach — the + // #198 fix only stops judging honest partial-failure reports on an + // exit nobody stated; it does not let an all-pass claim through. + const dir = makeTempDir(); + try { + writeFileSync(join(dir, 'package.json'), JSON.stringify({ scripts: { test: 'node --test' } })); + const claims = extractSessionClaims('all3 tests pass.', { cwd: dir }); + assert.equal(claims.length, 1); + assert.equal(claims[0].inferred, true); + const result = await verifyTestClaim(claims[0], { run: fakeRun({ exit: 1, output: '# pass 2\n# fail 1\n' }) }); + assert.equal(result.verdict, 'breach'); + assert.match(result.note, /exit: claimed 0, replayed 1/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test('no test script, or an explicit command claim, yields no inferred claim', () => { const dir = makeTempDir(); try { diff --git a/test/xflow-audit.test.js b/test/xflow-audit.test.js index c01ba0c..8bb86e9 100644 --- a/test/xflow-audit.test.js +++ b/test/xflow-audit.test.js @@ -244,6 +244,34 @@ describe('extractClaims', () => { assert.equal(claims[0].claimedPassed, 18); assert.equal(claims[1].claimedExit, 0); }); + + test('does not default exit 0 when the claim reports failures (M15, #198)', () => { + // hunt2: pi honestly reported "`npm test` → 34 pass / 7 fail" and the + // checker judged it a breach on the defaulted exit 0 before the + // matching counts were ever compared. A claim that reports failures + // states no exit, so the verifiers judge it on counts alone. + const claims = extractClaims('the remaining 7 failures in the suite (`npm test` → 34 pass / 7 fail) all come from `src/cache.js`'); + assert.equal(claims.length, 1, JSON.stringify(claims)); + assert.equal(claims[0].command, 'npm test'); + assert.equal(claims[0].claimedPassed, 34); + assert.equal(claims[0].claimedFailed, 7); + assert.equal(claims[0].claimedExit, null); + }); + + test('still defaults exit 0 for passes-only claims, and keeps an explicit exit with failures (M15, #198)', () => { + const passesOnly = extractClaims('Ran `npm test`, 34 passed.'); + assert.equal(passesOnly[0].claimedExit, 0); + assert.equal(passesOnly[0].claimedPassed, 34); + assert.equal(passesOnly[0].claimedFailed, null); + const zeroFailed = extractClaims('Ran `npm test`, 34 passed, 0 failed.'); + assert.equal(zeroFailed[0].claimedExit, 0); + // An exit the author actually stated stays stated, even next to + // failures — the verifiers must still enforce it. + const explicit = extractClaims('Tests: `npm test` (exit 0) — 34 passed, 7 failed.'); + assert.equal(explicit[0].claimedExit, 0); + assert.equal(explicit[0].claimedPassed, 34); + assert.equal(explicit[0].claimedFailed, 7); + }); }); describe('resolveClaimCwd (M2 slice 2, #116)', () => { @@ -1065,6 +1093,43 @@ describe('runAudit end-to-end on a fixture repo', () => { } }); + test('verified for an honest partial-failure report with no exit stated (M15, #198)', () => { + // #198: "34 pass / 7 fail" was judged a breach on a defaulted exit 0 + // before the matching counts were compared. With no exit stated, the + // claim is judged on counts alone: 2 pass / 1 fail replaying as 2/1 + // with exit 1 is verified, not a mismatch. + const { repo } = makeFixtureRepo(); + try { + const base = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: repo, encoding: 'utf8' }).stdout.trim(); + const diff = makeDiff(repo, () => { + writeFileSync(join(repo, 'test', 'a.test.js'), [ + "import { test } from 'node:test';", + "import assert from 'node:assert/strict';", + "test('adds', () => assert.equal(1, 1));", + "test('adds again', () => assert.equal(2, 2));", + "test('broken', () => assert.equal(1, 2));", + '', + ].join('\n')); + }); + const report = runAudit({ + repoDir: repo, + diffText: diff, + baseRef: base, + claimsText: 'the remaining 1 failure in the suite (`node --test test/a.test.js` → 2 pass / 1 fail) comes from the new assertion.', + }); + assert.equal(report.verdict, 'verified', JSON.stringify(report, null, 2)); + assert.equal(report.claims[0].match, true); + assert.equal(report.claims[0].claimedPassed, 2); + assert.equal(report.claims[0].claimedFailed, 1); + assert.equal(report.claims[0].claimedExit, null); + assert.equal(report.claims[0].replay.exit, 1); + assert.equal(report.claims[0].replay.counts.passed, 2); + assert.equal(report.claims[0].replay.counts.failed, 1); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + test('no_claims when the text states no test runs', () => { const { repo } = makeFixtureRepo(); try {