diff --git a/.changeset/vitest5-announced-report.md b/.changeset/vitest5-announced-report.md new file mode 100644 index 0000000000..754e09c865 --- /dev/null +++ b/.changeset/vitest5-announced-report.md @@ -0,0 +1,17 @@ +--- +id: vitest5-announced-report +tasks: [T13874] +kind: fix +summary: verify --run proves minCount from the JSON report vitest 5 writes under the package it ran in +--- + +With `--reporter=json` and no `outputFile`, vitest 5 writes its report to +`.vitest/json/output.json` under its own root, which is the child's cwd, and +prints `JSON report written to `. Behind `pnpm --filter exec` that +root is the package, so the gate looked at the project root, found nothing, +and failed every `minCount` gate with "emitted no machine-readable test report" +(VidaPeps T2018 to T2022 and T781). + +The gate now also reads the report path the run announces on stdout, as long +as it is inside the project. That report still has to postdate the spawn: +both its mtime and its `startTime` must be at or after the run started. diff --git a/packages/core/src/tasks/__tests__/gate-runner.test.ts b/packages/core/src/tasks/__tests__/gate-runner.test.ts index 7ef201cb63..df17f850c4 100644 --- a/packages/core/src/tasks/__tests__/gate-runner.test.ts +++ b/packages/core/src/tasks/__tests__/gate-runner.test.ts @@ -10,7 +10,7 @@ */ import { createHash } from 'node:crypto'; -import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import type { @@ -961,6 +961,87 @@ describe('minCount reads a JSON report file this run wrote (T13782)', () => { }); expect(result?.result).toBe('error'); }); + + /** + * vitest 5 behind `pnpm --filter exec`: the report lands under the + * package (the child's cwd), and vitest names it on stdout (T13874). + */ + const fromPackage = (dir: string, startTime: string) => + `process.chdir(${JSON.stringify(dir)});` + + writer(`require('node:path').resolve(${JSON.stringify(rel)})`, startTime) + + `console.log('JSON report written to ' + t);`; + + it('reads the report vitest wrote under the package it ran in (T13874)', async () => { + const pkg = join(projectRoot, 'lib', 'db'); + await mkdir(pkg, { recursive: true }); + try { + const [result] = await runGates([gate(fromPackage(pkg, 'Date.now()'))], { projectRoot }); + expect(result?.result, result?.errorMessage).toBe('pass'); + } finally { + await rm(join(projectRoot, 'lib'), { recursive: true, force: true }); + } + }); + + it('red: an announced report from before the run is refused (T13874)', async () => { + const pkg = join(projectRoot, 'lib', 'db'); + await mkdir(pkg, { recursive: true }); + try { + const [result] = await runGates([gate(fromPackage(pkg, 'Date.now() - 60000'))], { + projectRoot, + }); + expect(result?.result).toBe('error'); + } finally { + await rm(join(projectRoot, 'lib'), { recursive: true, force: true }); + } + }); + + it('red: an in-project symlink to a fresh report outside is never read (T13883)', async () => { + const outside = await mkdtemp(join(tmpdir(), 'cleo-gate-outside-')); + const link = join(projectRoot, 'lib', 'db', rel); + await mkdir(dirname(link), { recursive: true }); + await symlink(join(outside, 'output.json'), link); + try { + const source = + writer(JSON.stringify(join(outside, 'output.json')), 'Date.now()') + + `console.log('JSON report written to ' + ${JSON.stringify(link)});`; + const [result] = await runGates([gate(source)], { projectRoot }); + expect(result?.result).toBe('error'); + } finally { + await rm(join(projectRoot, 'lib'), { recursive: true, force: true }); + await rm(outside, { recursive: true, force: true }); + } + }); + + it('red: an earlier fabricated announcement loses to the last one (T13883)', async () => { + const pkg = join(projectRoot, 'lib', 'db'); + await mkdir(pkg, { recursive: true }); + const fake = join(projectRoot, 'fake-report.json'); + // The fabricated report proves 3; vitest's own (announced last) proves 1. + const source = + writer(JSON.stringify(fake), 'Date.now()') + + `console.log('JSON report written to ' + ${JSON.stringify(fake)});` + + `const real=${JSON.stringify(join(pkg, rel))};` + + `fs.mkdirSync(p.dirname(real),{recursive:true});` + + `fs.writeFileSync(real, JSON.stringify(Object.assign(${report(1, 0)},{startTime:Date.now()})));` + + `console.log('JSON report written to ' + real);`; + try { + const [result] = await runGates([gate(source)], { projectRoot }); + expect(result?.result).toBe('fail'); + } finally { + await rm(join(projectRoot, 'lib'), { recursive: true, force: true }); + await rm(fake, { force: true }); + } + }); + + it('red: an announced report outside the project is never read (T13874)', async () => { + const outside = await mkdtemp(join(tmpdir(), 'cleo-gate-outside-')); + try { + const [result] = await runGates([gate(fromPackage(outside, 'Date.now()'))], { projectRoot }); + expect(result?.result).toBe('error'); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); }); describe('gate runner immutable lifetime and bounded evidence', () => { diff --git a/packages/core/src/tasks/gate-runner.ts b/packages/core/src/tasks/gate-runner.ts index d05cfaa31c..8a1a35a701 100644 --- a/packages/core/src/tasks/gate-runner.ts +++ b/packages/core/src/tasks/gate-runner.ts @@ -1813,6 +1813,42 @@ function structuredTestCounts(captured: ProcessCaptureResult): StructuredTestCou */ const REDIRECTED_JSON_REPORT = join('.vitest', 'json', 'output.json'); +/** + * The line vitest 5 prints after writing its JSON report to a file. With no + * `outputFile` it writes `.vitest/json/output.json` under its own root, which + * is the child's cwd (`pnpm --filter exec vitest` runs in the package), + * not the gate's (T13874). + */ +const REPORT_WRITTEN_LINE = /^JSON report written to (.+?)\s*$/gm; + +/** + * The report file the run's own output says it wrote (T13874), or null. Only + * the LAST announcement counts: vitest prints its line at the end, so an + * earlier one a test printed cannot stand in for it. The path is realpathed + * before the containment check, so a symlink inside the project that points + * outside it is refused. + * + * @param stdout - The run's captured stdout. + * @param projectRoot - The project the gate belongs to. + * @returns The announced report's real path inside the project, or null. + * @task T13874 + * @task T13883 + */ +async function announcedReportFile(stdout: string, projectRoot: string): Promise { + const announced = [...stdout.matchAll(REPORT_WRITTEN_LINE)].at(-1)?.[1]; + if (announced === undefined) return null; + try { + const [root, path] = await Promise.all([ + realpath(projectRoot), + realpath(resolve(projectRoot, announced)), + ]); + const rel = relative(root, path); + return rel.startsWith('..') || isAbsolute(rel) ? null : path; + } catch { + return null; // missing or unreadable: not a report of this run + } +} + /** * Report files a gate's JSON reporter may write instead of stdout (T13782): * every `--outputFile[.json]` the gate names, and {@link REDIRECTED_JSON_REPORT}. @@ -1971,6 +2007,10 @@ async function runTestGate( ); // T13752: a structured report decides the verdict; the text heuristic only // runs without one, since report JSON and test names are full of "failure". + // T13874: a path the run announced has no pre-spawn stamp; mtime and the + // report's startTime still have to postdate the spawn. + const announced = await announcedReportFile(captured.stdout, projectRoot); + if (announced !== null && !before.has(announced)) before.set(announced, null); const counts = structuredTestCounts(captured) ?? (await freshReportCounts(before, startedAt)); // T12308: the counts come from this run's own validated report; only their