Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .changeset/vitest5-announced-report.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---
id: vitest5-announced-report
tasks: [T13874]
kind: fix
summary: verify --run proves minCount from the JSON report vitest 5 writes under the package it ran in
---

With `--reporter=json` and no `outputFile`, vitest 5 writes its report to
`.vitest/json/output.json` under its own root, which is the child's cwd, and
prints `JSON report written to <path>`. Behind `pnpm --filter <pkg> exec` that
root is the package, so the gate looked at the project root, found nothing,
and failed every `minCount` gate with "emitted no machine-readable test report"
(VidaPeps T2018 to T2022 and T781).

The gate now also reads the report path the run announces on stdout, as long
as it is inside the project. That report still has to postdate the spawn:
both its mtime and its `startTime` must be at or after the run started.
83 changes: 82 additions & 1 deletion packages/core/src/tasks/__tests__/gate-runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
*/

import { createHash } from 'node:crypto';
import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import type {
Expand Down Expand Up @@ -961,6 +961,87 @@ describe('minCount reads a JSON report file this run wrote (T13782)', () => {
});
expect(result?.result).toBe('error');
});

/**
* vitest 5 behind `pnpm --filter <pkg> exec`: the report lands under the
* package (the child's cwd), and vitest names it on stdout (T13874).
*/
const fromPackage = (dir: string, startTime: string) =>
`process.chdir(${JSON.stringify(dir)});` +
writer(`require('node:path').resolve(${JSON.stringify(rel)})`, startTime) +
`console.log('JSON report written to ' + t);`;

it('reads the report vitest wrote under the package it ran in (T13874)', async () => {
const pkg = join(projectRoot, 'lib', 'db');
await mkdir(pkg, { recursive: true });
try {
const [result] = await runGates([gate(fromPackage(pkg, 'Date.now()'))], { projectRoot });
expect(result?.result, result?.errorMessage).toBe('pass');
} finally {
await rm(join(projectRoot, 'lib'), { recursive: true, force: true });
}
});

it('red: an announced report from before the run is refused (T13874)', async () => {
const pkg = join(projectRoot, 'lib', 'db');
await mkdir(pkg, { recursive: true });
try {
const [result] = await runGates([gate(fromPackage(pkg, 'Date.now() - 60000'))], {
projectRoot,
});
expect(result?.result).toBe('error');
} finally {
await rm(join(projectRoot, 'lib'), { recursive: true, force: true });
}
});

it('red: an in-project symlink to a fresh report outside is never read (T13883)', async () => {
const outside = await mkdtemp(join(tmpdir(), 'cleo-gate-outside-'));
const link = join(projectRoot, 'lib', 'db', rel);
await mkdir(dirname(link), { recursive: true });
await symlink(join(outside, 'output.json'), link);
try {
const source =
writer(JSON.stringify(join(outside, 'output.json')), 'Date.now()') +
`console.log('JSON report written to ' + ${JSON.stringify(link)});`;
const [result] = await runGates([gate(source)], { projectRoot });
expect(result?.result).toBe('error');
} finally {
await rm(join(projectRoot, 'lib'), { recursive: true, force: true });
await rm(outside, { recursive: true, force: true });
}
});

it('red: an earlier fabricated announcement loses to the last one (T13883)', async () => {
const pkg = join(projectRoot, 'lib', 'db');
await mkdir(pkg, { recursive: true });
const fake = join(projectRoot, 'fake-report.json');
// The fabricated report proves 3; vitest's own (announced last) proves 1.
const source =
writer(JSON.stringify(fake), 'Date.now()') +
`console.log('JSON report written to ' + ${JSON.stringify(fake)});` +
`const real=${JSON.stringify(join(pkg, rel))};` +
`fs.mkdirSync(p.dirname(real),{recursive:true});` +
`fs.writeFileSync(real, JSON.stringify(Object.assign(${report(1, 0)},{startTime:Date.now()})));` +
`console.log('JSON report written to ' + real);`;
try {
const [result] = await runGates([gate(source)], { projectRoot });
expect(result?.result).toBe('fail');
} finally {
await rm(join(projectRoot, 'lib'), { recursive: true, force: true });
await rm(fake, { force: true });
}
});

it('red: an announced report outside the project is never read (T13874)', async () => {
const outside = await mkdtemp(join(tmpdir(), 'cleo-gate-outside-'));
try {
const [result] = await runGates([gate(fromPackage(outside, 'Date.now()'))], { projectRoot });
expect(result?.result).toBe('error');
} finally {
await rm(outside, { recursive: true, force: true });
}
});
});

describe('gate runner immutable lifetime and bounded evidence', () => {
Expand Down
40 changes: 40 additions & 0 deletions packages/core/src/tasks/gate-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1813,6 +1813,42 @@ function structuredTestCounts(captured: ProcessCaptureResult): StructuredTestCou
*/
const REDIRECTED_JSON_REPORT = join('.vitest', 'json', 'output.json');

/**
* The line vitest 5 prints after writing its JSON report to a file. With no
* `outputFile` it writes `.vitest/json/output.json` under its own root, which
* is the child's cwd (`pnpm --filter <pkg> exec vitest` runs in the package),
* not the gate's (T13874).
*/
const REPORT_WRITTEN_LINE = /^JSON report written to (.+?)\s*$/gm;

/**
* The report file the run's own output says it wrote (T13874), or null. Only
* the LAST announcement counts: vitest prints its line at the end, so an
* earlier one a test printed cannot stand in for it. The path is realpathed
* before the containment check, so a symlink inside the project that points
* outside it is refused.
*
* @param stdout - The run's captured stdout.
* @param projectRoot - The project the gate belongs to.
* @returns The announced report's real path inside the project, or null.
* @task T13874
* @task T13883
*/
async function announcedReportFile(stdout: string, projectRoot: string): Promise<string | null> {
const announced = [...stdout.matchAll(REPORT_WRITTEN_LINE)].at(-1)?.[1];
if (announced === undefined) return null;
try {
const [root, path] = await Promise.all([
realpath(projectRoot),
realpath(resolve(projectRoot, announced)),
]);
const rel = relative(root, path);
return rel.startsWith('..') || isAbsolute(rel) ? null : path;
} catch {
return null; // missing or unreadable: not a report of this run
}
}

/**
* Report files a gate's JSON reporter may write instead of stdout (T13782):
* every `--outputFile[.json]` the gate names, and {@link REDIRECTED_JSON_REPORT}.
Expand Down Expand Up @@ -1971,6 +2007,10 @@ async function runTestGate(
);
// T13752: a structured report decides the verdict; the text heuristic only
// runs without one, since report JSON and test names are full of "failure".
// T13874: a path the run announced has no pre-spawn stamp; mtime and the
// report's startTime still have to postdate the spawn.
const announced = await announcedReportFile(captured.stdout, projectRoot);
if (announced !== null && !before.has(announced)) before.set(announced, null);
const counts = structuredTestCounts(captured) ?? (await freshReportCounts(before, startedAt));

// T12308: the counts come from this run's own validated report; only their
Expand Down
Loading