diff --git a/src/eval/live.ts b/src/eval/live.ts index 6e1b516..340d811 100644 --- a/src/eval/live.ts +++ b/src/eval/live.ts @@ -154,7 +154,45 @@ export async function runLiveEval(opts: LiveEvalOptions): Promise { ); const mergedModels = [...keepFromPrevious, ...manifestModels]; + // A carried-forward cell is a real feature (run one model at a time, + // accumulate a run) and a real hazard: the report cannot otherwise be + // told apart from one where every cell ran. Mark them, and say so + // loudly at the point of generation, because the alternative is + // discovering it in a published table weeks later. + // A directory with no manifest entry at all is still aggregated by + // loadCells, which falls back to the directory name as the canonical + // id. Such a cell would otherwise appear in the results table with no + // roster entry, no marker and no warning, which is the same failure + // as a carried-forward cell but harder to notice. + const onDisk = (await readdir(samplesRoot, { withFileTypes: true }).catch(() => [])) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + const unmanifested = onDisk.filter( + (name) => !mergedModels.some((m) => m.sanitizedId === name), + ); + for (const name of unmanifested) { + mergedModels.push({ + spec: "(unknown)", + canonicalId: name, + sanitizedId: name, + provider: "found on disk, no manifest entry", + }); + } + + const carriedForward = [ + ...keepFromPrevious.map((m) => m.canonicalId), + ...unmanifested, + ]; + if (carriedForward.length > 0) { + console.warn( + `warning: ${carriedForward.length} cell(s) carried forward from a previous run in ` + + `${samplesRoot} and were not measured now: ${carriedForward.join(", ")}. ` + + `Use a clean output directory if this was not intended.`, + ); + } + const runManifest: RunManifest = { + carriedForward: carriedForward.length > 0 ? carriedForward : undefined, token: opts.token, briefPath: opts.briefPath, n: opts.n, diff --git a/src/eval/runner.ts b/src/eval/runner.ts index 711a39f..953fe65 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -218,8 +218,19 @@ export function formatEvalReport(r: EvalReport): string { lines.push(`- Samples per cell: **${r.runManifest.n}**`); lines.push(`- Max tokens: ${r.runManifest.maxTokens}`); lines.push(`- Models:`); + const carried = new Set(r.runManifest.carriedForward ?? []); for (const m of r.runManifest.models) { - lines.push(` - \`${m.canonicalId}\` (${m.provider}) · spec \`${m.spec}\``); + const mark = carried.has(m.canonicalId) ? " · **not run in this invocation**" : ""; + lines.push(` - \`${m.canonicalId}\` (${m.provider}) · spec \`${m.spec}\`${mark}`); + } + if (carried.size > 0) { + lines.push(""); + lines.push( + `> ${carried.size} cell(s) above were carried forward from an earlier run in the ` + + `output directory and were not measured by this invocation. Their figures come ` + + `from whatever samples were already on disk. Check them against the replay block ` + + `before citing them.`, + ); } lines.push(""); } diff --git a/src/eval/types.ts b/src/eval/types.ts index f2e2d5d..33031c7 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -53,6 +53,12 @@ export interface EvalReport { } export interface RunManifest { + /** + * Canonical ids present in the merged manifest that were not measured + * by the invocation that wrote this report. Present only when the + * output directory already held results. + */ + carriedForward?: string[]; token: string; briefPath: string; n: number; diff --git a/tests/eval.test.ts b/tests/eval.test.ts index 644dba1..aa349c9 100644 --- a/tests/eval.test.ts +++ b/tests/eval.test.ts @@ -126,3 +126,85 @@ describe("eval runner · honest accounting", () => { expect(text).toContain("Caveats"); }); }); + +describe("carried-forward cells", () => { + // eval-live merges its manifest with whatever it finds in the output + // directory, so one model can be run at a time and accumulated into a + // run. That is deliberate. What is not acceptable is doing it + // silently: the 17 and 24 August 2026 weekly reports listed ten cells + // when five ran, and nothing in the report said so. A carried-forward + // cell must be visible in the artifact that quotes its figures. + it("marks cells that were not measured by the invocation", () => { + const report = formatEvalReport({ + token: "swiss-editorial", + cells: [], + deltas: [], + perTellFrequency: {}, + caveats: [], + runManifest: { + carriedForward: ["claude-opus-4-7"], + token: "swiss-editorial", + briefPath: "briefs/landing.yml", + n: 30, + maxTokens: 12000, + runAt: "2026-08-26T00:00:00.000Z", + models: [ + { + spec: "cf:@cf/openai/gpt-oss-120b", + canonicalId: "@cf/openai/gpt-oss-120b", + sanitizedId: "_cf_openai_gpt-oss-120b", + provider: "cloudflare-workers-ai", + }, + { + spec: "claude-code:claude-opus-4-7", + canonicalId: "claude-opus-4-7", + sanitizedId: "claude-opus-4-7", + provider: "claude-code-cli", + }, + ], + }, + } as never); + + expect(report).toContain("**not run in this invocation**"); + expect(report).toContain("carried forward from an earlier run"); + // The cell that did run must not be marked. + const gptLine = report.split("\n").find((l) => l.includes("gpt-oss-120b")); + expect(gptLine).not.toContain("not run in this invocation"); + }); +}); + +describe("unmanifested sample directories", () => { + // loadCells aggregates every directory under the samples root and + // falls back to the directory name when there is no manifest entry. + // A stray directory therefore reached the results table with no roster + // entry, no marker and no warning: the same failure as a + // carried-forward cell, and harder to spot. This exercises + // runLiveEval, not the formatter, because the classification is what + // broke. + it("marks a model directory that has no manifest entry", async () => { + const { runLiveEval } = await import("../src/eval/live.js"); + const dir = await mkdtemp(join(tmpdir(), "ahd-stray-")); + const token = "swiss-editorial"; + for (const cond of ["raw", "compiled"]) { + await mkdir(join(dir, token, "ghost-model", cond), { recursive: true }); + await writeFile( + join(dir, token, "ghost-model", cond, "sample-001.html"), + "x

S

body

", + ); + } + + const report = await runLiveEval({ + tokensDir: resolve(__dirname, "..", "tokens"), + token, + briefPath: "briefs/landing.yml", + models: ["mock-swiss"], + n: 1, + outDir: dir, + } as never); + + expect(report.runManifest.carriedForward).toContain("ghost-model"); + const roster = report.runManifest.models.map((m) => m.canonicalId); + expect(roster).toContain("ghost-model"); + expect(formatEvalReport(report)).toContain("not run in this invocation"); + }); +});