From cf7343c6054d8bc5491fc83cd022ef6f6f890776 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Wed, 26 Aug 2026 02:54:34 +0000 Subject: [PATCH] feat(eval): mark cells carried forward from a previous run Merging an existing manifest supports running one model at a time, and that stays. Doing it silently does not: the 17 and 24 August reports listed ten cells when five ran and the artifact said nothing. Carried cells are now named in the manifest, marked in the report and warned about at generation time. --- src/eval/live.ts | 38 +++++++++++++++++++++ src/eval/runner.ts | 13 +++++++- src/eval/types.ts | 6 ++++ tests/eval.test.ts | 82 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 1 deletion(-) diff --git a/src/eval/live.ts b/src/eval/live.ts index 6e1b516..340d811 100644 --- a/src/eval/live.ts +++ b/src/eval/live.ts @@ -154,7 +154,45 @@ export async function runLiveEval(opts: LiveEvalOptions): Promise { ); const mergedModels = [...keepFromPrevious, ...manifestModels]; + // A carried-forward cell is a real feature (run one model at a time, + // accumulate a run) and a real hazard: the report cannot otherwise be + // told apart from one where every cell ran. Mark them, and say so + // loudly at the point of generation, because the alternative is + // discovering it in a published table weeks later. + // A directory with no manifest entry at all is still aggregated by + // loadCells, which falls back to the directory name as the canonical + // id. Such a cell would otherwise appear in the results table with no + // roster entry, no marker and no warning, which is the same failure + // as a carried-forward cell but harder to notice. + const onDisk = (await readdir(samplesRoot, { withFileTypes: true }).catch(() => [])) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + const unmanifested = onDisk.filter( + (name) => !mergedModels.some((m) => m.sanitizedId === name), + ); + for (const name of unmanifested) { + mergedModels.push({ + spec: "(unknown)", + canonicalId: name, + sanitizedId: name, + provider: "found on disk, no manifest entry", + }); + } + + const carriedForward = [ + ...keepFromPrevious.map((m) => m.canonicalId), + ...unmanifested, + ]; + if (carriedForward.length > 0) { + console.warn( + `warning: ${carriedForward.length} cell(s) carried forward from a previous run in ` + + `${samplesRoot} and were not measured now: ${carriedForward.join(", ")}. ` + + `Use a clean output directory if this was not intended.`, + ); + } + const runManifest: RunManifest = { + carriedForward: carriedForward.length > 0 ? carriedForward : undefined, token: opts.token, briefPath: opts.briefPath, n: opts.n, diff --git a/src/eval/runner.ts b/src/eval/runner.ts index 711a39f..953fe65 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -218,8 +218,19 @@ export function formatEvalReport(r: EvalReport): string { lines.push(`- Samples per cell: **${r.runManifest.n}**`); lines.push(`- Max tokens: ${r.runManifest.maxTokens}`); lines.push(`- Models:`); + const carried = new Set(r.runManifest.carriedForward ?? []); for (const m of r.runManifest.models) { - lines.push(` - \`${m.canonicalId}\` (${m.provider}) · spec \`${m.spec}\``); + const mark = carried.has(m.canonicalId) ? " · **not run in this invocation**" : ""; + lines.push(` - \`${m.canonicalId}\` (${m.provider}) · spec \`${m.spec}\`${mark}`); + } + if (carried.size > 0) { + lines.push(""); + lines.push( + `> ${carried.size} cell(s) above were carried forward from an earlier run in the ` + + `output directory and were not measured by this invocation. Their figures come ` + + `from whatever samples were already on disk. Check them against the replay block ` + + `before citing them.`, + ); } lines.push(""); } diff --git a/src/eval/types.ts b/src/eval/types.ts index f2e2d5d..33031c7 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -53,6 +53,12 @@ export interface EvalReport { } export interface RunManifest { + /** + * Canonical ids present in the merged manifest that were not measured + * by the invocation that wrote this report. Present only when the + * output directory already held results. + */ + carriedForward?: string[]; token: string; briefPath: string; n: number; diff --git a/tests/eval.test.ts b/tests/eval.test.ts index 644dba1..aa349c9 100644 --- a/tests/eval.test.ts +++ b/tests/eval.test.ts @@ -126,3 +126,85 @@ describe("eval runner · honest accounting", () => { expect(text).toContain("Caveats"); }); }); + +describe("carried-forward cells", () => { + // eval-live merges its manifest with whatever it finds in the output + // directory, so one model can be run at a time and accumulated into a + // run. That is deliberate. What is not acceptable is doing it + // silently: the 17 and 24 August 2026 weekly reports listed ten cells + // when five ran, and nothing in the report said so. A carried-forward + // cell must be visible in the artifact that quotes its figures. + it("marks cells that were not measured by the invocation", () => { + const report = formatEvalReport({ + token: "swiss-editorial", + cells: [], + deltas: [], + perTellFrequency: {}, + caveats: [], + runManifest: { + carriedForward: ["claude-opus-4-7"], + token: "swiss-editorial", + briefPath: "briefs/landing.yml", + n: 30, + maxTokens: 12000, + runAt: "2026-08-26T00:00:00.000Z", + models: [ + { + spec: "cf:@cf/openai/gpt-oss-120b", + canonicalId: "@cf/openai/gpt-oss-120b", + sanitizedId: "_cf_openai_gpt-oss-120b", + provider: "cloudflare-workers-ai", + }, + { + spec: "claude-code:claude-opus-4-7", + canonicalId: "claude-opus-4-7", + sanitizedId: "claude-opus-4-7", + provider: "claude-code-cli", + }, + ], + }, + } as never); + + expect(report).toContain("**not run in this invocation**"); + expect(report).toContain("carried forward from an earlier run"); + // The cell that did run must not be marked. + const gptLine = report.split("\n").find((l) => l.includes("gpt-oss-120b")); + expect(gptLine).not.toContain("not run in this invocation"); + }); +}); + +describe("unmanifested sample directories", () => { + // loadCells aggregates every directory under the samples root and + // falls back to the directory name when there is no manifest entry. + // A stray directory therefore reached the results table with no roster + // entry, no marker and no warning: the same failure as a + // carried-forward cell, and harder to spot. This exercises + // runLiveEval, not the formatter, because the classification is what + // broke. + it("marks a model directory that has no manifest entry", async () => { + const { runLiveEval } = await import("../src/eval/live.js"); + const dir = await mkdtemp(join(tmpdir(), "ahd-stray-")); + const token = "swiss-editorial"; + for (const cond of ["raw", "compiled"]) { + await mkdir(join(dir, token, "ghost-model", cond), { recursive: true }); + await writeFile( + join(dir, token, "ghost-model", cond, "sample-001.html"), + "x

S

body

", + ); + } + + const report = await runLiveEval({ + tokensDir: resolve(__dirname, "..", "tokens"), + token, + briefPath: "briefs/landing.yml", + models: ["mock-swiss"], + n: 1, + outDir: dir, + } as never); + + expect(report.runManifest.carriedForward).toContain("ghost-model"); + const roster = report.runManifest.models.map((m) => m.canonicalId); + expect(roster).toContain("ghost-model"); + expect(formatEvalReport(report)).toContain("not run in this invocation"); + }); +});