feat(eval): mark cells carried forward from a previous run

src/eval/live.ts

@@ -154,7 +154,45 @@ export async function runLiveEval(opts: LiveEvalOptions): Promise<EvalReport> {
   );
   const mergedModels = [...keepFromPrevious, ...manifestModels];
 
+  // A carried-forward cell is a real feature (run one model at a time,
+  // accumulate a run) and a real hazard: the report cannot otherwise be
+  // told apart from one where every cell ran. Mark them, and say so
+  // loudly at the point of generation, because the alternative is
+  // discovering it in a published table weeks later.
+  // A directory with no manifest entry at all is still aggregated by
+  // loadCells, which falls back to the directory name as the canonical
+  // id. Such a cell would otherwise appear in the results table with no
+  // roster entry, no marker and no warning, which is the same failure
+  // as a carried-forward cell but harder to notice.
+  const onDisk = (await readdir(samplesRoot, { withFileTypes: true }).catch(() => []))
+    .filter((d) => d.isDirectory())
+    .map((d) => d.name);
+  const unmanifested = onDisk.filter(
+    (name) => !mergedModels.some((m) => m.sanitizedId === name),
+  );
+  for (const name of unmanifested) {
+    mergedModels.push({
+      spec: "(unknown)",
+      canonicalId: name,
+      sanitizedId: name,
+      provider: "found on disk, no manifest entry",
+    });
+  }
+
+  const carriedForward = [
+    ...keepFromPrevious.map((m) => m.canonicalId),
+    ...unmanifested,
+  ];
+  if (carriedForward.length > 0) {
+    console.warn(
+      `warning: ${carriedForward.length} cell(s) carried forward from a previous run in ` +
+        `${samplesRoot} and were not measured now: ${carriedForward.join(", ")}. ` +
+        `Use a clean output directory if this was not intended.`,
+    );
+  }
+
   const runManifest: RunManifest = {
+    carriedForward: carriedForward.length > 0 ? carriedForward : undefined,
     token: opts.token,
     briefPath: opts.briefPath,
     n: opts.n,

src/eval/runner.ts

@@ -218,8 +218,19 @@ export function formatEvalReport(r: EvalReport): string {
     lines.push(`- Samples per cell: **${r.runManifest.n}**`);
     lines.push(`- Max tokens: ${r.runManifest.maxTokens}`);
     lines.push(`- Models:`);
+    const carried = new Set(r.runManifest.carriedForward ?? []);
     for (const m of r.runManifest.models) {
-      lines.push(`  - \`${m.canonicalId}\` (${m.provider}) · spec \`${m.spec}\``);
+      const mark = carried.has(m.canonicalId) ? " · **not run in this invocation**" : "";
+      lines.push(`  - \`${m.canonicalId}\` (${m.provider}) · spec \`${m.spec}\`${mark}`);
+    }
+    if (carried.size > 0) {
+      lines.push("");
+      lines.push(
+        `> ${carried.size} cell(s) above were carried forward from an earlier run in the ` +
+          `output directory and were not measured by this invocation. Their figures come ` +
+          `from whatever samples were already on disk. Check them against the replay block ` +
+          `before citing them.`,
+      );
     }
     lines.push("");
   }

src/eval/types.ts

@@ -53,6 +53,12 @@ export interface EvalReport {
 }
 
 export interface RunManifest {
+  /**
+   * Canonical ids present in the merged manifest that were not measured
+   * by the invocation that wrote this report. Present only when the
+   * output directory already held results.
+   */
+  carriedForward?: string[];
   token: string;
   briefPath: string;
   n: number;

tests/eval.test.ts

@@ -126,3 +126,85 @@ describe("eval runner · honest accounting", () => {
     expect(text).toContain("Caveats");
   });
 });
+
+describe("carried-forward cells", () => {
+  // eval-live merges its manifest with whatever it finds in the output
+  // directory, so one model can be run at a time and accumulated into a
+  // run. That is deliberate. What is not acceptable is doing it
+  // silently: the 17 and 24 August 2026 weekly reports listed ten cells
+  // when five ran, and nothing in the report said so. A carried-forward
+  // cell must be visible in the artifact that quotes its figures.
+  it("marks cells that were not measured by the invocation", () => {
+    const report = formatEvalReport({
+      token: "swiss-editorial",
+      cells: [],
+      deltas: [],
+      perTellFrequency: {},
+      caveats: [],
+      runManifest: {
+        carriedForward: ["claude-opus-4-7"],
+        token: "swiss-editorial",
+        briefPath: "briefs/landing.yml",
+        n: 30,
+        maxTokens: 12000,
+        runAt: "2026-08-26T00:00:00.000Z",
+        models: [
+          {
+            spec: "cf:@cf/openai/gpt-oss-120b",
+            canonicalId: "@cf/openai/gpt-oss-120b",
+            sanitizedId: "_cf_openai_gpt-oss-120b",
+            provider: "cloudflare-workers-ai",
+          },
+          {
+            spec: "claude-code:claude-opus-4-7",
+            canonicalId: "claude-opus-4-7",
+            sanitizedId: "claude-opus-4-7",
+            provider: "claude-code-cli",
+          },
+        ],
+      },
+    } as never);
+
+    expect(report).toContain("**not run in this invocation**");
+    expect(report).toContain("carried forward from an earlier run");
+    // The cell that did run must not be marked.
+    const gptLine = report.split("\n").find((l) => l.includes("gpt-oss-120b"));
+    expect(gptLine).not.toContain("not run in this invocation");
+  });
+});
+
+describe("unmanifested sample directories", () => {
+  // loadCells aggregates every directory under the samples root and
+  // falls back to the directory name when there is no manifest entry.
+  // A stray directory therefore reached the results table with no roster
+  // entry, no marker and no warning: the same failure as a
+  // carried-forward cell, and harder to spot. This exercises
+  // runLiveEval, not the formatter, because the classification is what
+  // broke.
+  it("marks a model directory that has no manifest entry", async () => {
+    const { runLiveEval } = await import("../src/eval/live.js");
+    const dir = await mkdtemp(join(tmpdir(), "ahd-stray-"));
+    const token = "swiss-editorial";
+    for (const cond of ["raw", "compiled"]) {
+      await mkdir(join(dir, token, "ghost-model", cond), { recursive: true });
+      await writeFile(
+        join(dir, token, "ghost-model", cond, "sample-001.html"),
+        "<!doctype html><html><head><title>x</title></head><body><main><h1>S</h1><p>body</p></main></body></html>",
+      );
+    }
+
+    const report = await runLiveEval({
+      tokensDir: resolve(__dirname, "..", "tokens"),
+      token,
+      briefPath: "briefs/landing.yml",
+      models: ["mock-swiss"],
+      n: 1,
+      outDir: dir,
+    } as never);
+
+    expect(report.runManifest.carriedForward).toContain("ghost-model");
+    const roster = report.runManifest.models.map((m) => m.canonicalId);
+    expect(roster).toContain("ghost-model");
+    expect(formatEvalReport(report)).toContain("not run in this invocation");
+  });
+});