Consolidate Workshop research observations for Synth Index

visuals/chrome/BenchmarkStatus.tsx

@@ -0,0 +1,37 @@
+import type { BenchmarkObservation, PhaseClock } from "../runtime/benchmarkObservation.ts";
+
+const number = (value: number | null): string => value === null ? "—" : value.toLocaleString(undefined, { maximumFractionDigits: 2 });
+const clock = (value: PhaseClock): string => `${number(value.elapsed_seconds)}/${number(value.limit_seconds)}s (${value.scope})`;
+
+/** Common producer values. Domain boards and scientific reports remain separate. */
+export function BenchmarkStatus({ rows }: { rows: BenchmarkObservation[] }) {
+  if (!rows.length) return null;
+  const work = rows.some((row) => row.work.elapsed_seconds !== null || row.work.limit_seconds !== null);
+  const verify = rows.some((row) => row.verify.elapsed_seconds !== null || row.verify.limit_seconds !== null);
+  const calls = rows.some((row) => row.calls !== null || row.call_limit !== null || row.call_limit_unbounded);
+  const steps = rows.some((row) => row.steps !== null || row.step_limit !== null);
+  return <section aria-label="Benchmark status" style={{ overflowX: "auto" }}>
+    <table className="sv-table">
+      <thead><tr>
+        <th scope="col">Task / lane</th><th scope="col">State</th>
+        {work && <th scope="col">Work</th>}{verify && <th scope="col">Verify</th>}
+        {calls && <th scope="col">Calls</th>}{steps && <th scope="col">Steps</th>}
+        <th scope="col">Score</th><th scope="col">Usage</th><th scope="col">Stop / evidence</th>
+      </tr></thead>
+      <tbody>{rows.map((row) => <tr key={row.lane}>
+        <th scope="row">{row.lane}</th><td>{row.phase}</td>
+        {work && <td>{clock(row.work)}</td>}{verify && <td>{clock(row.verify)}</td>}
+        {calls && <td>{number(row.calls)}/{row.call_limit_unbounded ? "∞" : number(row.call_limit)} ({row.limit_scope})</td>}
+        {steps && <td>{number(row.steps)}/{number(row.step_limit)}</td>}
+        <td>{row.scientific_status === "ungraded" ? "UNGRADED" : number(row.score)}<br /><small>{row.scientific_status}</small></td>
+        <td title={`${row.usage.source} · ${row.usage.coverage}`}>
+          {number(row.usage.total_tokens)} tokens · ${number(row.usage.cost_usd)}/${number(row.spend_limit_usd)} · {row.usage.status}
+          <br /><small>{row.spend_enforcement}</small>
+        </td>
+        <td>{row.stop_reason || "—"}<br /><small>Observed {row.observed_at}</small>
+          {row.artifact_path && <details><summary>Evidence location</summary><code>{row.artifact_path}</code></details>}
+        </td>
+      </tr>)}</tbody>
+    </table>
+  </section>;
+}

visuals/families/first_class_example_containers/live.craftax.v1/shell.tsx

@@ -1,4 +1,6 @@
 import { useEffect, useMemo, useState } from "react";
+import { BenchmarkStatus } from "../../../chrome/BenchmarkStatus.tsx";
+import { projectLiveEval } from "../../../runtime/liveEvalReducer.ts";
 import { Identifier } from "../../../chrome/Identifier.tsx";
 import { useLiveEvalStream } from "../../../chrome/useLiveEvalStream.ts";
 import { counted, formatMissingNumber, formatMissingUsd } from "../../../runtime/liveStream.ts";
@@ -838,6 +840,7 @@ export function Shell(props: ShellProps) {
 		data-active-surface={surface}
 		data-journal-hydrating={journalHydrating ? "true" : "false"}
 	>
+      <BenchmarkStatus rows={projectLiveEval(events).benchmarkObservations} />
       <header className="cv-topbar">
         <div><p className="cv-eyebrow">Live eval · Craftax{scope?.campaign_id ? <> · <Identifier value={scope.campaign_id} label="campaign" max={18} copy={false} /></> : null}</p><h2>{props.title ?? "Policy through time"}</h2>{props.lede ? <p className="cv-lede">{props.lede}</p> : null}</div>
         <div className="cv-connection" role="status"><span className={visualLive ? "live" : !lifecycleFailed && ready ? "ready" : lifecycleFailed ? "failed" : ""} />{connectionState}</div>

visuals/families/first_class_example_containers/live.harbor_eval.v1/shell.tsx

@@ -1,3 +1,5 @@
+import { BenchmarkStatus } from "../../../chrome/BenchmarkStatus.tsx";
+import { benchmarkSnapshotRows, latestBenchmarkObservations } from "../../../runtime/benchmarkObservation.ts";
 /**
  * Harbor eval live viewer (A2 posture): trial → attempt evidence as it
  * streams, verifier truth (reward.txt fails closed; native and wrapped
@@ -209,7 +211,11 @@ export function Shell(props: ShellProps) {
     () => harborEvalSnapshot(props.experiment ?? (props.data as { experiment?: unknown } | undefined)?.experiment),
     [props.experiment, props.data]
   );
-  const settled = snapshot?.lifecycle === "terminal";
+  const benchmarkRows = latestBenchmarkObservations([
+    ...benchmarkSnapshotRows(props.experiment ?? props.data), ...projectLiveEval(visibleEvents).benchmarkObservations
+  ]);
+  const settled = snapshot?.lifecycle === "terminal" ||
+    (benchmarkRows.length > 0 && benchmarkRows.every((row) => row.terminal));
   const terminal =
     settled || ["completed", "finished", "failed", "cancelled"].includes(statusText.toLowerCase());
   // A reopened terminal visual has no stream to rejoin: the producer sealed
@@ -232,6 +238,7 @@ export function Shell(props: ShellProps) {
       testId="visual-live-harbor-eval"
       footer="live.harbor_eval.v1 · ATIF is a projection of this evidence, not the log"
     >
+      <BenchmarkStatus rows={benchmarkRows} />
       <MetricStrip
         metrics={
           restored && snapshot

visuals/runtime/benchmarkObservation.ts

@@ -0,0 +1,84 @@
+/** Decode producer-projected eval fields. No local billing or lifecycle inference. */
+export type PhaseClock = { elapsed_seconds: number | null; limit_seconds: number | null; scope: string };
+export type BenchmarkObservation = {
+  lane: string;
+  phase: string;
+  observed_at: string;
+  work: PhaseClock;
+  verify: PhaseClock;
+  usage: {
+    input_tokens: number | null;
+    output_tokens: number | null;
+    total_tokens: number | null;
+    cost_usd: number | null;
+    source: string;
+    coverage: string;
+    status: "unknown" | "provisional" | "final";
+  };
+  calls: number | null;
+  call_limit: number | null;
+  call_limit_unbounded: boolean;
+  limit_scope: string;
+  spend_limit_usd: number | null;
+  spend_enforcement: string;
+  steps: number | null;
+  step_limit: number | null;
+  score: number | null;
+  scientific_status: string;
+  stop_reason: string;
+  terminal: boolean;
+  failed: boolean;
+  artifact_path: string | null;
+};
+
+type Row = Record<string, unknown>;
+const object = (v: unknown): Row | null => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Row : null;
+const text = (v: unknown): string => typeof v === "string" ? v : "";
+const finite = (v: unknown): number | null => typeof v === "number" && Number.isFinite(v) ? v : null;
+function clock(v: unknown): PhaseClock {
+  const row = object(v);
+  return { elapsed_seconds: finite(row?.elapsed_seconds), limit_seconds: finite(row?.limit_seconds), scope: text(row?.scope) || "phase" };
+}
+
+export function decodeBenchmarkObservation(value: unknown): BenchmarkObservation | null {
+  const row = object(value);
+  if (!row || !text(row.lane) || !text(row.observed_at) || !text(row.phase)) return null;
+  const usage = object(row.usage);
+  return {
+    lane: text(row.lane), phase: text(row.phase), observed_at: text(row.observed_at),
+    work: clock(row.work), verify: clock(row.verify),
+    usage: {
+      input_tokens: finite(usage?.input_tokens), output_tokens: finite(usage?.output_tokens),
+      total_tokens: finite(usage?.total_tokens), cost_usd: finite(usage?.cost_usd),
+      source: text(usage?.source), coverage: text(usage?.coverage),
+      status: usage?.status === "final" ? "final" : usage?.status === "provisional" ? "provisional" : "unknown"
+    },
+    calls: finite(row.calls), call_limit: finite(row.call_limit), call_limit_unbounded: row.call_limit_unbounded === true, limit_scope: text(row.limit_scope),
+    spend_limit_usd: finite(row.spend_limit_usd), spend_enforcement: text(row.spend_enforcement),
+    steps: finite(row.steps), step_limit: finite(row.step_limit), score: finite(row.score),
+    scientific_status: text(row.scientific_status), stop_reason: text(row.stop_reason),
+    terminal: row.terminal === true, failed: row.failed === true,
+    artifact_path: text(row.artifact_path) || null
+  };
+}
+
+/** The existing evals snapshot carries the same already-projected lane rows. */
+export function benchmarkSnapshotRows(value: unknown): BenchmarkObservation[] {
+  const snapshot = object(value);
+  if (snapshot?.schema_version !== "evals.live-rollout.v1" || !Array.isArray(snapshot.lanes)) return [];
+  return snapshot.lanes.flatMap((lane) => {
+    const decoded = decodeBenchmarkObservation(object(lane)?.benchmark_observation);
+    return decoded ? [decoded] : [];
+  });
+}
+
+/** Select the newest producer snapshot per lane; do not sum cumulative counters. */
+export function latestBenchmarkObservations(rows: BenchmarkObservation[]): BenchmarkObservation[] {
+  const latest = new Map<string, BenchmarkObservation>();
+  for (const row of rows) {
+    const previous = latest.get(row.lane);
+    if (previous && (previous.observed_at > row.observed_at || (previous.terminal && !row.terminal))) continue;
+    latest.set(row.lane, row);
+  }
+  return [...latest.values()].sort((a, b) => a.lane.localeCompare(b.lane));
+}

visuals/runtime/liveEvalReducer.ts

@@ -3,9 +3,12 @@
  * Missing reward / usage / score stay null. Control envelopes are not evidence.
  */
 
+import { decodeBenchmarkObservation, latestBenchmarkObservations, type BenchmarkObservation } from "./benchmarkObservation.ts";
+
 import { formatMissingNumber, isControlEnvelope, type LiveEnvelope } from "./liveStream.ts";
 
 export type LiveEvalProjection = {
+  benchmarkObservations: BenchmarkObservation[];
   events: LiveEnvelope[];
   kinds: string[];
   has_live_frames: boolean;
@@ -84,6 +87,10 @@ export function projectLiveEval(
       }
     : null;
   const projection: LiveEvalProjection = {
+    benchmarkObservations: latestBenchmarkObservations(rows.flatMap((event) => {
+      const observation = decodeBenchmarkObservation(event.payload?.benchmark_observation);
+      return observation ? [observation] : [];
+    })),
     events: rows,
     kinds,
     has_live_frames,