From 5bc6f7a14a627f7ae8693a67ac1e4ce67a5920f4 Mon Sep 17 00:00:00 2001 From: Josh Purtell Date: Fri, 11 Sep 2026 15:31:05 -0400 Subject: [PATCH] Render shared benchmark observations in Workshop visuals --- visuals/chrome/BenchmarkStatus.tsx | 37 ++++++++ .../live.craftax.v1/shell.tsx | 3 + .../live.harbor_eval.v1/shell.tsx | 9 +- visuals/runtime/benchmarkObservation.ts | 84 +++++++++++++++++++ visuals/runtime/liveEvalReducer.ts | 7 ++ 5 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 visuals/chrome/BenchmarkStatus.tsx create mode 100644 visuals/runtime/benchmarkObservation.ts diff --git a/visuals/chrome/BenchmarkStatus.tsx b/visuals/chrome/BenchmarkStatus.tsx new file mode 100644 index 000000000..82d0fa3ab --- /dev/null +++ b/visuals/chrome/BenchmarkStatus.tsx @@ -0,0 +1,37 @@ +import type { BenchmarkObservation, PhaseClock } from "../runtime/benchmarkObservation.ts"; + +const number = (value: number | null): string => value === null ? "—" : value.toLocaleString(undefined, { maximumFractionDigits: 2 }); +const clock = (value: PhaseClock): string => `${number(value.elapsed_seconds)}/${number(value.limit_seconds)}s (${value.scope})`; + +/** Common producer values. Domain boards and scientific reports remain separate. */ +export function BenchmarkStatus({ rows }: { rows: BenchmarkObservation[] }) { + if (!rows.length) return null; + const work = rows.some((row) => row.work.elapsed_seconds !== null || row.work.limit_seconds !== null); + const verify = rows.some((row) => row.verify.elapsed_seconds !== null || row.verify.limit_seconds !== null); + const calls = rows.some((row) => row.calls !== null || row.call_limit !== null || row.call_limit_unbounded); + const steps = rows.some((row) => row.steps !== null || row.step_limit !== null); + return
+ + + + {work && }{verify && } + {calls && }{steps && } + + + {rows.map((row) => + + {work && }{verify && } + {calls && } + {steps && } + + + + )} +
Task / laneStateWorkVerifyCallsStepsScoreUsageStop / evidence
{row.lane}{row.phase}{clock(row.work)}{clock(row.verify)}{number(row.calls)}/{row.call_limit_unbounded ? "∞" : number(row.call_limit)} ({row.limit_scope}){number(row.steps)}/{number(row.step_limit)}{row.scientific_status === "ungraded" ? "UNGRADED" : number(row.score)}
{row.scientific_status}		 + {number(row.usage.total_tokens)} tokens · ${number(row.usage.cost_usd)}/${number(row.spend_limit_usd)} · {row.usage.status} +
{row.spend_enforcement} + 		{row.stop_reason || "—"}
Observed {row.observed_at} + {row.artifact_path &&
Evidence location{row.artifact_path}
} +
+
; +} diff --git a/visuals/families/first_class_example_containers/live.craftax.v1/shell.tsx b/visuals/families/first_class_example_containers/live.craftax.v1/shell.tsx index 1cdee41a2..d01c7955f 100644 --- a/visuals/families/first_class_example_containers/live.craftax.v1/shell.tsx +++ b/visuals/families/first_class_example_containers/live.craftax.v1/shell.tsx @@ -1,4 +1,6 @@ import { useEffect, useMemo, useState } from "react"; +import { BenchmarkStatus } from "../../../chrome/BenchmarkStatus.tsx"; +import { projectLiveEval } from "../../../runtime/liveEvalReducer.ts"; import { Identifier } from "../../../chrome/Identifier.tsx"; import { useLiveEvalStream } from "../../../chrome/useLiveEvalStream.ts"; import { counted, formatMissingNumber, formatMissingUsd } from "../../../runtime/liveStream.ts"; @@ -838,6 +840,7 @@ export function Shell(props: ShellProps) { data-active-surface={surface} data-journal-hydrating={journalHydrating ? "true" : "false"} > +

Live eval · Craftax{scope?.campaign_id ? <> · : null}

{props.title ?? "Policy through time"}

{props.lede ?

{props.lede}

: null}
{connectionState}
diff --git a/visuals/families/first_class_example_containers/live.harbor_eval.v1/shell.tsx b/visuals/families/first_class_example_containers/live.harbor_eval.v1/shell.tsx index d892d4c87..10ef07366 100644 --- a/visuals/families/first_class_example_containers/live.harbor_eval.v1/shell.tsx +++ b/visuals/families/first_class_example_containers/live.harbor_eval.v1/shell.tsx @@ -1,3 +1,5 @@ +import { BenchmarkStatus } from "../../../chrome/BenchmarkStatus.tsx"; +import { benchmarkSnapshotRows, latestBenchmarkObservations } from "../../../runtime/benchmarkObservation.ts"; /** * Harbor eval live viewer (A2 posture): trial → attempt evidence as it * streams, verifier truth (reward.txt fails closed; native and wrapped @@ -209,7 +211,11 @@ export function Shell(props: ShellProps) { () => harborEvalSnapshot(props.experiment ?? (props.data as { experiment?: unknown } | undefined)?.experiment), [props.experiment, props.data] ); - const settled = snapshot?.lifecycle === "terminal"; + const benchmarkRows = latestBenchmarkObservations([ + ...benchmarkSnapshotRows(props.experiment ?? props.data), ...projectLiveEval(visibleEvents).benchmarkObservations + ]); + const settled = snapshot?.lifecycle === "terminal" || + (benchmarkRows.length > 0 && benchmarkRows.every((row) => row.terminal)); const terminal = settled || ["completed", "finished", "failed", "cancelled"].includes(statusText.toLowerCase()); // A reopened terminal visual has no stream to rejoin: the producer sealed @@ -232,6 +238,7 @@ export function Shell(props: ShellProps) { testId="visual-live-harbor-eval" footer="live.harbor_eval.v1 · ATIF is a projection of this evidence, not the log" > + ; +const object = (v: unknown): Row | null => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Row : null; +const text = (v: unknown): string => typeof v === "string" ? v : ""; +const finite = (v: unknown): number | null => typeof v === "number" && Number.isFinite(v) ? v : null; +function clock(v: unknown): PhaseClock { + const row = object(v); + return { elapsed_seconds: finite(row?.elapsed_seconds), limit_seconds: finite(row?.limit_seconds), scope: text(row?.scope) || "phase" }; +} + +export function decodeBenchmarkObservation(value: unknown): BenchmarkObservation | null { + const row = object(value); + if (!row || !text(row.lane) || !text(row.observed_at) || !text(row.phase)) return null; + const usage = object(row.usage); + return { + lane: text(row.lane), phase: text(row.phase), observed_at: text(row.observed_at), + work: clock(row.work), verify: clock(row.verify), + usage: { + input_tokens: finite(usage?.input_tokens), output_tokens: finite(usage?.output_tokens), + total_tokens: finite(usage?.total_tokens), cost_usd: finite(usage?.cost_usd), + source: text(usage?.source), coverage: text(usage?.coverage), + status: usage?.status === "final" ? "final" : usage?.status === "provisional" ? "provisional" : "unknown" + }, + calls: finite(row.calls), call_limit: finite(row.call_limit), call_limit_unbounded: row.call_limit_unbounded === true, limit_scope: text(row.limit_scope), + spend_limit_usd: finite(row.spend_limit_usd), spend_enforcement: text(row.spend_enforcement), + steps: finite(row.steps), step_limit: finite(row.step_limit), score: finite(row.score), + scientific_status: text(row.scientific_status), stop_reason: text(row.stop_reason), + terminal: row.terminal === true, failed: row.failed === true, + artifact_path: text(row.artifact_path) || null + }; +} + +/** The existing evals snapshot carries the same already-projected lane rows. */ +export function benchmarkSnapshotRows(value: unknown): BenchmarkObservation[] { + const snapshot = object(value); + if (snapshot?.schema_version !== "evals.live-rollout.v1" || !Array.isArray(snapshot.lanes)) return []; + return snapshot.lanes.flatMap((lane) => { + const decoded = decodeBenchmarkObservation(object(lane)?.benchmark_observation); + return decoded ? [decoded] : []; + }); +} + +/** Select the newest producer snapshot per lane; do not sum cumulative counters. */ +export function latestBenchmarkObservations(rows: BenchmarkObservation[]): BenchmarkObservation[] { + const latest = new Map(); + for (const row of rows) { + const previous = latest.get(row.lane); + if (previous && (previous.observed_at > row.observed_at || (previous.terminal && !row.terminal))) continue; + latest.set(row.lane, row); + } + return [...latest.values()].sort((a, b) => a.lane.localeCompare(b.lane)); +} diff --git a/visuals/runtime/liveEvalReducer.ts b/visuals/runtime/liveEvalReducer.ts index a882a8cf6..4142c7960 100644 --- a/visuals/runtime/liveEvalReducer.ts +++ b/visuals/runtime/liveEvalReducer.ts @@ -3,9 +3,12 @@ * Missing reward / usage / score stay null. Control envelopes are not evidence. */ +import { decodeBenchmarkObservation, latestBenchmarkObservations, type BenchmarkObservation } from "./benchmarkObservation.ts"; + import { formatMissingNumber, isControlEnvelope, type LiveEnvelope } from "./liveStream.ts"; export type LiveEvalProjection = { + benchmarkObservations: BenchmarkObservation[]; events: LiveEnvelope[]; kinds: string[]; has_live_frames: boolean; @@ -84,6 +87,10 @@ export function projectLiveEval( } : null; const projection: LiveEvalProjection = { + benchmarkObservations: latestBenchmarkObservations(rows.flatMap((event) => { + const observation = decodeBenchmarkObservation(event.payload?.benchmark_observation); + return observation ? [observation] : []; + })), events: rows, kinds, has_live_frames,