Let Core own the runtime history window

apps/web/e2e/unified-settings.spec.ts

@@ -105,7 +105,7 @@ test("settings content width does not constrain reset and node dialog backdrops"
     await page.getByRole("button", { name: action, exact: true }).first().click();
     const dialog = page.getByRole("dialog");
     await expect(dialog).toBeVisible();
-    const backdrop = page.locator(".modal-backdrop");
+    const backdrop = page.locator(".modal-backdrop:not(.closing)");
     const bounds = await backdrop.boundingBox();
     expect(bounds?.x).toBe(0);
     expect(bounds?.y).toBe(0);

apps/web/src/features/dashboard/runtime-history.test.ts

@@ -277,20 +277,30 @@ describe("Runtime Durable Dashboard history", () => {
     expect(samples[1]).toMatchObject({ inputTokensPerMinute: null, outputTokensPerMinute: 20 });
   });
 
-  it("loads each hosted Session's history through its project with a bounded common range", async () => {
-    const calls: Array<{ sessionID: string; start: number; end: number; maxPoints?: number }> = [];
+  it("keeps only the buckets every Session's history covers when Core's clock ended them a bucket apart", () => {
+    const otherID = "44444444-4444-4444-8444-444444444444";
+    const times = new Set(["start", "end", "first_observed_at", "last_observed_at", "sampled_at"]);
+    const later: RuntimeHistory = JSON.parse(JSON.stringify(history()), (key, value) => (times.has(key) && typeof value === "number" ? value + 30 : value));
+    later.session_id = otherID;
+    later.series[0]!.allocation_id = "55555555-5555-4555-8555-555555555555";
+    const samples = runtimeDurableTrendSamples([session, { ...session, id: otherID } as AgentSession], [history(), later]);
+    expect(samples.map((sample) => [sample.sampledAt, sample.activeSandboxCount])).toEqual([[160_000, 2]]);
+  });
+
+  it("loads each hosted Session's history through its project for the named range", async () => {
+    const calls: Array<{ sessionID: string; range: string }> = [];
     const reader: RuntimeHistoryReader = {
-      retrieveRuntimeHistory: async (id, query) => {
-        calls.push({ sessionID: id, start: query.start, end: query.end, maxPoints: query.maxPoints });
-        return history({ requested_range: { start: query.start, end: query.end } });
+      retrieveRuntimeHistory: async (id, range) => {
+        calls.push({ sessionID: id, range });
+        return history();
       },
     };
     const readers: string[] = [];
     const snapshot: RuntimeDashboardSnapshot = { sessions: [session], observations: [observation], owners: new Map([[sessionID, "proj_a"]]), loadedAt: 1 };
-    const result = await loadRuntimeDurableSnapshot((id) => { readers.push(snapshot.owners?.get(id) ?? ""); return reader; }, snapshot, 60 * 60_000, undefined, () => 7_200_000);
+    const result = await loadRuntimeDurableSnapshot((id) => { readers.push(snapshot.owners?.get(id) ?? ""); return reader; }, snapshot, "1h");
     expect(readers).toEqual(["proj_a"]);
-    expect(calls).toEqual([{ sessionID, start: 3_600, end: 7_200, maxPoints: 120 }]);
-    expect(result).toMatchObject({ rangeStart: 3_600_000, rangeEnd: 7_200_000, targetCount: 1, sampleCount: 2 });
+    expect(calls).toEqual([{ sessionID, range: "1h" }]);
+    expect(result).toMatchObject({ targetCount: 1, sampleCount: 2 });
   });
 
   it("queries each managed Session only once when observations contain duplicates", async () => {
@@ -306,7 +316,7 @@ describe("Runtime Durable Dashboard history", () => {
       observations: [observation, { ...observation }],
       loadedAt: 1,
     };
-    await loadRuntimeDurableSnapshot(() => reader, snapshot, 60 * 60_000);
+    await loadRuntimeDurableSnapshot(() => reader, snapshot, "1h");
     expect(queried).toEqual([sessionID]);
   });
 
@@ -315,7 +325,7 @@ describe("Runtime Durable Dashboard history", () => {
       retrieveRuntimeHistory: async () => { throw new AgentCoreError("No history.", 404); },
     };
     const snapshot: RuntimeDashboardSnapshot = { sessions: [session], observations: [observation], loadedAt: 1 };
-    await expect(loadRuntimeDurableSnapshot(() => reader, snapshot, 60 * 60_000)).resolves.toBeNull();
+    await expect(loadRuntimeDurableSnapshot(() => reader, snapshot, "1h")).resolves.toBeNull();
   });
 
   it("leaves out a Session without history and keeps the others", async () => {
@@ -332,21 +342,21 @@ describe("Runtime Durable Dashboard history", () => {
       observations: [observation, { ...observation, id: otherID, session_id: otherID }],
       loadedAt: 1,
     };
-    await expect(loadRuntimeDurableSnapshot(() => reader, snapshot, 60 * 60_000)).resolves.toMatchObject({ targetCount: 1 });
+    await expect(loadRuntimeDurableSnapshot(() => reader, snapshot, "1h")).resolves.toMatchObject({ targetCount: 1 });
   });
 
   it("propagates other history failures", async () => {
     const reader: RuntimeHistoryReader = {
       retrieveRuntimeHistory: async () => { throw new AgentCoreError("Unavailable.", 503); },
     };
     const snapshot: RuntimeDashboardSnapshot = { sessions: [session], observations: [observation], loadedAt: 1 };
-    await expect(loadRuntimeDurableSnapshot(() => reader, snapshot, 60 * 60_000)).rejects.toMatchObject({ status: 503 });
+    await expect(loadRuntimeDurableSnapshot(() => reader, snapshot, "1h")).rejects.toMatchObject({ status: 503 });
   });
 
   it("fails closed instead of publishing a partial oversized Dashboard", async () => {
     const reader: RuntimeHistoryReader = { retrieveRuntimeHistory: async () => history() };
     const snapshot: RuntimeDashboardSnapshot = { sessions: [session], observations: [observation], loadedAt: 1 };
-    await expect(loadRuntimeDurableSnapshot(() => reader, snapshot, 60 * 60_000, undefined, Date.now, 0))
+    await expect(loadRuntimeDurableSnapshot(() => reader, snapshot, "1h", undefined, 0))
       .rejects.toBeInstanceOf(RuntimeDurableHistoryIncompleteError);
   });
 });

apps/web/src/features/dashboard/runtime-history.ts

@@ -1,33 +1,27 @@
-import { AgentCoreError, type AgentSession, type RuntimeHistory } from "@oac/agents-client";
+import { AgentCoreError, type AgentSession, type RuntimeHistory, type RuntimeHistoryRange } from "@oac/agents-client";
 
 import type { ProjectClient } from "../../lib/projects";
 import type { RuntimeDashboardSnapshot } from "./runtime-snapshot";
 import { deriveTokenThroughput, type RuntimeTrendSample, type RuntimeTrendTarget } from "./runtime-trends";
 
 export const RUNTIME_DURABLE_TARGET_LIMIT = 24;
-export const RUNTIME_DURABLE_MAX_POINTS = 120;
-export const RUNTIME_DURABLE_RANGES = [
-  { label: "1h", milliseconds: 60 * 60 * 1_000 },
-  { label: "6h", milliseconds: 6 * 60 * 60 * 1_000 },
-  { label: "24h", milliseconds: 24 * 60 * 60 * 1_000 },
-] as const;
+/** Ranges Core ends on its own clock, so the browser's clock never enters a read. */
+export const RUNTIME_DURABLE_RANGES = ["1h", "6h", "24h"] as const satisfies readonly RuntimeHistoryRange[];
 
-export type RuntimeDurableRange = typeof RUNTIME_DURABLE_RANGES[number]["milliseconds"];
+export type RuntimeDurableRange = typeof RUNTIME_DURABLE_RANGES[number];
 
 export interface RuntimeDurableSnapshot {
   samples: RuntimeTrendSample[];
-  rangeStart: number;
-  rangeEnd: number;
   resolutionSeconds: number;
   targetCount: number;
   sampleCount: number;
   expectedSampleCount: number;
-  loadedAt: number;
 }
 
+/** Web's own read bound: more Sessions than `limit`, or, when it is null, histories of different bucket widths. */
 export class RuntimeDurableHistoryIncompleteError extends Error {
-  constructor(message: string) {
-    super(message);
+  constructor(readonly limit: number | null) {
+    super(limit === null ? "Durable Runtime histories returned inconsistent resolutions." : `Durable Runtime history exceeds the ${limit} target Web query budget.`);
     this.name = "RuntimeDurableHistoryIncompleteError";
   }
 }
@@ -138,7 +132,11 @@ export function runtimeDurableTrendSamples(
     }
   }
 
-  const samples = [...buckets.values()].sort((left, right) => left.sampledAt - right.sampledAt).map((value) => {
+  // Core ends each range at its last complete bucket, so reads on either side of
+  // a bucket boundary differ by one bucket: keep the buckets every history covers.
+  const from = Math.max(...histories.map((history) => history.requested_range.start)) * 1_000;
+  const to = Math.min(...histories.map((history) => history.requested_range.end)) * 1_000;
+  const samples = [...buckets.values()].filter((value) => value.sampledAt > from && value.sampledAt <= to).sort((left, right) => left.sampledAt - right.sampledAt).map((value) => {
     const observedMemory = [...value.memory.values()];
     return {
       sampledAt: value.sampledAt,
@@ -178,27 +176,17 @@ export async function loadRuntimeDurableSnapshot(
   snapshot: RuntimeDashboardSnapshot,
   range: RuntimeDurableRange,
   signal?: AbortSignal,
-  now: () => number = Date.now,
   targetLimit = RUNTIME_DURABLE_TARGET_LIMIT,
 ): Promise<RuntimeDurableSnapshot | null> {
   const targets = durableTargets(snapshot);
   if (targets.length > targetLimit) {
-    throw new RuntimeDurableHistoryIncompleteError(
-      `Durable Runtime history exceeds the ${targetLimit} target Web query budget.`,
-    );
+    throw new RuntimeDurableHistoryIncompleteError(targetLimit);
   }
-  const rangeEndSeconds = Math.floor(now() / 1_000);
-  const rangeStartSeconds = rangeEndSeconds - range / 1_000;
   const results = await mapBounded(targets, 4, async (session): Promise<RuntimeHistory | null> => {
     const reader = readerFor(session.id);
     if (!reader) return null;
     try {
-      return await reader.retrieveRuntimeHistory(session.id, {
-        start: rangeStartSeconds,
-        end: rangeEndSeconds,
-        maxPoints: RUNTIME_DURABLE_MAX_POINTS,
-        signal,
-      });
+      return await reader.retrieveRuntimeHistory(session.id, range, { signal });
     } catch (error) {
       if (isNotFound(error)) return null;
       throw error;
@@ -210,16 +198,13 @@ export async function loadRuntimeDurableSnapshot(
   const covered = targets.filter((_, index) => results[index] !== null);
   const resolutions = new Set(histories.map((history) => history.resolution_seconds));
   if (resolutions.size > 1) {
-    throw new RuntimeDurableHistoryIncompleteError("Durable Runtime histories returned inconsistent resolutions.");
+    throw new RuntimeDurableHistoryIncompleteError(null);
   }
   return {
     samples: runtimeDurableTrendSamples(covered, histories),
-    rangeStart: rangeStartSeconds * 1_000,
-    rangeEnd: rangeEndSeconds * 1_000,
     resolutionSeconds: histories[0]?.resolution_seconds ?? 30,
     targetCount: covered.length,
     sampleCount: histories.reduce((total, history) => total + history.coverage.sample_count, 0),
     expectedSampleCount: histories.reduce((total, history) => total + history.coverage.expected_sample_count, 0),
-    loadedAt: now(),
   };
 }

apps/web/src/features/metrics/SandboxMetricsPage.tsx

@@ -25,9 +25,10 @@ import { ListToolbar, listSummary, NameCell, SearchField } from "../../component
 import { useConsoleNavigation } from "../../lib/console-navigation";
 import { formatBytes, formatClock, formatCompact, formatCores, formatDateTime, formatDuration, formatInteger, formatPercent, formatRelative, MISSING } from "../../lib/format";
 import { projectClient, ProjectName, useProjects } from "../../lib/projects";
+import { readError } from "../../lib/read-error";
 import { sandboxStateLabel } from "../../lib/sandbox-labels";
 import { nodeProviderDiagnostic } from "../../lib/sandbox-diagnostic";
-import { loadRuntimeDurableSnapshot, RUNTIME_DURABLE_RANGES, type RuntimeDurableRange } from "../dashboard/runtime-history";
+import { loadRuntimeDurableSnapshot, RUNTIME_DURABLE_RANGES, RuntimeDurableHistoryIncompleteError, type RuntimeDurableRange } from "../dashboard/runtime-history";
 import type { RuntimeDashboardSnapshot } from "../dashboard/runtime-snapshot";
 import { RUNTIME_SNAPSHOT_REFRESH_MS } from "../dashboard/runtime-snapshot";
 import { DiagnosticTip } from "../fleet/DiagnosticTip";
@@ -104,7 +105,7 @@ export function SandboxMetricsPage() {
   const refreshing = runtimeState.status === "loading" || (fleetState.status === "ready" && fleetState.refreshing);
   const updatedAt = runtimeState.load?.loadedAt ?? fleet?.loadedAt ?? null;
   const message = fleetMessage(fleetState, t);
-  const [range, setRange] = useState<RuntimeDurableRange>(RUNTIME_DURABLE_RANGES[0].milliseconds);
+  const [range, setRange] = useState<RuntimeDurableRange>(RUNTIME_DURABLE_RANGES[0]);
   // A node or a sandbox opens in a dialog with its own figures, instead of leaving the page.
   const [openNode, setOpenNode] = useState<string | null>(null);
   const [openRuntime, setOpenRuntime] = useState<string | null>(null);
@@ -123,9 +124,9 @@ export function SandboxMetricsPage() {
         actions={<>
           <SegmentedControl
             label={t("range.label")}
-            value={String(range)}
-            options={RUNTIME_DURABLE_RANGES.map((entry) => ({ value: String(entry.milliseconds), label: t(`range.${entry.label}`) }))}
-            onChange={(value) => setRange(Number(value) as RuntimeDurableRange)}
+            value={range}
+            options={RUNTIME_DURABLE_RANGES.map((entry) => ({ value: entry, label: t(`range.${entry}`) }))}
+            onChange={setRange}
           />
           <RefreshButton refreshing={refreshing} updatedAt={updatedAt ? formatClock(updatedAt, locale) : null} onClick={() => { refreshRuntime(); refreshFleet(); }} />
         </>}
@@ -286,11 +287,21 @@ function useRuntimeHistory(snapshot: RuntimeDashboardSnapshot | null, range: Run
     placeholderData: keepPreviousData,
     refetchInterval: 30_000,
     refetchIntervalInBackground: false,
+    // One read per Session per poll: the 30-second poll is the retry, and a
+    // retry here would read every Session again.
+    retry: false,
   });
 }
 
+/** Core's typed errors and Web's own read bound, in the console's language. */
+function historyFailure(error: unknown, t: TFunction<"metrics">, tCommon: TFunction<"common">): string {
+  if (!(error instanceof RuntimeDurableHistoryIncompleteError)) return readError(error, tCommon);
+  return error.limit === null ? t("sandbox.charts.historyMixed") : t("sandbox.charts.historyBudget", { limit: error.limit });
+}
+
 function HostedRuntimeSection({ state, stale, fleet, range, onOpen }: { state: RuntimeState; stale: string | null; fleet: ReturnType<typeof fleetSnapshot>; range: RuntimeDurableRange; onOpen: (sessionId: string) => void }) {
   const { t, i18n } = useTranslation("metrics");
+  const { t: tCommon } = useTranslation("common");
   const locale = i18n.resolvedLanguage;
   const load = state.load;
   const usage = useMemo(() => (load ? hostedRuntimeUsage(load.observations) : null), [load]);
@@ -311,7 +322,7 @@ function HostedRuntimeSection({ state, stale, fleet, range, onOpen }: { state: R
     body = (
       <>
         {durable ? <RuntimeCharts samples={durable.samples} resolutionSeconds={durable.resolutionSeconds} />
-          : history.isError ? <p className="page-status" role="alert">{t("sandbox.charts.historyFailed", { reason: history.error instanceof Error ? history.error.message : "" })}</p>
+          : history.isError ? <p className="page-status" role="alert">{t("sandbox.charts.historyFailed", { reason: historyFailure(history.error, t, tCommon) })}</p>
             : history.isFetched ? <p className="page-status">{t("sandbox.charts.historyUnavailable")}</p> : null}
         <RuntimeTable rows={hostedRuntimeRows(load, "", fleet)} showProject showNode={fleet?.deployment.mode !== "direct"} showCompute={Boolean(fleet?.deployment.suspension)} onOpen={onOpen} />
       </>
@@ -464,6 +475,7 @@ function NodeDialog({ node, rows, load, range, stale, suspends, onClose }: {
   onClose: () => void;
 }) {
   const { t, i18n } = useTranslation("metrics");
+  const { t: tCommon } = useTranslation("common");
   const locale = i18n.resolvedLanguage;
   const { navigate } = useConsoleNavigation();
   const shown = useLast(node);
@@ -474,9 +486,8 @@ function NodeDialog({ node, rows, load, range, stale, suspends, onClose }: {
   );
   const history = useRuntimeHistory(snapshot, range);
   // The machine itself: its last heartbeat and host history over the page's range.
-  const hostRange = RUNTIME_DURABLE_RANGES.find((entry) => entry.milliseconds === range)?.label ?? "1h";
   const detail = useQuery({
-    ...nodeDetailQuery(shown?.id ?? "", hostRange),
+    ...nodeDetailQuery(shown?.id ?? "", range),
     enabled: node !== null,
     placeholderData: keepPreviousData,
     refetchInterval: RUNTIME_SNAPSHOT_REFRESH_MS,
@@ -530,7 +541,7 @@ function NodeDialog({ node, rows, load, range, stale, suspends, onClose }: {
           >
             {!sessionIds.size ? <EmptyState title={t("sandbox.nodeDialog.noRuntimes")} />
               : history.data ? <RuntimeCharts samples={history.data.samples} resolutionSeconds={history.data.resolutionSeconds} height={150} />
-                : history.isError ? <p className="page-status" role="alert">{t("sandbox.charts.historyFailed", { reason: history.error instanceof Error ? history.error.message : "" })}</p>
+                : history.isError ? <p className="page-status" role="alert">{t("sandbox.charts.historyFailed", { reason: historyFailure(history.error, t, tCommon) })}</p>
                   : history.isFetched ? <p className="page-status">{t("sandbox.charts.historyUnavailable")}</p>
                     : <TableSkeleton label={t("sandbox.runtimeLoading")} rows={3} columns={4} />}
           </Section>

apps/web/src/features/sessions/SessionRuntimeSection.tsx

@@ -7,6 +7,7 @@ import { failedLast, useFailureToast } from "../../components/Toast";
 import { DashboardSkeleton } from "../../components/Skeleton";
 import { Kpi, KpiStrip, Section, SegmentedControl, type Tone } from "../../components/console-ui";
 import { formatBytes, formatCores, formatDateTime, formatPercent, formatRelative, MISSING } from "../../lib/format";
+import { readError } from "../../lib/read-error";
 import { RuntimeCharts } from "../metrics/RuntimeCharts";
 import type { RuntimeTrendSample } from "../dashboard/runtime-trends";
 
@@ -20,7 +21,6 @@ function sampleSpacing(samples: readonly RuntimeTrendSample[]): number {
 }
 
 const HISTORY_REFRESH_MS = 30_000;
-type RangeLabel = (typeof SESSION_RUNTIME_RANGES)[number]["label"];
 
 function message(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
@@ -57,9 +57,9 @@ export function SessionRuntimeSection({
   refreshToken: number;
 }) {
   const { t, i18n } = useTranslation("sessions");
+  const { t: tCommon } = useTranslation("common");
   const locale = i18n.resolvedLanguage;
-  const [rangeLabel, setRangeLabel] = useState<RangeLabel>(SESSION_RUNTIME_RANGES[0].label);
-  const range = SESSION_RUNTIME_RANGES.find((entry) => entry.label === rangeLabel)!.milliseconds as SessionRuntimeRange;
+  const [range, setRange] = useState<SessionRuntimeRange>(SESSION_RUNTIME_RANGES[0]);
 
   const observationRead = useQuery(sessionObservationQuery(projectId, session.id));
   useOnChange(revision, observationRead.refetch);
@@ -76,7 +76,7 @@ export function SessionRuntimeSection({
   const history: { state: "loading" | "ready" | "unavailable" | "failed"; value: SessionRuntimeHistory | null; error: string | null } = {
     state: historyRead.isError ? "failed" : historyRead.data === undefined ? "loading" : historyRead.data === null ? "unavailable" : "ready",
     value: historyRead.data ?? null,
-    error: historyRead.isError ? message(historyRead.error) : null,
+    error: historyRead.isError ? readError(historyRead.error, tCommon) : null,
   };
   // Both reads repeat while the Session runs; a failure is reported once while it lasts.
   useFailureToast(failedLast(observationRead), t("runtime.observationFailed", { reason: observationError ?? "" }), "session-runtime-observation");
@@ -104,9 +104,9 @@ export function SessionRuntimeSection({
       actions={(
         <SegmentedControl
           label={t("runtime.range")}
-          value={rangeLabel}
-          options={SESSION_RUNTIME_RANGES.map((entry) => ({ value: entry.label, label: t(`runtime.ranges.${entry.label}`) }))}
-          onChange={setRangeLabel}
+          value={range}
+          options={SESSION_RUNTIME_RANGES.map((entry) => ({ value: entry, label: t(`runtime.ranges.${entry}`) }))}
+          onChange={setRange}
         />
       )}
     >