From 6f9257bc328dc68d1d6c04da8b99cbd16f98af47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:36:29 +0100 Subject: [PATCH 1/4] Read UK staging contract v3: blocked runs, block details and failure classes Microcosm's UK staging documents move to schema_version 3 (PolicyEngine/microcosm#1147): a real `blocked` run status with a `block` (gate phase, blocking failure count, blocking gate ids), and `failure_class` as a fifth failure key. Version 2 keeps reading exactly as before. - staging-contract: accept v3 exactly (status and event enums, `block`, the five-key failure, blocked/failed/other rules); versions 2 and 3 are the structured contract everywhere a check was pinned to 2; a run cannot mix versions. - staging-artifact, build-runs: v3 manifests list as structured runs, their diagnostics keep the declared path and digest check, and `blocked` is final. - build-monitor: `blocked` maps straight to the blocked state with a failure line built from the block (falling back to the terminal event, the collector's copy); the gate-count inference stays for version 2 only; a v3 failure's class and code come from the run's failure; recorded refusals count their class in the stop statistics. - views: labels for the new failure classes, a warning tone and pill for blocked runs on the staging page. - Fixtures: microcosm's v3 staging fixtures copied byte for byte from PolicyEngine/microcosm#1147 head 0e80b0361, SHA256SUMS digest pinned. Co-Authored-By: Claude Opus 5.5 --- docs/build-monitor.md | 13 +- .../microcosm/build-monitor-view.tsx | 12 +- .../microcosm/microcosm-staging-view.tsx | 28 ++- frontend/lib/microcosm/build-monitor.test.ts | 209 ++++++++++++++++++ frontend/lib/microcosm/build-monitor.ts | 121 +++++++--- frontend/lib/microcosm/build-runs.ts | 9 +- .../fixtures/staging-contract/v3/SHA256SUMS | 14 ++ .../runs/uk-blocked-v3-fixture/events.ndjson | 4 + .../runs/uk-blocked-v3-fixture/progress.json | 54 +++++ .../uk-blocked-v3-fixture/run_manifest.json | 48 ++++ .../calibration_progress.json | 21 ++ .../uk-calibration-v3-fixture/events.ndjson | 5 + .../uk-calibration-v3-fixture/progress.json | 38 ++++ .../run_manifest.json | 42 ++++ .../runs/uk-spine-v3-fixture/events.ndjson | 8 + .../runs/uk-spine-v3-fixture/progress.json | 38 ++++ .../uk-spine-v3-fixture/run_manifest.json | 42 ++++ .../staging-contract/v3/contract-cases.json | 44 ++++ .../runs/uk-failed-v3-fixture/events.ndjson | 3 + .../runs/uk-failed-v3-fixture/progress.json | 42 ++++ .../uk-failed-v3-fixture/run_manifest.json | 46 ++++ .../lib/microcosm/staging-artifact.test.ts | 106 +++++++++ frontend/lib/microcosm/staging-artifact.ts | 41 ++-- .../lib/microcosm/staging-contract.test.ts | 199 ++++++++++++++++- frontend/lib/microcosm/staging-contract.ts | 154 +++++++++---- 25 files changed, 1229 insertions(+), 112 deletions(-) create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/SHA256SUMS create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/events.ndjson create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/progress.json create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/run_manifest.json create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/calibration_progress.json create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/events.ndjson create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/progress.json create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/run_manifest.json create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/events.ndjson create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/progress.json create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/run_manifest.json create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/contract-cases.json create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/events.ndjson create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/progress.json create mode 100644 frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/run_manifest.json diff --git a/docs/build-monitor.md b/docs/build-monitor.md index 4955642d..43d7fbdd 100644 --- a/docs/build-monitor.md +++ b/docs/build-monitor.md @@ -20,7 +20,7 @@ checks stop builds. The run selected on either tab carries over to the other. organization, or an unavailable network does not stop a build. Events remain in the local retry queue for a later Microcosm run to deliver. - **Historical Hugging Face telemetry.** During migration, the hosted view also - reads version 1 and version 2 run documents from each country's Hugging Face + reads version 1, 2 and 3 run documents from each country's Hugging Face staging repository. A collector record replaces a historical record with the same run id. - **Local run folders.** The dashboard can still inspect legacy or diagnostic @@ -66,14 +66,14 @@ checks stop builds. The run selected on either tab carries over to the other. With `MICROCOSM_LOCAL_RUNS_DIR=/tmp/smoke` the run appears on the UK Build progress tab as it builds. -- **Unit tests** cover both telemetry schemas, the forecast, gate refusals and +- **Unit tests** cover every telemetry schema, the forecast, gate refusals and local discovery: `frontend/lib/microcosm/build-monitor.test.ts` and `local-build-runs.test.ts`. ## Forecast The forecast compares the run with finished runs of the same pipeline from -the same source (version 2 runs by pipeline id; version 1 US runs as one +the same source (version 2 and 3 runs by pipeline id; version 1 US runs as one release pipeline), up to the 12 most recent. Runs of one pipeline can still differ in what they do: UK local candidates kept @@ -143,7 +143,12 @@ The local telemetry emitter service reports: the median run time spent by then and what the runs recorded: error types and codes, failure classes, and messages or gate failure lines. It shows recorded values only. A run that recorded nothing says so, and a run that went silent -wrote no final event, so it has nothing about the cause. "Run history" lists +wrote no final event, so it has nothing about the cause. A version 3 run records +a refusal itself: status `blocked`, with the gate phase (`preflight`, +`terminal`, ...) and the gates that refused, which may be none by name. A +version 2 run closed a refusal as `completed`; the monitor reads it from the +gate battery's blocking failure count, so it is not counted as a recorded +failure class. "Run history" lists every run of the pipeline, newest first, on one time scale; selecting a row opens that run, and runs with a different stage sequence are listed apart. diff --git a/frontend/components/microcosm/build-monitor-view.tsx b/frontend/components/microcosm/build-monitor-view.tsx index c6741586..68e46934 100644 --- a/frontend/components/microcosm/build-monitor-view.tsx +++ b/frontend/components/microcosm/build-monitor-view.tsx @@ -91,8 +91,16 @@ const FAILURE_CLASS_LABEL: Record = { terminated: "Terminated (SIGTERM)", interrupted: "Interrupted", out_of_memory: "Out of memory", - refused: "Refused before building", + // The build exited non-zero without raising and without a recorded gate block. + refused: "Refused without a gate block", + // A sampled spine build hit a named edge and discarded the attempt. + aborted: "Rung aborted", + // The gates refused, but the block's details could not be recorded. + unrecorded_gate_block: "Gate block not recorded", error: "Error", + build_failure: "Build failure", + unexpected_process_exit: "Process exited unexpectedly", + dry_run_refusal: "Refused in a dry run", stopped_without_final_event: "Stopped without a final event", unclassified: "Failed, no class recorded", }; @@ -479,7 +487,7 @@ function RunOverview({ running && forecast?.remaining_p90_ms != null ? `Up to ${fmtDuration(forecast.remaining_p90_ms)} (90%)` : run.failure?.stage - ? `Failed in ${formatStageName(run.failure.stage)}` + ? `${run.state === "blocked" ? "Refused at" : "Failed in"} ${formatStageName(run.failure.stage)}` : undefined } size="sm" diff --git a/frontend/components/microcosm/microcosm-staging-view.tsx b/frontend/components/microcosm/microcosm-staging-view.tsx index 14862f9b..d48a502a 100644 --- a/frontend/components/microcosm/microcosm-staging-view.tsx +++ b/frontend/components/microcosm/microcosm-staging-view.tsx @@ -97,7 +97,7 @@ function statusTone(status: string | null | undefined): StatusTone { return "success"; } if (status === "failed") return "danger"; - if (status === "stalled") return "warning"; + if (status === "blocked" || status === "stalled") return "warning"; if (status === "running" || status === "queued") return "info"; return "neutral"; } @@ -297,15 +297,25 @@ function agoLabel(value: string | null | undefined): string { function detailChips(details: unknown): [string, string][] { if (!details || typeof details !== "object") return []; return Object.entries(details as Record) - .filter(([, v]) => v == null || ["string", "number", "boolean"].includes(typeof v)) + .filter( + ([, v]) => + v == null || + ["string", "number", "boolean"].includes(typeof v) || + // A list of names, such as the gates that refused a candidate. + (Array.isArray(v) && v.every((item) => typeof item === "string")), + ) .slice(0, 8) .map(([k, v]) => { const num = typeof v === "number"; - const shown = num - ? Math.abs(v as number) >= 1000 - ? fmtCompact(v as number) - : fmt(v as number, { digits: Math.abs(v as number) < 1 ? 4 : 2 }) - : String(v); + const shown = Array.isArray(v) + ? v.length + ? v.join(", ") + : "none" + : num + ? Math.abs(v as number) >= 1000 + ? fmtCompact(v as number) + : fmt(v as number, { digits: Math.abs(v as number) < 1 ? 4 : 2 }) + : String(v); return [k, shown] as [string, string]; }); } @@ -488,6 +498,7 @@ function RunInternalsPanel({ : null; const chips = detailChips(event.details); const failed = event.status === "failed"; + const blocked = event.status === "blocked"; return ( {failed && failed} + {blocked && blocked} {timeLabel(time)} - {index === all.length - 1 && event.stage !== "complete" && !failed + {index === all.length - 1 && event.stage !== "complete" && !failed && !blocked ? status === "stalled" ? "⚠ last event" : "…" diff --git a/frontend/lib/microcosm/build-monitor.test.ts b/frontend/lib/microcosm/build-monitor.test.ts index 8c2c4939..f9db4e9c 100644 --- a/frontend/lib/microcosm/build-monitor.test.ts +++ b/frontend/lib/microcosm/build-monitor.test.ts @@ -1,3 +1,6 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + import { describe, expect, test } from "bun:test"; import { @@ -20,6 +23,11 @@ import { stageStatistics, STALL_MS, } from "./build-monitor"; +import { + parseStagingEvents, + parseStagingManifest, + parseStagingProgress, +} from "./staging-contract"; const T0 = Date.parse("2026-09-26T10:00:00Z"); const min = (n: number) => n * 60_000; @@ -127,6 +135,7 @@ describe("buildTimeline, version 1 transitions", () => { error_code: null, failure_class: null, diagnostic_reference: null, + block: null, }); }); @@ -319,7 +328,12 @@ describe("gate refusals in completed version 2 runs", () => { ); expect(run.state).toBe("blocked"); expect(run.failure?.stage).toBe("gate_battery"); + expect(run.failure?.block).toBeNull(); expect(run.spans[0].failures).toEqual(["coverage: failed", "loss: blocked"]); + // The refusal is inferred from gate counts, so no class is counted as recorded. + const [stop] = stopStatistics([run]); + expect(stop.failure_classes).toEqual([]); + expect(stop.reasons.map((reason) => reason.value)).toEqual(["coverage", "loss"]); }); test("a clean gate battery leaves the run finished", () => { @@ -985,3 +999,198 @@ describe("solver passes inside calibration", () => { expect(forecast.solver).toBeNull(); }); }); + +describe("version 3 outcomes", () => { + const FIXTURES = join(import.meta.dir, "fixtures", "staging-contract", "v3"); + const fixtureText = (path: string) => readFileSync(join(FIXTURES, path)).toString(); + const fixtureJson = (path: string) => JSON.parse(fixtureText(path)) as Record; + + // A run folder from the copied Microcosm fixtures, read as the staging page reads it. + function fixtureRun(root: string, runId: string): BuildRunDocuments { + return { + run_id: runId, + source: "staging", + country: "uk", + progress: parseStagingProgress(fixtureJson(`${root}/progress.json`)), + run_manifest: parseStagingManifest(fixtureJson(`${root}/run_manifest.json`)), + calibration_progress: null, + events: parseStagingEvents(fixtureText(`${root}/events.ndjson`)), + }; + } + + const FIXTURE_END = Date.parse("2026-01-05T00:00:03+00:00"); + + test("a run blocked at the terminal gates reads as blocked with its gates", () => { + const run = buildTimeline( + fixtureRun("blocked/runs/uk-blocked-v3-fixture", "uk-blocked-v3-fixture"), + FIXTURE_END + min(1), + ); + expect(run.schema_version).toBe(3); + expect(run.state).toBe("blocked"); + expect(run.ended_ms).toBe(FIXTURE_END); + expect(run.failure).toEqual({ + stage: "gate_battery", + message: "The gates refused the candidate at terminal (1 blocking failure: uk_target_fit).", + error_type: null, + error_code: null, + failure_class: "gate_refused", + diagnostic_reference: null, + block: { phase: "terminal", blocking_failure_count: 1, blocking_gate_ids: ["uk_target_fit"] }, + }); + // The terminal event is a marker, not a stage of its own. + expect(run.spans.map((span) => span.stage)).toEqual(["target_compilation", "gate_battery"]); + expect(run.spans[1]).toMatchObject({ refused: true, failures: ["uk_target_fit: failed"] }); + + const [stop] = stopStatistics([run]); + expect(stop).toMatchObject({ outcome: "blocked", stage: "gate_battery", unexplained: 0 }); + expect(stop.failure_classes).toEqual([{ value: "gate_refused", count: 1 }]); + expect(stop.reasons).toEqual([{ value: "uk_target_fit", count: 1 }]); + }); + + test("a run blocked at preflight, without gate counts, reads as blocked from its block", () => { + const event = (sequence: number, offset: number, stage: string, status: string, details: Record = {}) => ({ + schema_version: 3, + sequence, + run_id: "uk-preflight", + event_type: "stage", + stage, + stage_id: stage, + status, + time: at(offset), + message: null, + details, + }); + const block = { phase: "preflight", blocking_failure_count: 2, blocking_gate_ids: [] }; + const documents: BuildRunDocuments = { + run_id: "uk-preflight", + source: "staging", + country: "uk", + progress: { + schema_version: 3, + run_id: "uk-preflight", + country_code: "GB", + pipeline: { id: "uk_local_candidate", version: "2026.10" }, + started_at: at(0), + updated_at: at(min(5)), + status: "blocked", + current_stage: "blocked", + failure: null, + block, + }, + run_manifest: null, + calibration_progress: null, + events: [ + event(1, 0, "created", "started"), + event(2, min(1), "uk.full.gates.preflight", "started"), + event(3, min(4), "uk.full.gates.preflight", "completed"), + event(4, min(5), "blocked", "blocked", block), + ], + }; + + // A recorded end: the run never goes stalled, however long ago it ended. + const run = buildTimeline(documents, T0 + min(5) + STALL_MS + min(1)); + expect(run.state).toBe("blocked"); + expect(run.failure).toMatchObject({ + stage: null, + message: "The gates refused the candidate at preflight (2 blocking failures, no gate named).", + failure_class: "gate_refused", + block, + }); + const [stop] = stopStatistics([run]); + expect(stop).toMatchObject({ outcome: "blocked", stage: "uk.full.gates.preflight", unexplained: 0 }); + expect(stop.failure_classes).toEqual([{ value: "gate_refused", count: 1 }]); + expect(stop.reasons).toEqual([{ value: "2 blocking failures, no gate named", count: 1 }]); + }); + + test("a version 3 completed run is not inferred blocked from gate counts", () => { + const documents = fixtureRun("completed-spine/runs/uk-spine-v3-fixture", "uk-spine-v3-fixture"); + const complete = documents.events[documents.events.length - 1]; + // A gate battery that reports blocking failures, before the run completes. + const gates = { + ...complete, + stage: "gate_battery", + stage_id: "gate_battery", + message: null, + details: { blocking_failure_count: 1 }, + }; + const run = buildTimeline( + { + ...documents, + events: [ + ...documents.events.slice(0, -1), + gates, + { ...complete, sequence: Number(complete.sequence) + 1 }, + ], + }, + Date.parse("2026-01-03T00:00:00Z"), + ); + expect(run.spans.find((span) => span.stage === "gate_battery")?.refused).toBe(true); + expect(run.state).toBe("passed"); + }); + + test("a version 3 failed run carries its code and class into the stop statistics", () => { + const run = buildTimeline( + fixtureRun("failed/runs/uk-failed-v3-fixture", "uk-failed-v3-fixture"), + Date.parse("2026-01-04T00:00:03+00:00"), + ); + expect(run.state).toBe("failed"); + expect(run.failure).toEqual({ + stage: "input_verification", + message: "The build failed during input_verification.", + error_type: "RuntimeError", + error_code: "BUILD_FAILED", + failure_class: "error", + diagnostic_reference: "diagnostics/operator-error.txt", + block: null, + }); + const [stop] = stopStatistics([run]); + expect(stop.error_codes).toEqual([{ value: "BUILD_FAILED", count: 1 }]); + expect(stop.failure_classes).toEqual([{ value: "error", count: 1 }]); + }); + + test("a collector run closed by a blocked run event reads its block from that event", () => { + const details = { + phase: "terminal", + blocking_failure_count: 3, + blocking_gate_ids: ["a", "b", "c", "d", "e", "f"], + gate_statuses: { a: "failed" }, + }; + const run = buildTimeline( + { + run_id: "collector-blocked", + source: "staging", + country: "uk", + // The collector's documents keep the version 2 layout. + progress: { + schema_version: 2, + run_id: "collector-blocked", + country_code: "GB", + pipeline: { id: "uk_local_candidate", version: "collector-v1" }, + started_at: at(0), + updated_at: at(min(9)), + status: "blocked", + current_stage: "blocked", + failure: null, + }, + run_manifest: null, + calibration_progress: null, + events: [ + { schema_version: 2, sequence: 1, event_type: "stage", stage_id: "calibrating", status: "started", timestamp: at(0), details: {} }, + { schema_version: 2, sequence: 2, event_type: "stage", stage_id: "calibrating", status: "completed", timestamp: at(min(8)), details: {} }, + { schema_version: 2, sequence: 3, event_type: "stage", stage_id: "blocked", status: "blocked", timestamp: at(min(9)), details }, + ], + }, + T0 + min(10), + ); + expect(run.state).toBe("blocked"); + expect(run.ended_ms).toBe(T0 + min(9)); + expect(run.failure?.block).toEqual({ + phase: "terminal", + blocking_failure_count: 3, + blocking_gate_ids: ["a", "b", "c", "d", "e", "f"], + }); + expect(run.failure?.message).toBe( + "The gates refused the candidate at terminal (3 blocking failures: a, b, c, d, e and 1 more).", + ); + }); +}); diff --git a/frontend/lib/microcosm/build-monitor.ts b/frontend/lib/microcosm/build-monitor.ts index ac31f879..b864ff16 100644 --- a/frontend/lib/microcosm/build-monitor.ts +++ b/frontend/lib/microcosm/build-monitor.ts @@ -3,19 +3,21 @@ // completion forecast from comparable past runs, and cross-run timing and // gate statistics. // -// Both telemetry schemas are supported: +// Every telemetry schema is supported: // - version 1 (US fiscal-refresh release): every event is a transition into a // stage, so a stage lasts until the next event. Passing gates are silent; // failing gates emit a `failed` transition, and `telemetry.fail` appends a // terminal `failed` stage carrying the error. // - version 2 (UK): explicit `started` / `completed` / `failed` events that // nest, plus `calibration` events that are progress, not stages. +// - version 3 (UK): version 2 plus a `blocked` run status with the gate phase +// and gates that refused the candidate, and a class on every failure. type JsonObject = Record; export type BuildRunSource = "local" | "staging"; -// `blocked`: the process ran to the end but its gates refused the candidate. -// UK builds close such runs as `completed` (microcosm full_build_cli), so the +// `blocked`: the gates refused the candidate. Version 3 records it as the run's +// status; version 2 UK builds closed such runs as `completed`, so for those the // monitor reads the gate counts to tell them apart. export type BuildRunState = "running" | "passed" | "blocked" | "failed" | "stalled"; export type BuildPhase = "setup" | "checks" | "compile" | "calibrate" | "export"; @@ -35,7 +37,7 @@ export const STALL_MS = 6 * 60 * 60 * 1000; // the process was most likely killed (out of memory, SIGKILL). export const HEARTBEAT_STALE_MS = 5 * 60 * 1000; const HISTORY_LIMIT = 12; -const MARKER_STAGES = new Set(["created", "complete", "failed"]); +const MARKER_STAGES = new Set(["created", "complete", "failed", "blocked"]); const GATE_PATTERN = /(^|_)(gate|gates|check|checks|validation|composition|presence|smoke|audit|evaluation)(_|$)/; @@ -148,13 +150,24 @@ export interface BuildFailure { stage: string | null; message: string | null; error_type: string | null; - // A machine-readable code (version 2), such as BUILD_FAILED. + // A machine-readable code (version 2 and later), such as BUILD_FAILED. error_code: string | null; - // gate_refused, terminated, interrupted, out_of_memory, refused or error, - // as the run recorded it; null when it recorded none. + // gate_refused for a refusal by the gates; otherwise the class the run + // recorded (terminated, interrupted, out_of_memory, error, ...), or null. failure_class: string | null; - // Where the build machine keeps the details (version 2), when it says. + // Where the build machine keeps the details (version 2 and later), when it says. diagnostic_reference: string | null; + // The refusal a version 3 run recorded; null when it recorded none (a + // version 2 refusal is inferred from its gate counts). + block: RecordedBlock | null; +} + +export interface RecordedBlock { + // The gate phase that refused the candidate: preflight, terminal, ... + phase: string | null; + blocking_failure_count: number | null; + // Empty when the refusal named no gate. + blocking_gate_ids: string[]; } export interface BuildTimeline { @@ -163,7 +176,7 @@ export interface BuildTimeline { country: string; pipeline: string; pipeline_label: string; - schema_version: 1 | 2; + schema_version: 1 | 2 | 3; state: BuildRunState; started_ms: number | null; updated_ms: number | null; @@ -246,13 +259,13 @@ export function formatStageName(stage: string): string { function pipelineIdentity( documents: BuildRunDocuments, - schemaVersion: 1 | 2, + schemaVersion: 1 | 2 | 3, ): { key: string; label: string } { const identity = documents.progress ?? documents.run_manifest; const pipeline = obj(identity?.pipeline); const pipelineId = str(pipeline?.id); const country = str(identity?.country_code) ?? documents.country.toUpperCase(); - if (schemaVersion === 2 && pipelineId) { + if (schemaVersion >= 2 && pipelineId) { // A smoke run on a synthetic fixture is not comparable with a full run of // the same pipeline, so the run kind is part of the key. const kind = str(identity?.run_kind); @@ -278,6 +291,7 @@ function runState( return "passed"; } if (rawStatus === "failed") return "failed"; + if (rawStatus === "blocked") return "blocked"; if (updatedMs != null && nowMs - updatedMs > stallMs) return "stalled"; return "running"; } @@ -475,7 +489,7 @@ function spansFromTransitions(events: JsonObject[]): { return { spans, terminal }; } -// Version 2: explicit, nestable started/completed/failed events. +// Version 2 and later: explicit, nestable started/completed/failed events. function spansFromLifecycle(events: JsonObject[]): { spans: StageSpan[]; terminal: { stage: string; event: JsonObject; time: number } | null; @@ -712,19 +726,53 @@ export function solverSegments( return segments; } +// The refusal a version 3 run recorded: on its run documents, or in the +// details of its terminal `blocked` event (the collector's copy). +function recordedBlock( + documents: BuildRunDocuments, + terminal: { stage: string; event: JsonObject } | null, +): RecordedBlock | null { + const block = + obj(documents.progress?.block) ?? + obj(documents.run_manifest?.block) ?? + (terminal?.stage === "blocked" ? obj(terminal.event.details) : null); + if (!block) return null; + return { + phase: str(block.phase), + blocking_failure_count: num(block.blocking_failure_count), + blocking_gate_ids: stringList(block.blocking_gate_ids), + }; +} + +const BLOCK_GATES_SHOWN = 5; + +// "3 blocking failures: coverage, fit, loss", or "2 blocking failures, no gate named". +function blockSummary(block: RecordedBlock): string { + const count = block.blocking_failure_count; + const failures = + count == null ? "blocking failures" : `${count} blocking failure${count === 1 ? "" : "s"}`; + const gates = block.blocking_gate_ids; + if (!gates.length) return `${failures}, no gate named`; + const shown = gates.slice(0, BLOCK_GATES_SHOWN).join(", "); + const more = gates.length > BLOCK_GATES_SHOWN ? ` and ${gates.length - BLOCK_GATES_SHOWN} more` : ""; + return `${failures}: ${shown}${more}`; +} + export function buildTimeline( documents: BuildRunDocuments, nowMs: number = Date.now(), ): BuildTimeline { const progress = documents.progress; const identity = progress ?? documents.run_manifest; - const schemaVersion: 1 | 2 = identity?.schema_version === 2 ? 2 : 1; + const declaredVersion = num(identity?.schema_version); + const schemaVersion: 1 | 2 | 3 = + declaredVersion != null && declaredVersion >= 3 ? 3 : declaredVersion === 2 ? 2 : 1; const events = [...documents.events].sort((a, b) => { const seq = (num(a.sequence) ?? 0) - (num(b.sequence) ?? 0); return seq || (eventTime(a) ?? 0) - (eventTime(b) ?? 0); }); const { spans, terminal } = - schemaVersion === 2 ? spansFromLifecycle(events) : spansFromTransitions(events); + schemaVersion >= 2 ? spansFromLifecycle(events) : spansFromTransitions(events); const eventTimes = events.map(eventTime).filter((t): t is number => t != null); const calibration = calibrationPoints(documents.calibration_progress, progress); @@ -754,7 +802,10 @@ export function buildTimeline( ); if (state === "running" && terminal?.stage === "failed") state = "failed"; if (state === "running" && terminal?.stage === "complete") state = "passed"; - if (state === "passed" && spans.some((span) => span.refused)) { + if (state === "running" && terminal?.stage === "blocked") state = "blocked"; + // Version 3 records a refusal as `blocked`; a version 2 run closed it as + // completed, with the blocking failures in its gate battery's details. + if (schemaVersion === 2 && state === "passed" && spans.some((span) => span.refused)) { state = "blocked"; } const finished = state === "passed" || state === "blocked" || state === "failed"; @@ -787,30 +838,35 @@ export function buildTimeline( let failure: BuildFailure | null = null; if (state === "blocked") { - const gate = spans.find((span) => span.refused)!; + const gate = spans.find((span) => span.refused) ?? null; + const block = recordedBlock(documents, terminal); failure = { - stage: gate.stage, - message: `The run finished, but its gates refused the candidate (${gate.message ?? "blocking failures"}).`, + stage: gate?.stage ?? null, + message: block + ? `The gates refused the candidate${block.phase ? ` at ${block.phase}` : ""} (${blockSummary(block)}).` + : `The run finished, but its gates refused the candidate (${gate?.message ?? "blocking failures"}).`, error_type: null, error_code: null, failure_class: "gate_refused", diagnostic_reference: null, + block, }; } else if (state === "failed") { - const v2Failure = obj(progress?.failure); + const runFailure = obj(progress?.failure); const failedSpan = [...spans].reverse().find((span) => span.status === "failed") ?? null; const terminalDetails = obj(terminal?.event.details); failure = { stage: failedSpan?.stage ?? null, message: - str(v2Failure?.message) ?? + str(runFailure?.message) ?? (terminal?.stage === "failed" ? str(terminal.event.message) : null) ?? failedSpan?.message ?? str(progress?.message), - error_type: str(v2Failure?.error_type) ?? str(terminalDetails?.error_type), - error_code: str(v2Failure?.error_code), - failure_class: str(terminalDetails?.failure_class), - diagnostic_reference: str(v2Failure?.local_diagnostic_reference), + error_type: str(runFailure?.error_type) ?? str(terminalDetails?.error_type), + error_code: str(runFailure?.error_code) ?? str(terminalDetails?.error_code), + failure_class: str(runFailure?.failure_class) ?? str(terminalDetails?.failure_class), + diagnostic_reference: str(runFailure?.local_diagnostic_reference), + block: null, }; } @@ -1697,14 +1753,23 @@ export function stopStatistics(runs: BuildTimeline[]): StopStat[] { .filter((span) => span.status === "failed") .flatMap((span) => span.failures.map(failureReason)); // A refusal's own message is the dashboard's summary; its failure lines - // are what the gates recorded. + // are what the gates recorded, or else the gates its block names. const message = outcome === "failed" && failure?.message ? failureReason(failure.message) : null; - const reasons = new Set(lines.length ? lines : message ? [message] : []); + const block = failure?.block ?? null; + const blockLines = block + ? block.blocking_gate_ids.length + ? block.blocking_gate_ids + : [blockSummary(block)] + : []; + const reasons = new Set(lines.length ? lines : message ? [message] : blockLines); + // A failure's class is recorded, and so is a version 3 refusal; a + // version 2 refusal is inferred from gate counts, so its class is not counted. + const recordedClass = outcome === "failed" || block != null ? failure?.failure_class : null; count(entry.errorTypes, failure?.error_type); count(entry.errorCodes, failure?.error_code); - if (outcome === "failed") count(entry.classes, failure?.failure_class); + count(entry.classes, recordedClass); for (const reason of reasons) count(entry.reasons, reason); - if (!failure?.error_type && !failure?.error_code && !(outcome === "failed" && failure?.failure_class) && !reasons.size) { + if (!failure?.error_type && !failure?.error_code && !recordedClass && !reasons.size) { entry.unexplained += 1; } entry.last = run; diff --git a/frontend/lib/microcosm/build-runs.ts b/frontend/lib/microcosm/build-runs.ts index 7b6cdfda..79d2fd1c 100644 --- a/frontend/lib/microcosm/build-runs.ts +++ b/frontend/lib/microcosm/build-runs.ts @@ -129,9 +129,12 @@ function message(error: unknown): string { return error instanceof Error ? error.message : String(error); } +// A run in one of these statuses has ended and its telemetry no longer changes. +const FINAL_STATUSES = new Set(["passed", "completed", "blocked", "failed"]); + function isFinal(documents: BuildRunDocuments): boolean { const status = documents.progress?.status ?? documents.run_manifest?.status; - return status === "passed" || status === "completed" || status === "failed"; + return typeof status === "string" && FINAL_STATUSES.has(status); } async function loadLocal(country: MicrocosmCountry): Promise { @@ -231,9 +234,7 @@ async function loadCollector(country: MicrocosmCountry): Promise { summaries.slice(0, STAGING_RUN_LIMIT).map(async (summary) => { const key = `collector:${country}:${summary.run_id}`; const cached = stagingRunCache.get(key); - const summaryFinal = ["passed", "completed", "failed"].includes( - summary.status, - ); + const summaryFinal = FINAL_STATUSES.has(summary.status); if ( cached?.final && summaryFinal && diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/SHA256SUMS b/frontend/lib/microcosm/fixtures/staging-contract/v3/SHA256SUMS new file mode 100644 index 00000000..a4caf098 --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/SHA256SUMS @@ -0,0 +1,14 @@ +35e62f17c830fe9651f8c5546430e6bb14e6ddfd0f15a10494540ec5b1e83164 blocked/runs/uk-blocked-v3-fixture/events.ndjson +f815083c6bdbb35c899975563f08d81995cb5930be062ec9987c3098d7d78781 blocked/runs/uk-blocked-v3-fixture/progress.json +d3f8a58d1e6c352c3aacee5f42b95b0d95f481c44aae9b45be9f9efbab16c70a blocked/runs/uk-blocked-v3-fixture/run_manifest.json +dc419d0f8ab3babae8371ac020bf0c8348eb2df80492583182103735fcce4d39 calibration/runs/uk-calibration-v3-fixture/calibration_progress.json +df00f9100e5d645224126ab0214d75a744cf65cbf55c0339baed6073e4c49dac calibration/runs/uk-calibration-v3-fixture/events.ndjson +cd8e51d2cb918c3e1f413b64d4ad65f0449369f2898698371d0f47614668edbb calibration/runs/uk-calibration-v3-fixture/progress.json +29deee29b720e702caaf27fa88e7868a37cb26aabf9ef1eab52192a0ff319113 calibration/runs/uk-calibration-v3-fixture/run_manifest.json +cec0bf866e479da8f7309028f98e24c308f411d9866fe5e50f7d0cd99b52511a completed-spine/runs/uk-spine-v3-fixture/events.ndjson +d37e63aa3d430050d335e986f3b569335a18ecc8409ef725e676453f6f51a95b completed-spine/runs/uk-spine-v3-fixture/progress.json +31db516ef7409838c5b66d192523578f612d5e165142093551e4e8de5e03e31c completed-spine/runs/uk-spine-v3-fixture/run_manifest.json +491fb1569ae3c39d234d57ddec20dda3f8871518f626b703abfe2d5e330eff32 contract-cases.json +ad6303cee988ab245793bf7201b58760861f11c5117a7f75fed073df1b5cc361 failed/runs/uk-failed-v3-fixture/events.ndjson +eff3de6e27cbd9a0591be951d4316559a838fff8f1d98a1032e6cc6758bc04b4 failed/runs/uk-failed-v3-fixture/progress.json +d6a7a044bff0f777982d80d78c062ad89422876aefa0736365375ba09ceb7edd failed/runs/uk-failed-v3-fixture/run_manifest.json diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/events.ndjson b/frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/events.ndjson new file mode 100644 index 00000000..4de9633b --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/events.ndjson @@ -0,0 +1,4 @@ +{"details":{},"event_type":"stage","message":"Staging run created.","run_id":"uk-blocked-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":1,"stage_id":"created","status":"started","timestamp":"2026-01-05T00:00:00+00:00"} +{"details":{"target_count":4},"event_type":"stage","message":null,"run_id":"uk-blocked-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":2,"stage_id":"target_compilation","status":"completed","timestamp":"2026-01-05T00:00:01+00:00"} +{"details":{"blocking_failure_count":1,"diagnostic_failure_count":0,"gate_statuses":{"uk_target_fit":"failed","uk_weight_ess":"passed"}},"event_type":"stage","message":null,"run_id":"uk-blocked-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":3,"stage_id":"gate_battery","status":"completed","timestamp":"2026-01-05T00:00:02+00:00"} +{"details":{"blocking_failure_count":1,"blocking_gate_ids":["uk_target_fit"],"gate_statuses":{"uk_target_fit":"failed","uk_weight_ess":"passed"},"phase":"terminal"},"event_type":"stage","message":"The gates refused the candidate at terminal.","run_id":"uk-blocked-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":4,"stage_id":"blocked","status":"blocked","timestamp":"2026-01-05T00:00:03+00:00"} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/progress.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/progress.json new file mode 100644 index 00000000..108e6524 --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/progress.json @@ -0,0 +1,54 @@ +{ + "block": { + "blocking_failure_count": 1, + "blocking_gate_ids": [ + "uk_target_fit" + ], + "phase": "terminal" + }, + "candidate_id": "uk-blocked-v3-fixture", + "country_code": "GB", + "current_stage": "blocked", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-blocked-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "details": { + "blocking_failure_count": 1, + "blocking_gate_ids": [ + "uk_target_fit" + ], + "gate_statuses": { + "uk_target_fit": "failed", + "uk_weight_ess": "passed" + }, + "phase": "terminal" + }, + "failure": null, + "message": "The gates refused the candidate at terminal.", + "non_release": true, + "operation_id": "uk_rowwise_candidate", + "pipeline": { + "id": "uk_local_candidate", + "version": "2026.10" + }, + "release_id": null, + "run_id": "uk-blocked-v3-fixture", + "run_kind": "calibration", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.progress", + "schema_version": 3, + "started_at": "2026-01-05T00:00:00+00:00", + "status": "blocked", + "updated_at": "2026-01-05T00:00:03+00:00" +} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/run_manifest.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/run_manifest.json new file mode 100644 index 00000000..b0045578 --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/blocked/runs/uk-blocked-v3-fixture/run_manifest.json @@ -0,0 +1,48 @@ +{ + "artifacts": [], + "block": { + "blocking_failure_count": 1, + "blocking_gate_ids": [ + "uk_target_fit" + ], + "phase": "terminal" + }, + "candidate_id": "uk-blocked-v3-fixture", + "country_code": "GB", + "current_stage": "blocked", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-blocked-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "failure": null, + "non_release": true, + "operation_id": "uk_rowwise_candidate", + "paths": { + "calibration_progress": null, + "events": "runs/uk-blocked-v3-fixture/events.ndjson", + "progress": "runs/uk-blocked-v3-fixture/progress.json" + }, + "pipeline": { + "id": "uk_local_candidate", + "version": "2026.10" + }, + "release_id": null, + "run_id": "uk-blocked-v3-fixture", + "run_kind": "calibration", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 3, + "started_at": "2026-01-05T00:00:00+00:00", + "status": "blocked", + "updated_at": "2026-01-05T00:00:03+00:00" +} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/calibration_progress.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/calibration_progress.json new file mode 100644 index 00000000..6c6fd33b --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/calibration_progress.json @@ -0,0 +1,21 @@ +{ + "candidate_id": "uk-calibration-candidate", + "events": [ + { + "budget_iteration": null, + "budget_iters": null, + "budget_search": null, + "epoch": 1, + "epochs": 3, + "iteration": 4, + "l0_lambda": null, + "loss": 2.5, + "phase": "solve", + "timestamp": "2026-01-03T00:00:02+00:00" + } + ], + "run_id": "uk-calibration-v3-fixture", + "schema_name": "microcosm.staging.calibration-progress", + "schema_version": 3, + "updated_at": "2026-01-03T00:00:04+00:00" +} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/events.ndjson b/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/events.ndjson new file mode 100644 index 00000000..a6e16860 --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/events.ndjson @@ -0,0 +1,5 @@ +{"details":{},"event_type":"stage","message":"Staging run created.","run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":1,"stage_id":"created","status":"started","timestamp":"2026-01-03T00:00:00+00:00"} +{"details":{"target_count":2},"event_type":"stage","message":null,"run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":2,"stage_id":"target_compilation","status":"completed","timestamp":"2026-01-03T00:00:01+00:00"} +{"details":{"budget_iteration":null,"budget_iters":null,"budget_search":null,"epoch":1,"epochs":3,"iteration":4,"l0_lambda":null,"loss":2.5,"phase":"solve"},"event_type":"calibration","message":null,"run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":3,"stage_id":"calibrating","status":"progress","timestamp":"2026-01-03T00:00:02+00:00"} +{"details":{"check_count":4},"event_type":"stage","message":null,"run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":4,"stage_id":"diagnostics","status":"completed","timestamp":"2026-01-03T00:00:03+00:00"} +{"details":{},"event_type":"stage","message":"Staging run completed.","run_id":"uk-calibration-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":5,"stage_id":"complete","status":"completed","timestamp":"2026-01-03T00:00:04+00:00"} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/progress.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/progress.json new file mode 100644 index 00000000..6fc30a21 --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/progress.json @@ -0,0 +1,38 @@ +{ + "block": null, + "candidate_id": "uk-calibration-candidate", + "country_code": "GB", + "current_stage": "complete", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-calibration-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "details": {}, + "failure": null, + "message": "Staging run completed.", + "non_release": false, + "operation_id": "uk_national_calibration", + "pipeline": { + "id": "uk_national_calibration", + "version": "2026.09" + }, + "release_id": "populace-uk-2023-v3-fixture", + "run_id": "uk-calibration-v3-fixture", + "run_kind": "calibration", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.progress", + "schema_version": 3, + "started_at": "2026-01-03T00:00:00+00:00", + "status": "completed", + "updated_at": "2026-01-03T00:00:04+00:00" +} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/run_manifest.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/run_manifest.json new file mode 100644 index 00000000..9f76195f --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/calibration/runs/uk-calibration-v3-fixture/run_manifest.json @@ -0,0 +1,42 @@ +{ + "artifacts": [], + "block": null, + "candidate_id": "uk-calibration-candidate", + "country_code": "GB", + "current_stage": "complete", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-calibration-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "failure": null, + "non_release": false, + "operation_id": "uk_national_calibration", + "paths": { + "calibration_progress": "runs/uk-calibration-v3-fixture/calibration_progress.json", + "events": "runs/uk-calibration-v3-fixture/events.ndjson", + "progress": "runs/uk-calibration-v3-fixture/progress.json" + }, + "pipeline": { + "id": "uk_national_calibration", + "version": "2026.09" + }, + "release_id": "populace-uk-2023-v3-fixture", + "run_id": "uk-calibration-v3-fixture", + "run_kind": "calibration", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 3, + "started_at": "2026-01-03T00:00:00+00:00", + "status": "completed", + "updated_at": "2026-01-03T00:00:04+00:00" +} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/events.ndjson b/frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/events.ndjson new file mode 100644 index 00000000..13336def --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/events.ndjson @@ -0,0 +1,8 @@ +{"details":{},"event_type":"stage","message":"Staging run created.","run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":1,"stage_id":"created","status":"started","timestamp":"2026-01-02T00:00:00+00:00"} +{"details":{"input_count":3},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":2,"stage_id":"input_verification","status":"completed","timestamp":"2026-01-02T00:00:01+00:00"} +{"details":{"household_rows":9},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":3,"stage_id":"sampling","status":"completed","timestamp":"2026-01-02T00:00:02+00:00"} +{"details":{"produced_columns":12},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":4,"stage_id":"construction","status":"completed","timestamp":"2026-01-02T00:00:03+00:00"} +{"details":{"household_rows":9},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":5,"stage_id":"validation","status":"completed","timestamp":"2026-01-02T00:00:04+00:00"} +{"details":{"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":6,"stage_id":"spine_h5_creation","status":"completed","timestamp":"2026-01-02T00:00:05+00:00"} +{"details":{"sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"event_type":"stage","message":null,"run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":7,"stage_id":"sidecar_creation","status":"completed","timestamp":"2026-01-02T00:00:06+00:00"} +{"details":{},"event_type":"stage","message":"Staging run completed.","run_id":"uk-spine-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":8,"stage_id":"complete","status":"completed","timestamp":"2026-01-02T00:00:07+00:00"} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/progress.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/progress.json new file mode 100644 index 00000000..073aea4c --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/progress.json @@ -0,0 +1,38 @@ +{ + "block": null, + "candidate_id": "uk-spine-v3-fixture", + "country_code": "GB", + "current_stage": "complete", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-spine-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "details": {}, + "failure": null, + "message": "Staging run completed.", + "non_release": true, + "operation_id": "uk_frs_spine", + "pipeline": { + "id": "uk_national_spine", + "version": "2026.09" + }, + "release_id": null, + "run_id": "uk-spine-v3-fixture", + "run_kind": "smoke", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.progress", + "schema_version": 3, + "started_at": "2026-01-02T00:00:00+00:00", + "status": "completed", + "updated_at": "2026-01-02T00:00:07+00:00" +} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/run_manifest.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/run_manifest.json new file mode 100644 index 00000000..e630819b --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/completed-spine/runs/uk-spine-v3-fixture/run_manifest.json @@ -0,0 +1,42 @@ +{ + "artifacts": [], + "block": null, + "candidate_id": "uk-spine-v3-fixture", + "country_code": "GB", + "current_stage": "complete", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-spine-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "failure": null, + "non_release": true, + "operation_id": "uk_frs_spine", + "paths": { + "calibration_progress": null, + "events": "runs/uk-spine-v3-fixture/events.ndjson", + "progress": "runs/uk-spine-v3-fixture/progress.json" + }, + "pipeline": { + "id": "uk_national_spine", + "version": "2026.09" + }, + "release_id": null, + "run_id": "uk-spine-v3-fixture", + "run_kind": "smoke", + "sample": { + "mode": "full" + }, + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 3, + "started_at": "2026-01-02T00:00:00+00:00", + "status": "completed", + "updated_at": "2026-01-02T00:00:07+00:00" +} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/contract-cases.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/contract-cases.json new file mode 100644 index 00000000..af85dc8b --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/contract-cases.json @@ -0,0 +1,44 @@ +{ + "deliberate_opt_out": { + "configured_repository": null, + "contract_version": 2, + "enabled": false, + "last_error_code": null, + "mode": "disabled", + "opt_out_reason": "--no-staging", + "read_back": "not_requested", + "run_id": null, + "upload_attempts": 0, + "upload_successes": 0 + }, + "delivery_failure": { + "configured_repository": "policyengine/populace-uk-staging", + "contract_version": 2, + "enabled": true, + "last_error_code": "READ_BACK_FAILED", + "mode": "local_and_remote", + "opt_out_reason": null, + "read_back": "failed", + "run_id": "uk-delivery-failure", + "upload_attempts": 3, + "upload_successes": 0 + }, + "delivery_success": { + "configured_repository": "policyengine/populace-uk-staging", + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_and_remote", + "opt_out_reason": null, + "read_back": "passed", + "run_id": "uk-delivery-success", + "upload_attempts": 6, + "upload_successes": 6 + }, + "schema_name": "microcosm.staging.fixture-cases", + "schema_version": 3, + "unknown_version": { + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 999 + } +} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/events.ndjson b/frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/events.ndjson new file mode 100644 index 00000000..a17c2837 --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/events.ndjson @@ -0,0 +1,3 @@ +{"details":{},"event_type":"stage","message":"Staging run created.","run_id":"uk-failed-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":1,"stage_id":"created","status":"started","timestamp":"2026-01-04T00:00:00+00:00"} +{"details":{},"event_type":"stage","message":null,"run_id":"uk-failed-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":2,"stage_id":"input_verification","status":"started","timestamp":"2026-01-04T00:00:01+00:00"} +{"details":{"error_code":"BUILD_FAILED","error_type":"RuntimeError","failure_class":"error"},"event_type":"stage","message":"The build failed during input_verification.","run_id":"uk-failed-v3-fixture","schema_name":"microcosm.staging.event","schema_version":3,"sequence":3,"stage_id":"failed","status":"failed","timestamp":"2026-01-04T00:00:02+00:00"} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/progress.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/progress.json new file mode 100644 index 00000000..a33052ed --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/progress.json @@ -0,0 +1,42 @@ +{ + "block": null, + "candidate_id": "uk-failed-v3-fixture", + "country_code": "GB", + "current_stage": "failed", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-failed-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "details": {}, + "failure": { + "error_code": "BUILD_FAILED", + "error_type": "RuntimeError", + "failure_class": "error", + "local_diagnostic_reference": "diagnostics/operator-error.txt", + "message": "The build failed during input_verification." + }, + "message": "The build failed during input_verification.", + "non_release": true, + "operation_id": "uk_frs_spine", + "pipeline": { + "id": "uk_national_spine", + "version": "2026.09" + }, + "release_id": null, + "run_id": "uk-failed-v3-fixture", + "run_kind": "smoke", + "sample": null, + "schema_name": "microcosm.staging.progress", + "schema_version": 3, + "started_at": "2026-01-04T00:00:00+00:00", + "status": "failed", + "updated_at": "2026-01-04T00:00:02+00:00" +} diff --git a/frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/run_manifest.json b/frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/run_manifest.json new file mode 100644 index 00000000..285d33f8 --- /dev/null +++ b/frontend/lib/microcosm/fixtures/staging-contract/v3/failed/runs/uk-failed-v3-fixture/run_manifest.json @@ -0,0 +1,46 @@ +{ + "artifacts": [], + "block": null, + "candidate_id": "uk-failed-v3-fixture", + "country_code": "GB", + "current_stage": "failed", + "delivery": { + "configured_repository": null, + "contract_version": 2, + "enabled": true, + "last_error_code": null, + "mode": "local_only", + "opt_out_reason": null, + "read_back": "not_requested", + "run_id": "uk-failed-v3-fixture", + "upload_attempts": 0, + "upload_successes": 0 + }, + "failure": { + "error_code": "BUILD_FAILED", + "error_type": "RuntimeError", + "failure_class": "error", + "local_diagnostic_reference": "diagnostics/operator-error.txt", + "message": "The build failed during input_verification." + }, + "non_release": true, + "operation_id": "uk_frs_spine", + "paths": { + "calibration_progress": null, + "events": "runs/uk-failed-v3-fixture/events.ndjson", + "progress": "runs/uk-failed-v3-fixture/progress.json" + }, + "pipeline": { + "id": "uk_national_spine", + "version": "2026.09" + }, + "release_id": null, + "run_id": "uk-failed-v3-fixture", + "run_kind": "smoke", + "sample": null, + "schema_name": "microcosm.staging.run-manifest", + "schema_version": 3, + "started_at": "2026-01-04T00:00:00+00:00", + "status": "failed", + "updated_at": "2026-01-04T00:00:02+00:00" +} diff --git a/frontend/lib/microcosm/staging-artifact.test.ts b/frontend/lib/microcosm/staging-artifact.test.ts index 1d655683..2cd11d41 100644 --- a/frontend/lib/microcosm/staging-artifact.test.ts +++ b/frontend/lib/microcosm/staging-artifact.test.ts @@ -163,6 +163,20 @@ function v2RunManifest( }; } +// Version 3 adds `block`, required on every run document (null unless blocked). +function v3RunManifest( + runId: string, + updatedAt: string, + overrides: Record = {}, +) { + return { + ...v2RunManifest(runId, updatedAt), + schema_version: 3, + block: null, + ...overrides, + }; +} + function calibrationDiagnostics() { return { schema_version: 6, @@ -341,6 +355,51 @@ test("rejects version 2 calibration diagnostics whose bytes do not match the dec ); }); +test("a version 3 run reads its declared diagnostics and checks their digest", async () => { + const diagnostics = serializedDiagnostics(); + const declared = (runId: string, sha256: string) => + v3RunManifest(runId, "2026-01-03T00:00:03+00:00", { + operation_id: "uk_national_calibration", + run_kind: "calibration", + artifacts: [ + { + logical_name: "calibration_diagnostics", + artifact_kind: "aggregate_diagnostics", + contract_relative_path: "artifacts/calibration_diagnostics.json", + media_type: "application/json", + sha256, + classification: "aggregate", + }, + ], + }); + const manifests = new Map([ + ["uk-v3-diagnostics", declared("uk-v3-diagnostics", diagnostics.sha256)], + ["uk-v3-digest-mismatch", declared("uk-v3-digest-mismatch", "0".repeat(64))], + ]); + const requested: string[] = []; + globalThis.fetch = (async (input) => { + const url = String(input); + requested.push(url); + const manifest = /\/runs\/([^/]+)\/run_manifest\.json$/.exec(url); + if (manifest && manifests.has(manifest[1])) { + return Response.json(manifests.get(manifest[1])); + } + if (url.endsWith("/artifacts/calibration_diagnostics.json")) { + return new Response(diagnostics.body); + } + return new Response(null, { status: 404 }); + }) as typeof fetch; + + const calibration = await loadStagingCalibration("uk-v3-diagnostics", 0, "uk"); + expect(calibration?.rows).toHaveLength(1); + expect( + requested.some((url) => url.endsWith("/runs/uk-v3-diagnostics/calibration_diagnostics.json")), + ).toBe(false); + await expect( + loadStagingCalibration("uk-v3-digest-mismatch", 0, "uk"), + ).rejects.toThrow(/digest does not match its run manifest declaration/); +}); + test("keeps the version 1 calibration diagnostics path", async () => { const runId = "legacy-calibration"; globalThis.fetch = (async (input) => { @@ -555,6 +614,52 @@ test("discovers and orders version 2 runs from paginated manifests only", async expect(requested.some((url) => url.includes("/directory/"))).toBe(false); }); +test("lists version 3 runs, blocked ones included, beside version 2 runs", async () => { + const manifests = new Map>([ + ["older-v2", v2RunManifest("older-v2", "2026-01-02T00:00:00+00:00")], + [ + "blocked-v3", + v3RunManifest("blocked-v3", "2026-01-03T00:00:00+00:00", { + status: "blocked", + current_stage: "blocked", + block: { + phase: "preflight", + blocking_failure_count: 1, + blocking_gate_ids: ["uk_target_fit"], + }, + }), + ], + ]); + const requested: string[] = []; + globalThis.fetch = (async (input) => { + const url = String(input); + requested.push(url); + if (url.includes("/api/datasets/")) { + return Response.json([ + { type: "file", path: "runs/older-v2/run_manifest.json" }, + { type: "file", path: "runs/blocked-v3/run_manifest.json" }, + ]); + } + const match = /\/runs\/([^/]+)\/run_manifest\.json$/.exec(url); + if (match && manifests.has(match[1])) { + return Response.json(manifests.get(match[1])); + } + return new Response(null, { status: 404 }); + }) as typeof fetch; + + const result = await loadStagingRuns(0, "uk"); + + expect(result.incompatible_runs).toEqual([]); + expect( + result.runs.map((run) => [run.run_id, run.schema_version, run.status, run.stage]), + ).toEqual([ + ["blocked-v3", 3, "blocked", "blocked"], + ["older-v2", 2, "completed", "complete"], + ]); + // Structured runs never fall back to the version 1 run index. + expect(requested.some((url) => url.endsWith("/runs.json"))).toBe(false); +}); + test("reports a version 2 manifest whose run id differs from its path", async () => { globalThis.fetch = (async (input) => { const url = String(input); @@ -697,6 +802,7 @@ test("target-change cache duration follows whether a run can still change", () = expect(stagingTargetChangeCacheTtlSeconds("published")).toBe(21_600); expect(stagingTargetChangeCacheTtlSeconds("completed")).toBe(21_600); expect(stagingTargetChangeCacheTtlSeconds("failed")).toBe(21_600); + expect(stagingTargetChangeCacheTtlSeconds("blocked")).toBe(21_600); expect(stagingTargetChangeCacheTtlSeconds("running")).toBe(30); expect(stagingTargetChangeCacheTtlSeconds("stalled")).toBe(30); expect(stagingTargetChangeCacheTtlSeconds(null)).toBe(30); diff --git a/frontend/lib/microcosm/staging-artifact.ts b/frontend/lib/microcosm/staging-artifact.ts index 8d35c49c..84163369 100644 --- a/frontend/lib/microcosm/staging-artifact.ts +++ b/frontend/lib/microcosm/staging-artifact.ts @@ -33,12 +33,14 @@ import { buildTargetChangeDatasetFromSummaryAndCalibration } from "@/lib/microco import { loadCalibrationComparisonSource } from "@/lib/microcosm/calibration-comparison-blob"; import { IncompatibleStagingDataError, + isStructuredStagingDocument, parseStagingCalibrationProgress, parseStagingEvents, parseStagingManifest, parseStagingProgress, parseStagingRunIndex, validateStagingRunConsistency, + type StagingSchemaVersion, } from "@/lib/microcosm/staging-contract"; type JsonObject = Record; @@ -338,7 +340,7 @@ export interface StagingRunSummary { country_code: string | null; run_kind: string | null; non_release: boolean | null; - schema_version: 1 | 2 | null; + schema_version: StagingSchemaVersion | null; status: string | null; stage: string | null; started_at: string | null; @@ -364,7 +366,7 @@ export interface StagingRunDetail { country_code: string | null; run_kind: string | null; non_release: boolean | null; - schema_version: 1 | 2 | null; + schema_version: StagingSchemaVersion | null; delivery: JsonObject | null; progress: JsonObject | null; run_manifest: JsonObject | null; @@ -385,8 +387,8 @@ function booleanValue(value: unknown): boolean | null { return typeof value === "boolean" ? value : null; } -function schemaVersionValue(value: unknown): 1 | 2 | null { - return value === 1 || value === 2 ? value : null; +function schemaVersionValue(value: unknown): StagingSchemaVersion | null { + return value === 1 || value === 2 || value === 3 ? value : null; } function objectOrNull(value: unknown): JsonObject | null { @@ -396,7 +398,7 @@ function objectOrNull(value: unknown): JsonObject | null { } export function stagingTargetChangeCacheTtlSeconds(status: unknown): number { - return ["passed", "published", "completed", "failed"].includes( + return ["passed", "published", "completed", "blocked", "failed"].includes( String(status ?? "").trim(), ) ? TARGET_CHANGE_FINAL_CACHE_SECONDS @@ -433,7 +435,7 @@ function summaryFromManifest( country_code: stringValue(manifest.country_code), run_kind: stringValue(manifest.run_kind), non_release: booleanValue(manifest.non_release), - schema_version: 2, + schema_version: schemaVersionValue(manifest.schema_version), status: stringValue(manifest.status), stage: stringValue(manifest.stage), started_at: stringValue(manifest.started_at), @@ -533,11 +535,9 @@ export async function loadStagingRuns( const incompatibleRuns = manifestResults.flatMap(({ problem }) => problem == null ? [] : [problem], ); - const v2Manifests = listedManifests.filter( - (manifest) => manifest.schema_version === 2, - ); - const v2ManifestIds = new Set( - v2Manifests.map((manifest) => String(manifest.run_id)), + const structuredManifests = listedManifests.filter(isStructuredStagingDocument); + const structuredManifestIds = new Set( + structuredManifests.map((manifest) => String(manifest.run_id)), ); const v1ManifestIds = new Set( listedManifests @@ -586,7 +586,7 @@ export async function loadStagingRuns( const v1RunIds = new Set(v1ManifestIds); if (index?.schema_version === 1) { for (const runId of listedRunIds) { - if (!v2ManifestIds.has(runId)) v1RunIds.add(runId); + if (!structuredManifestIds.has(runId)) v1RunIds.add(runId); } } const missing = [...v1RunIds] @@ -607,7 +607,7 @@ export async function loadStagingRuns( }), ); for (const run of fetched) byId.set(run.run_id, run); - for (const manifest of v2Manifests) { + for (const manifest of structuredManifests) { const runId = String(manifest.run_id); byId.set(runId, summaryFromManifest(runId, manifest)); } @@ -853,10 +853,10 @@ export async function resolveStagingCalibrationSource( runId; const artifacts = objectOrNull(runManifest?.artifacts); const diagnosticsArtifact = objectOrNull(artifacts?.calibration_diagnostics); - const diagnosticsPath = - runManifest?.schema_version === 2 - ? stringValue(diagnosticsArtifact?.staging_path) - : `runs/${runId}/calibration_diagnostics.json`; + const structured = isStructuredStagingDocument(runManifest); + const diagnosticsPath = structured + ? stringValue(diagnosticsArtifact?.staging_path) + : `runs/${runId}/calibration_diagnostics.json`; if (!diagnosticsPath) { // No telemetry diagnostics artifact: a run that staged its dataset bundle // carries the same diagnostics there. @@ -879,7 +879,7 @@ export async function resolveStagingCalibrationSource( staging.revision, ); if (!diagnostics) { - if (runManifest?.schema_version === 2 && diagnosticsArtifact) { + if (structured && diagnosticsArtifact) { throw new IncompatibleStagingDataError( "artifact calibration_diagnostics is declared but absent.", ); @@ -887,10 +887,7 @@ export async function resolveStagingCalibrationSource( return null; } const expectedDigest = stringValue(diagnosticsArtifact?.sha256); - if ( - runManifest?.schema_version === 2 && - diagnostics.sha256 !== expectedDigest - ) { + if (structured && diagnostics.sha256 !== expectedDigest) { throw new IncompatibleStagingDataError( `artifact calibration_diagnostics digest does not match its run manifest declaration.`, ); diff --git a/frontend/lib/microcosm/staging-contract.test.ts b/frontend/lib/microcosm/staging-contract.test.ts index bc821da4..accc7d73 100644 --- a/frontend/lib/microcosm/staging-contract.test.ts +++ b/frontend/lib/microcosm/staging-contract.test.ts @@ -15,20 +15,25 @@ import { } from "./staging-contract"; const FIXTURES = join(import.meta.dir, "fixtures", "staging-contract"); +// Version 3 is copied byte for byte from microcosm +// packages/microcosm-build/tests/fixtures/staging/v3. const MANIFEST_DIGESTS = { v1: "165d24caf29b82afdb0ce241b65d088da552abd59d6ddabf4b2183b9a61be75b", v2: "0d54c630088152b4fb4a7b7a9853c851233d0be76579f949331ae4bcd7b455e5", + v3: "e31a5e337840e24d7a1203b83eb1a5039eeb5f35777a895ab6ab28163bf3def5", }; -function bytes(version: "v1" | "v2", path: string): Buffer { +type FixtureVersion = keyof typeof MANIFEST_DIGESTS; + +function bytes(version: FixtureVersion, path: string): Buffer { return readFileSync(join(FIXTURES, version, path)); } -function json(version: "v1" | "v2", path: string): Record { +function json(version: FixtureVersion, path: string): Record { return JSON.parse(bytes(version, path).toString()); } -for (const version of ["v1", "v2"] as const) { +for (const version of ["v1", "v2", "v3"] as const) { test(`${version} fixture files match the Microcosm digest manifest`, () => { const manifest = bytes(version, "SHA256SUMS"); expect(createHash("sha256").update(manifest).digest("hex")).toBe( @@ -378,6 +383,194 @@ describe("version 2", () => { }); }); +describe("version 3", () => { + const blockedRoot = "blocked/runs/uk-blocked-v3-fixture"; + const failedRoot = "failed/runs/uk-failed-v3-fixture"; + const runs = [ + ["uk-spine-v3-fixture", "completed-spine/runs/uk-spine-v3-fixture"], + ["uk-calibration-v3-fixture", "calibration/runs/uk-calibration-v3-fixture"], + ["uk-failed-v3-fixture", failedRoot], + ["uk-blocked-v3-fixture", blockedRoot], + ] as const; + + test("reads every fixture run as one consistent version 3 run", () => { + for (const [runId, root] of runs) { + const manifest = parseStagingManifest(json("v3", `${root}/run_manifest.json`)); + const paths = manifest.paths as Record; + expect(() => + validateStagingRunConsistency(runId, { + progress: parseStagingProgress(json("v3", `${root}/progress.json`)), + runManifest: manifest, + calibrationProgress: paths.calibration_progress + ? parseStagingCalibrationProgress( + json("v3", `${root}/calibration_progress.json`), + ) + : null, + events: parseStagingEvents(bytes("v3", `${root}/events.ndjson`).toString()), + }), + ).not.toThrow(); + expect(manifest.schema_version).toBe(3); + } + }); + + test("a blocked run carries its block and ends with a blocked event", () => { + const block = { + phase: "terminal", + blocking_failure_count: 1, + blocking_gate_ids: ["uk_target_fit"], + }; + const manifest = parseStagingManifest(json("v3", `${blockedRoot}/run_manifest.json`)); + const progress = parseStagingProgress(json("v3", `${blockedRoot}/progress.json`)); + const events = parseStagingEvents( + bytes("v3", `${blockedRoot}/events.ndjson`).toString(), + ); + + expect(manifest).toMatchObject({ status: "blocked", failure: null, block }); + expect(progress).toMatchObject({ status: "blocked", stage: "blocked", failure: null, block }); + expect(events[events.length - 1]).toMatchObject({ + stage: "blocked", + status: "blocked", + details: { ...block, gate_statuses: { uk_target_fit: "failed" } }, + }); + }); + + test("a failure carries its class, and a block may name no gate", () => { + const failed = parseStagingProgress(json("v3", `${failedRoot}/progress.json`)); + expect(failed.failure).toMatchObject({ + error_code: "BUILD_FAILED", + failure_class: "error", + }); + expect(failed.block).toBeNull(); + + const source = json("v3", `${blockedRoot}/progress.json`); + const unnamed = parseStagingProgress({ + ...source, + block: { phase: "preflight", blocking_failure_count: 2, blocking_gate_ids: [] }, + }); + expect(unnamed.block).toEqual({ + phase: "preflight", + blocking_failure_count: 2, + blocking_gate_ids: [], + }); + expect( + parseStagingProgress({ + ...json("v3", `${failedRoot}/progress.json`), + failure: { + ...(failed.failure as Record), + failure_class: null, + }, + }).failure, + ).toMatchObject({ failure_class: null }); + }); + + test("rejects failures and blocks that break the version 3 rules", () => { + const failedSource = json("v3", `${failedRoot}/progress.json`); + const failure = failedSource.failure as Record; + const blockedSource = json("v3", `${blockedRoot}/progress.json`); + const block = blockedSource.block as Record; + const withoutBlock = { ...blockedSource }; + delete withoutBlock.block; + const { failure_class: _class, ...version2Failure } = failure; + + const cases: [Record, RegExp][] = [ + [ + { ...failedSource, failure: { ...failure, traceback: "File \"x.py\"" } }, + /version 3 failure contains unexpected field traceback/, + ], + [{ ...failedSource, failure: version2Failure }, /version 3 failure is missing failure_class/], + [ + { ...failedSource, failure: { ...failure, failure_class: "OutOfMemory" } }, + /failure_class is invalid/, + ], + [{ ...failedSource, block }, /non-blocked status cannot contain block data/], + [{ ...blockedSource, block: null }, /blocked status requires block data/], + [ + { ...blockedSource, failure, current_stage: "blocked" }, + /non-failed status cannot contain failure data/, + ], + [withoutBlock, /version 3 progress is missing block/], + [ + { ...blockedSource, block: { ...block, blocking_failure_count: 0 } }, + /blocking_failure_count must be an integer greater than or equal to 1/, + ], + [ + { ...blockedSource, block: { ...block, blocking_gate_ids: [""] } }, + /blocking_gate_ids 0 must be a string between 1 and 200 characters/, + ], + [ + { ...blockedSource, block: { ...block, gate_statuses: {} } }, + /version 3 block contains unexpected field gate_statuses/, + ], + [ + { ...blockedSource, block: { ...block, phase: "pre flight" } }, + /block phase must be a safe identifier/, + ], + ]; + for (const [payload, error] of cases) { + expect(() => parseStagingProgress(payload)).toThrow(error); + } + expect(() => + parseStagingManifest({ + ...json("v3", `${failedRoot}/run_manifest.json`), + block, + }), + ).toThrow(/non-blocked status cannot contain block data/); + }); + + test("version 2 documents do not accept version 3 fields", () => { + const root = "completed-spine/runs/uk-spine-v2-fixture"; + const progress = json("v2", `${root}/progress.json`); + const failedProgress = json("v2", "failed/runs/uk-failed-v2-fixture/progress.json"); + const event = JSON.parse( + bytes("v2", `${root}/events.ndjson`).toString().trim().split("\n")[0], + ) as Record; + + expect(() => parseStagingProgress({ ...progress, block: null })).toThrow( + /version 2 progress contains unexpected field block/, + ); + expect(() => + parseStagingProgress({ ...progress, status: "blocked", current_stage: "blocked" }), + ).toThrow(/progress status is unsupported/); + expect(() => + parseStagingProgress({ + ...failedProgress, + failure: { + ...(failedProgress.failure as Record), + failure_class: "error", + }, + }), + ).toThrow(/version 2 failure contains unexpected field failure_class/); + expect(() => + parseStagingEvents(JSON.stringify({ ...event, status: "blocked" })), + ).toThrow(/status is unsupported/); + }); + + test("events stay contiguous and a run cannot mix versions 2 and 3", () => { + const lines = bytes("v3", `${blockedRoot}/events.ndjson`).toString().trim().split("\n"); + expect(() => parseStagingEvents([lines[0], lines[2]].join("\n"))).toThrow( + /version 3 event sequence is not contiguous/, + ); + + const v2Root = "completed-spine/runs/uk-spine-v2-fixture"; + const v3Root = "completed-spine/runs/uk-spine-v3-fixture"; + const v2Progress = parseStagingProgress({ + ...json("v2", `${v2Root}/progress.json`), + run_id: "uk-spine-v3-fixture", + candidate_id: "uk-spine-v3-fixture", + delivery: { + ...(json("v2", `${v2Root}/progress.json`).delivery as Record), + run_id: "uk-spine-v3-fixture", + }, + }); + expect(() => + validateStagingRunConsistency("uk-spine-v3-fixture", { + progress: v2Progress, + runManifest: parseStagingManifest(json("v3", `${v3Root}/run_manifest.json`)), + }), + ).toThrow(/a run cannot mix document versions 2 and 3/); + }); +}); + test("version 1 failure tracebacks are not served", () => { const failed = { time: "2026-09-26T13:48:00+00:00", diff --git a/frontend/lib/microcosm/staging-contract.ts b/frontend/lib/microcosm/staging-contract.ts index 3134c920..f5aae38e 100644 --- a/frontend/lib/microcosm/staging-contract.ts +++ b/frontend/lib/microcosm/staging-contract.ts @@ -9,7 +9,9 @@ export class IncompatibleStagingDataError extends Error { } } -const V2_SCHEMAS = { +// Versions 2 and 3 share these names. Version 3 adds the `blocked` run status +// with its `block` details, and `failure_class` on failures. +const STRUCTURED_SCHEMAS = { manifest: "microcosm.staging.run-manifest", progress: "microcosm.staging.progress", calibration: "microcosm.staging.calibration-progress", @@ -173,11 +175,21 @@ function nullableBoundedString( return value === null ? null : boundedString(value, label, options); } +export type StagingSchemaVersion = 1 | 2 | 3; + +// Versions 2 and later are the structured contract; version 1 is the US +// transition layout. +export function isStructuredStagingDocument( + document: JsonObject | null | undefined, +): boolean { + return typeof document?.schema_version === "number" && document.schema_version >= 2; +} + function version( payload: JsonObject, - kind: keyof typeof V2_SCHEMAS, + kind: keyof typeof STRUCTURED_SCHEMAS, options: { allowUnversionedV1?: boolean } = {}, -): 1 | 2 { +): StagingSchemaVersion { if (payload.schema_version === 1) { if (payload.schema_name != null) { throw new IncompatibleStagingDataError( @@ -186,13 +198,13 @@ function version( } return 1; } - if (payload.schema_version === 2) { - if (payload.schema_name !== V2_SCHEMAS[kind]) { + if (payload.schema_version === 2 || payload.schema_version === 3) { + if (payload.schema_name !== STRUCTURED_SCHEMAS[kind]) { throw new IncompatibleStagingDataError( - `expected ${V2_SCHEMAS[kind]} version 2, received ${String(payload.schema_name)}.`, + `expected ${STRUCTURED_SCHEMAS[kind]} version ${payload.schema_version}, received ${String(payload.schema_name)}.`, ); } - return 2; + return payload.schema_version; } if (options.allowUnversionedV1 && payload.schema_version == null && payload.schema_name == null) { return 1; @@ -394,15 +406,22 @@ function parseDelivery(value: unknown, envelopeRunId: string): JsonObject { return delivery; } -function parseFailure(value: unknown): JsonObject | null { +const FAILURE_FIELDS = [ + "error_code", + "error_type", + "message", + "local_diagnostic_reference", +] as const; + +function parseFailure(value: unknown, schemaVersion: 2 | 3): JsonObject | null { if (value === null) return null; - const failure = objectValue(value, "version 2 failure"); - assertExactKeys(failure, "version 2 failure", [ - "error_code", - "error_type", - "message", - "local_diagnostic_reference", - ]); + const label = `version ${schemaVersion} failure`; + const failure = objectValue(value, label); + assertExactKeys( + failure, + label, + schemaVersion === 3 ? [...FAILURE_FIELDS, "failure_class"] : FAILURE_FIELDS, + ); if (typeof failure.error_code !== "string" || !/^[A-Z0-9_]+$/.test(failure.error_code)) { throw new IncompatibleStagingDataError("failure error_code is invalid."); } @@ -418,15 +437,52 @@ function parseFailure(value: unknown): JsonObject | null { "failure local_diagnostic_reference", { maximum: 200 }, ); + if ( + schemaVersion === 3 && + failure.failure_class !== null && + (typeof failure.failure_class !== "string" || + !/^[a-z][a-z0-9_]*$/.test(failure.failure_class)) + ) { + throw new IncompatibleStagingDataError("failure failure_class is invalid."); + } return failure; } -function parseRunEnvelope(payload: JsonObject, label: string): { +// Version 3: which gate phase refused the candidate, and which gates. The gate +// ids may be empty when the refusal named no gate; the count is at least one. +function parseBlock(value: unknown): JsonObject | null { + if (value === null) return null; + const block = objectValue(value, "version 3 block"); + assertExactKeys(block, "version 3 block", [ + "phase", + "blocking_failure_count", + "blocking_gate_ids", + ]); + safeIdentifier(block.phase, "block phase"); + integerValue(block.blocking_failure_count, "block blocking_failure_count", 1); + if (!Array.isArray(block.blocking_gate_ids)) { + throw new IncompatibleStagingDataError("block blocking_gate_ids must be an array."); + } + for (const [index, gateId] of block.blocking_gate_ids.entries()) { + boundedString(gateId, `block blocking_gate_ids ${index}`, { + minimum: 1, + maximum: 200, + }); + } + return block; +} + +function parseRunEnvelope( + payload: JsonObject, + label: string, + schemaVersion: 2 | 3, +): { runId: string; candidateId: string; releaseId: string | null; pipeline: JsonObject; failure: JsonObject | null; + block: JsonObject | null; delivery: JsonObject; } { const runId = safeIdentifier(payload.run_id, `${label} run_id`); @@ -446,21 +502,30 @@ function parseRunEnvelope(payload: JsonObject, label: string): { } dateTimeValue(payload.started_at, `${label} started_at`); dateTimeValue(payload.updated_at, `${label} updated_at`); - const status = enumValue(payload.status, `${label} status`, [ - "running", - "completed", - "failed", - ] as const); + const status = enumValue( + payload.status, + `${label} status`, + schemaVersion === 3 + ? (["running", "completed", "blocked", "failed"] as const) + : (["running", "completed", "failed"] as const), + ); safeIdentifier(payload.current_stage, `${label} current_stage`); const delivery = parseDelivery(payload.delivery, runId); - const failure = parseFailure(payload.failure); + const failure = parseFailure(payload.failure, schemaVersion); if (status === "failed" && failure == null) { throw new IncompatibleStagingDataError(`${label} failed status requires failure data.`); } if (status !== "failed" && failure != null) { throw new IncompatibleStagingDataError(`${label} non-failed status cannot contain failure data.`); } - return { runId, candidateId, releaseId, pipeline, failure, delivery }; + const block = schemaVersion === 3 ? parseBlock(payload.block) : null; + if (status === "blocked" && block == null) { + throw new IncompatibleStagingDataError(`${label} blocked status requires block data.`); + } + if (status !== "blocked" && block != null) { + throw new IncompatibleStagingDataError(`${label} non-blocked status cannot contain block data.`); + } + return { runId, candidateId, releaseId, pipeline, failure, block, delivery }; } // Version 1 telemetry has no content policy. Its failure path records the @@ -482,7 +547,7 @@ export function parseStagingProgress(value: unknown): JsonObject { const runId = stringValue(payload.run_id, "progress run_id"); return { ...withoutTraceback(payload), run_id: runId }; } - assertExactKeys(payload, "version 2 progress", [ + assertExactKeys(payload, `version ${schemaVersion} progress`, [ "schema_name", "schema_version", ...RUN_FIELDS, @@ -491,8 +556,9 @@ export function parseStagingProgress(value: unknown): JsonObject { "message", "details", "failure", + ...(schemaVersion === 3 ? ["block"] : []), ]); - const envelope = parseRunEnvelope(payload, "progress"); + const envelope = parseRunEnvelope(payload, "progress", schemaVersion); const sample = parseSample(payload.sample); nullableBoundedString(payload.message, "progress message", { maximum: 500 }); objectValue(payload.details, "progress details"); @@ -504,6 +570,7 @@ export function parseStagingProgress(value: unknown): JsonObject { pipeline: envelope.pipeline, sample, failure: envelope.failure, + block: envelope.block, stage: payload.current_stage, delivery: envelope.delivery, }; @@ -563,7 +630,7 @@ export function parseStagingManifest(value: unknown): JsonObject { const runId = stringValue(payload.run_id, "run manifest run_id"); return { ...payload, run_id: runId }; } - assertExactKeys(payload, "version 2 run manifest", [ + assertExactKeys(payload, `version ${schemaVersion} run manifest`, [ "schema_name", "schema_version", ...RUN_FIELDS, @@ -571,9 +638,10 @@ export function parseStagingManifest(value: unknown): JsonObject { "delivery", "artifacts", "failure", + ...(schemaVersion === 3 ? ["block"] : []), "paths", ]); - const envelope = parseRunEnvelope(payload, "run manifest"); + const envelope = parseRunEnvelope(payload, "run manifest", schemaVersion); const sample = parseSample(payload.sample); const paths = objectValue(payload.paths, "run manifest paths"); assertExactKeys(paths, "run manifest paths", [ @@ -602,6 +670,7 @@ export function parseStagingManifest(value: unknown): JsonObject { pipeline: envelope.pipeline, sample, failure: envelope.failure, + block: envelope.block, stage: payload.current_stage, delivery: envelope.delivery, artifacts: parseArtifacts(payload.artifacts, envelope.runId), @@ -616,7 +685,7 @@ export function parseStagingCalibrationProgress(value: unknown): JsonObject { throw new IncompatibleStagingDataError("calibration events must be an array."); } if (schemaVersion === 1) return payload; - assertExactKeys(payload, "version 2 calibration progress", [ + assertExactKeys(payload, `version ${schemaVersion} calibration progress`, [ "schema_name", "schema_version", "run_id", @@ -708,12 +777,13 @@ export function parseStagingEvents(text: string | null): JsonObject[] { ] as const); safeIdentifier(payload.run_id, `event ${index} run_id`); const stageId = safeIdentifier(payload.stage_id, `event ${index} stage_id`); - enumValue(payload.status, `event ${index} status`, [ - "started", - "completed", - "failed", - "progress", - ] as const); + enumValue( + payload.status, + `event ${index} status`, + schemaVersion === 3 + ? (["started", "completed", "failed", "progress", "blocked"] as const) + : (["started", "completed", "failed", "progress"] as const), + ); nullableBoundedString(payload.message, `event ${index} message`, { maximum: 500 }); objectValue(payload.details, `event ${index} details`); return { @@ -722,12 +792,14 @@ export function parseStagingEvents(text: string | null): JsonObject[] { time: timestamp, }; }); - const v2 = events.filter((event) => event.schema_version === 2); + const structured = events.filter(isStructuredStagingDocument); if ( - v2.length && - v2.some((event, index) => event.sequence !== index + 1) + structured.length && + structured.some((event, index) => event.sequence !== index + 1) ) { - throw new IncompatibleStagingDataError("version 2 event sequence is not contiguous."); + throw new IncompatibleStagingDataError( + `version ${String(structured[0].schema_version)} event sequence is not contiguous.`, + ); } return events; } @@ -765,12 +837,12 @@ export function validateStagingRunConsistency( const versions = new Set( allDocuments.map((document) => document.schema_version ?? 1), ); - if (versions.has(2) && versions.size !== 1) { + if (versions.size !== 1) { throw new IncompatibleStagingDataError( - "a run cannot mix version 1 and version 2 documents.", + `a run cannot mix document versions ${[...versions].sort().join(" and ")}.`, ); } - if (!versions.has(2)) return; + if (versions.has(1)) return; for (const [label, document] of [ ["progress", progress], From fc3820e8c377034177a60172852504919f672072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:36:29 +0100 Subject: [PATCH 2/4] Accept blocked run events in the telemetry collector A `run` / `blocked` event (or a `blocked` stage event) ends the run as `blocked` with `ended_at` and keeps the block details (phase, blocking failure count, gate ids) in the materialized documents; failed events keep `error_code` in `failure`. No migration: status is Text, and the block is rebuilt from the events on every read. Co-Authored-By: Claude Opus 5.5 --- .../telemetry_collector/models.py | 2 +- .../telemetry_collector/storage.py | 24 +++- telemetry-service/tests/test_app.py | 55 ++++++++ telemetry-service/tests/test_storage.py | 133 ++++++++++++++++++ 4 files changed, 212 insertions(+), 2 deletions(-) diff --git a/telemetry-service/telemetry_collector/models.py b/telemetry-service/telemetry_collector/models.py index 947d1b36..f5fbe61d 100644 --- a/telemetry-service/telemetry_collector/models.py +++ b/telemetry-service/telemetry_collector/models.py @@ -69,7 +69,7 @@ class TelemetryEvent(BaseModel): timestamp: datetime event_type: Literal["run", "stage", "progress", "calibration", "heartbeat"] stage_id: str | None = Field(default=None, pattern=SAFE_IDENTIFIER) - status: Literal["started", "progress", "completed", "failed"] + status: Literal["started", "progress", "completed", "failed", "blocked"] message: str | None = Field(default=None, max_length=500) details: dict[str, Any] = Field(default_factory=dict) resources: ResourceSnapshot | None = None diff --git a/telemetry-service/telemetry_collector/storage.py b/telemetry-service/telemetry_collector/storage.py index 3210ad1c..c0bd62a0 100644 --- a/telemetry-service/telemetry_collector/storage.py +++ b/telemetry-service/telemetry_collector/storage.py @@ -119,7 +119,7 @@ def _apply_event(run: dict[str, Any], event: dict[str, Any]) -> None: if event["event_type"] == "heartbeat": run["heartbeat_at"] = timestamp stage_id = event.get("stage_id") - if stage_id and stage_id not in {"complete", "failed"}: + if stage_id and stage_id not in {"complete", "failed", "blocked"}: run["current_stage"] = stage_id if event["event_type"] == "run" and event["status"] == "started": # A later producer represents a restarted attempt for the same stable @@ -128,6 +128,7 @@ def _apply_event(run: dict[str, Any], event: dict[str, Any]) -> None: run["status"] = "running" run["ended_at"] = None run["failure"] = None + run["block"] = None details = event.get("details") or {} if event["event_type"] == "progress" and { "done", @@ -155,9 +156,24 @@ def _apply_event(run: dict[str, Any], event: dict[str, Any]) -> None: run["failure"] = { "message": event.get("message"), "error_type": details.get("error_type"), + "error_code": details.get("error_code"), "failure_class": details.get("failure_class"), "failed_during": details.get("failed_during") or stage_id, } + run["block"] = None + elif event["status"] == "blocked" and ( + event["event_type"] == "run" or stage_id == "blocked" + ): + # The gates refused the candidate: the run ended without failing. + run["status"] = "blocked" + run["current_stage"] = "blocked" + run["ended_at"] = timestamp + run["failure"] = None + run["block"] = { + "phase": details.get("phase"), + "blocking_failure_count": details.get("blocking_failure_count"), + "blocking_gate_ids": details.get("blocking_gate_ids") or [], + } elif event["status"] == "completed" and ( event["event_type"] == "run" or stage_id == "complete" ): @@ -165,6 +181,7 @@ def _apply_event(run: dict[str, Any], event: dict[str, Any]) -> None: run["current_stage"] = "complete" run["ended_at"] = timestamp run["failure"] = None + run["block"] = None def materialize_run( @@ -183,6 +200,8 @@ def materialize_run( "resources": None, "work": None, "failure": None, + # Rebuilt from the events on every read; not a stored column. + "block": None, } active_producer_id: str | None = None for event in canonical_events(events): @@ -206,6 +225,7 @@ def materialize_run( "resources": None, "work": None, "failure": None, + "block": None, } ) active_producer_id = producer_id @@ -291,11 +311,13 @@ def run_documents( "resources": run.get("resources"), "work": run.get("work"), "failure": run.get("failure"), + "block": run.get("block"), } manifest = { **common, "identity": identity, "failure": run.get("failure"), + "block": run.get("block"), "delivery": { "mode": "collector", "upload_attempts": None, diff --git a/telemetry-service/tests/test_app.py b/telemetry-service/tests/test_app.py index 2e225d22..a01f9859 100644 --- a/telemetry-service/tests/test_app.py +++ b/telemetry-service/tests/test_app.py @@ -327,6 +327,61 @@ def test_failed_validation_event_does_not_finish_the_run() -> None: assert detail.json()["progress"]["status"] == "running" +def test_blocked_run_event_is_accepted_and_ends_the_run() -> None: + store = FakeTelemetryStore() + client = TestClient( + create_app( + settings=settings(), + store=store, + huggingface_authenticator=StubHuggingFaceAuthenticator(), + ) + ) + token = exchange(client) + register(client, token) + response = client.post( + "/v1/runs/route-a-1/events", + headers={"Authorization": f"Bearer {token}"}, + json={ + "events": [ + { + "schema_version": 1, + "event_id": "gates-refused", + "run_id": "route-a-1", + "producer_id": "producer-1", + "sequence": 1, + "timestamp": datetime.now(UTC).isoformat(), + "event_type": "run", + "stage_id": "blocked", + "status": "blocked", + "message": "The gates refused the candidate at terminal.", + "details": { + "phase": "terminal", + "blocking_failure_count": 1, + "blocking_gate_ids": ["uk_target_fit"], + "gate_statuses": {"uk_target_fit": "failed"}, + }, + "resources": None, + } + ] + }, + ) + headers = {"X-Telemetry-Read-Token": "dashboard-read-token"} + detail = client.get("/v1/runs/route-a-1", headers=headers) + listed = client.get("/v1/runs?country=US", headers=headers) + + assert response.status_code == 202 + assert response.json() == {"accepted": 1, "duplicates": 0} + progress = detail.json()["progress"] + assert progress["status"] == "blocked" + assert progress["block"] == { + "phase": "terminal", + "blocking_failure_count": 1, + "blocking_gate_ids": ["uk_target_fit"], + } + assert listed.json()["runs"][0]["status"] == "blocked" + assert listed.json()["runs"][0]["ended_at"] is not None + + def test_health_does_not_require_credentials() -> None: client = TestClient( create_app( diff --git a/telemetry-service/tests/test_storage.py b/telemetry-service/tests/test_storage.py index 9e69fcfd..6b5d4009 100644 --- a/telemetry-service/tests/test_storage.py +++ b/telemetry-service/tests/test_storage.py @@ -213,3 +213,136 @@ def test_progress_work_uses_json_compatible_timestamp() -> None: materialized = materialize_run(_registered(registered_at), [event]) assert materialized["work"]["updated_at"] == registered_at.isoformat() + + +def test_blocked_run_event_ends_the_run_with_its_block() -> None: + registered_at = datetime(2026, 10, 1, tzinfo=UTC) + blocked = _event( + event_id="blocked", + producer_registered_at=registered_at, + sequence=2, + timestamp=registered_at + timedelta(minutes=9), + status="blocked", + stage_id="blocked", + ) + blocked["details"] = { + "phase": "terminal", + "blocking_failure_count": 1, + "blocking_gate_ids": ["uk_target_fit"], + "gate_statuses": {"uk_target_fit": "failed"}, + } + events = [ + _event( + event_id="gates", + producer_registered_at=registered_at, + sequence=1, + timestamp=registered_at + timedelta(minutes=8), + status="completed", + stage_id="gate_battery", + ), + blocked, + ] + events[0]["event_type"] = "stage" + + materialized = materialize_run(_registered(registered_at), events) + documents = run_documents(_registered(registered_at), events) + + block = { + "phase": "terminal", + "blocking_failure_count": 1, + "blocking_gate_ids": ["uk_target_fit"], + } + assert materialized["status"] == "blocked" + assert materialized["current_stage"] == "blocked" + assert materialized["ended_at"] == blocked["timestamp"] + assert materialized["failure"] is None + assert materialized["block"] == block + assert documents["progress"]["status"] == "blocked" + assert documents["progress"]["block"] == block + assert documents["run_manifest"]["block"] == block + assert documents["events"][-1]["details"]["gate_statuses"] == { + "uk_target_fit": "failed" + } + + +def test_blocked_run_without_gate_ids_or_statuses_keeps_its_count() -> None: + registered_at = datetime(2026, 10, 1, tzinfo=UTC) + blocked = _event( + event_id="blocked", + producer_registered_at=registered_at, + sequence=1, + timestamp=registered_at + timedelta(minutes=1), + status="blocked", + stage_id="blocked", + ) + blocked["details"] = { + "phase": "preflight", + "blocking_failure_count": 2, + "blocking_gate_ids": [], + } + + materialized = materialize_run(_registered(registered_at), [blocked]) + + assert materialized["status"] == "blocked" + assert materialized["block"] == { + "phase": "preflight", + "blocking_failure_count": 2, + "blocking_gate_ids": [], + } + + +def test_failed_run_keeps_its_error_code() -> None: + registered_at = datetime(2026, 10, 1, tzinfo=UTC) + failed = _event( + event_id="failed", + producer_registered_at=registered_at, + sequence=1, + timestamp=registered_at + timedelta(minutes=1), + status="failed", + stage_id="failed", + ) + failed["details"] = { + "error_type": "KeyboardInterrupt", + "error_code": "INTERRUPTED", + "failure_class": "interrupted", + "failed_during": "calibrating", + } + + assert materialize_run(_registered(registered_at), [failed])["failure"] == { + "message": "failed", + "error_type": "KeyboardInterrupt", + "error_code": "INTERRUPTED", + "failure_class": "interrupted", + "failed_during": "calibrating", + } + + +def test_restarted_producer_clears_a_previous_block() -> None: + registered_at = datetime(2026, 10, 1, tzinfo=UTC) + blocked = _event( + event_id="blocked", + producer_registered_at=registered_at, + sequence=1, + timestamp=registered_at + timedelta(minutes=1), + status="blocked", + stage_id="blocked", + ) + blocked["details"] = { + "phase": "terminal", + "blocking_failure_count": 1, + "blocking_gate_ids": ["uk_target_fit"], + } + restarted = _event( + event_id="restart-started", + producer_id="producer-2", + producer_registered_at=registered_at + timedelta(minutes=10), + sequence=1, + timestamp=registered_at + timedelta(minutes=11), + status="started", + stage_id="created", + ) + + materialized = materialize_run(_registered(registered_at), [blocked, restarted]) + + assert materialized["status"] == "running" + assert materialized["block"] is None From d36ee7ed0505e76d6d3e1097418f06d8ac0ce2db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:30:50 +0100 Subject: [PATCH 3/4] Address review: reject blocked events without a block, document producer responses - Collector: a `blocked` event must carry a valid block (safe-identifier phase, blocking_failure_count of at least 1, blocking_gate_ids, possibly empty), or ingestion answers 422; a block of nulls is never stored. gate_statuses stays optional. - README: which responses a producer treats as final (403, 404, 409, 413, 422) and which it retries, including why an events 404 means the run disappeared after it registered. - Name the producer of the dry_run_refusal class (the US fiscal-refresh release builder). Co-Authored-By: Claude Opus 5.5 --- .../microcosm/build-monitor-view.tsx | 2 + telemetry-service/README.md | 24 ++++++ .../telemetry_collector/models.py | 29 +++++++ telemetry-service/tests/test_app.py | 86 +++++++++++++++++++ 4 files changed, 141 insertions(+) diff --git a/frontend/components/microcosm/build-monitor-view.tsx b/frontend/components/microcosm/build-monitor-view.tsx index 68e46934..d599037b 100644 --- a/frontend/components/microcosm/build-monitor-view.tsx +++ b/frontend/components/microcosm/build-monitor-view.tsx @@ -100,6 +100,8 @@ const FAILURE_CLASS_LABEL: Record = { error: "Error", build_failure: "Build failure", unexpected_process_exit: "Process exited unexpectedly", + // The US fiscal-refresh release refused a gates dry run before its stop point + // (microcosm tools/build_us_fiscal_refresh_release.py). dry_run_refusal: "Refused in a dry run", stopped_without_final_event: "Stopped without a final event", unclassified: "Failed, no class recorded", diff --git a/telemetry-service/README.md b/telemetry-service/README.md index 168f3c73..cca99251 100644 --- a/telemetry-service/README.md +++ b/telemetry-service/README.md @@ -20,6 +20,30 @@ server retrieves this credential from Google Secret Manager at runtime. Vercel authenticates using its OIDC identity; the credential is never returned to the browser or passed through GitHub Actions. +## Responses a producer acts on + +Microcosm's delivery service (from PolicyEngine/microcosm#1147) treats any 4xx +other than 401, 408 and 429 as the collector's final answer for a run: it stops +delivering that run, keeps its telemetry on the build machine and warns once. +A 401 on registration or events makes it exchange its token again (a 401 or +403 from the token exchange itself is final); 408, 429, 5xx and the +maintenance revision's 503 are retried. So the collector answers 4xx only when +sending the same request again cannot succeed: + +- `403`: the Hugging Face user is not a member of the organization (token + exchange), the run belongs to another user, the producer was not registered + for it, or an event names a different run than its path. +- `404` on `POST /v1/runs/{run_id}/events` ("Run is not registered"): the + delivery service sends events only after its registration returned 201, so + this means the run disappeared after it registered, for example a database + reset while a build was running. That build's remaining telemetry stays local. +- `409` on `POST /v1/runs`: the run id is already registered with different + metadata. +- `413`: the request body is larger than 1 MiB. +- `422`: the request does not match the event schema, including a `blocked` + event without a valid block (a safe-identifier `phase`, a + `blocking_failure_count` of at least 1, and `blocking_gate_ids`). + ## Local development ```bash diff --git a/telemetry-service/telemetry_collector/models.py b/telemetry-service/telemetry_collector/models.py index f5fbe61d..193b6281 100644 --- a/telemetry-service/telemetry_collector/models.py +++ b/telemetry-service/telemetry_collector/models.py @@ -18,6 +18,29 @@ SAFE_IDENTIFIER = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,191}$" +def _require_block(details: dict[str, Any]) -> None: + """A ``blocked`` event carries the refusal as staging contract v3 defines a + block: the gate phase, at least one blocking failure, and the gate ids + (possibly none). ``gate_statuses`` and other details are optional.""" + + phase = details.get("phase") + count = details.get("blocking_failure_count") + gate_ids = details.get("blocking_gate_ids") + if not isinstance(phase, str) or not re.fullmatch(SAFE_IDENTIFIER, phase): + raise ValueError("a blocked event's phase must be a safe identifier") + if isinstance(count, bool) or not isinstance(count, int) or count < 1: + raise ValueError( + "a blocked event's blocking_failure_count must be an integer of at least 1" + ) + if not isinstance(gate_ids, list) or any( + not isinstance(gate_id, str) or not 1 <= len(gate_id) <= 200 + for gate_id in gate_ids + ): + raise ValueError( + "a blocked event's blocking_gate_ids must be a list of gate ids" + ) + + class PageCursor(RootModel[tuple[datetime, str]]): """Typed contents of an opaque run-list pagination cursor.""" @@ -81,6 +104,12 @@ def require_timezone(cls, value: datetime) -> datetime: raise ValueError("timestamp must include a timezone") return value + @model_validator(mode="after") + def require_block_details(self) -> TelemetryEvent: + if self.status == "blocked": + _require_block(self.details) + return self + class EventBatch(BaseModel): """Bounded event batch accepted by the ingestion endpoint.""" diff --git a/telemetry-service/tests/test_app.py b/telemetry-service/tests/test_app.py index a01f9859..707cdd49 100644 --- a/telemetry-service/tests/test_app.py +++ b/telemetry-service/tests/test_app.py @@ -382,6 +382,92 @@ def test_blocked_run_event_is_accepted_and_ends_the_run() -> None: assert listed.json()["runs"][0]["ended_at"] is not None +def _blocked_event(details: dict[str, object]) -> dict[str, object]: + return { + "schema_version": 1, + "event_id": "gates-refused", + "run_id": "route-a-1", + "producer_id": "producer-1", + "sequence": 1, + "timestamp": datetime.now(UTC).isoformat(), + "event_type": "run", + "stage_id": "blocked", + "status": "blocked", + "message": "The gates refused the candidate.", + "details": details, + "resources": None, + } + + +@pytest.mark.parametrize( + "details", + [ + {}, + {"blocking_failure_count": 1, "blocking_gate_ids": []}, + {"phase": "pre flight", "blocking_failure_count": 1, "blocking_gate_ids": []}, + {"phase": "terminal", "blocking_gate_ids": ["uk_target_fit"]}, + {"phase": "terminal", "blocking_failure_count": 0, "blocking_gate_ids": []}, + {"phase": "terminal", "blocking_failure_count": True, "blocking_gate_ids": []}, + {"phase": "terminal", "blocking_failure_count": 1}, + {"phase": "terminal", "blocking_failure_count": 1, "blocking_gate_ids": [""]}, + ], +) +def test_blocked_event_without_a_valid_block_is_rejected( + details: dict[str, object], +) -> None: + store = FakeTelemetryStore() + client = TestClient( + create_app( + settings=settings(), + store=store, + huggingface_authenticator=StubHuggingFaceAuthenticator(), + ) + ) + token = exchange(client) + register(client, token) + response = client.post( + "/v1/runs/route-a-1/events", + headers={"Authorization": f"Bearer {token}"}, + json={"events": [_blocked_event(details)]}, + ) + detail = client.get( + "/v1/runs/route-a-1", + headers={"X-Telemetry-Read-Token": "dashboard-read-token"}, + ) + + assert response.status_code == 422 + assert detail.json()["progress"]["status"] == "running" + + +def test_blocked_event_naming_no_gate_and_no_statuses_is_accepted() -> None: + client = TestClient( + create_app( + settings=settings(), + store=FakeTelemetryStore(), + huggingface_authenticator=StubHuggingFaceAuthenticator(), + ) + ) + token = exchange(client) + register(client, token) + response = client.post( + "/v1/runs/route-a-1/events", + headers={"Authorization": f"Bearer {token}"}, + json={ + "events": [ + _blocked_event( + { + "phase": "preflight", + "blocking_failure_count": 2, + "blocking_gate_ids": [], + } + ) + ] + }, + ) + + assert response.status_code == 202 + + def test_health_does_not_require_credentials() -> None: client = TestClient( create_app( From d229e02ca14bb71cc358fb1443c315313222f790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:33:39 +0100 Subject: [PATCH 4/4] Say that collector documents are not staging contract documents They declare schema_version 2 for the monitor's lifecycle reader but can carry version 3's blocked status, block and failure classes; the staging contract validates only staged run documents. Co-Authored-By: Claude Opus 5.5 --- docs/build-monitor.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/build-monitor.md b/docs/build-monitor.md index 43d7fbdd..57741c34 100644 --- a/docs/build-monitor.md +++ b/docs/build-monitor.md @@ -19,6 +19,13 @@ checks stop builds. The run selected on either tab carries over to the other. A missing credential, a user outside the PolicyEngine Hugging Face organization, or an unavailable network does not stop a build. Events remain in the local retry queue for a later Microcosm run to deliver. + + The collector rebuilds each run's documents from its events. They declare + `schema_version: 2` so the monitor reads them with the version 2 lifecycle, + but they are the collector's own shape: they can carry version 3's `blocked` + status, `block` and failure classes, and they are not validated against the + staging contract (`staging-contract.ts`). That contract applies only to staged + run documents, from the Hugging Face staging repository or a local run folder. - **Historical Hugging Face telemetry.** During migration, the hosted view also reads version 1, 2 and 3 run documents from each country's Hugging Face staging repository. A collector record replaces a historical record with the