diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7750ec0f..d45587e7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -237,6 +237,15 @@ jobs:
         run: |
           echo "Local: python -m pytest tests/acceptance/test_network_guard.py -v"
           python -m pytest tests/acceptance/test_network_guard.py -v
+      # The real-model nightly (model-smoke-nightly.yml) cannot run per-PR, but
+      # its decision logic can: checksum verification, the end-to-end
+      # assertions, budget arithmetic and failure classification.  Without this
+      # the harness is only ever exercised at 04:00 UTC, so a typo in it shows up
+      # as a mysteriously red nightly.
+      - name: Run nightly real-model smoke harness unit tests (no model needed)
+        run: |
+          echo "Local: python -m pytest tests/scripts/ -v"
+          python -m pytest tests/scripts/ -v
 
   desktop-check:
     name: Desktop Rust check
diff --git a/.github/workflows/model-smoke-nightly.yml b/.github/workflows/model-smoke-nightly.yml
index bdb9ab99..bd156e8a 100644
--- a/.github/workflows/model-smoke-nightly.yml
+++ b/.github/workflows/model-smoke-nightly.yml
@@ -1,33 +1,100 @@
 # SPDX-License-Identifier: Apache-2.0
-# Nightly real-model smoke test.
+# Nightly real-model smoke test (issue #457).
 #
-# Downloads the pinned starter model from the registry, starts a local
-# llama-server (llama-cpp-python's OpenAI-compatible server) plus convsim-core
-# wired to it, runs a scripted end-to-end session through convsim-core's REST
-# API, measures full-response latency, and fails if it regresses more than
-# 20 % above the documented budget.
+# Nightly, NOT per-PR: this is the only CI job that plays a conversation on a
+# real local model.  Every other job validates against the fake/scripted
+# runtimes, so a regression that only shows up under real inference — a prompt
+# that the model cannot satisfy, a grammar the adapter mis-serialises, a
+# debrief that never scores — is invisible until this job runs.
 #
-# convsim-core's turn endpoint is synchronous (token streaming lives in the
-# apps/api gateway, which is not stood up here), so the smoke measures full
-# end-to-end turn latency against the FULL_RESPONSE_MS budget (10 s).
+# What it does: download the smallest registry model (role `starter`, Qwen3 4B
+# Q4_K_M, ~2.5 GB) with SHA-256 verification, start llama-cpp-python's
+# OpenAI-compatible server on it, start convsim-core wired to that server
+# (CONVSIM_RUNTIME_ID=llama_cpp), play a scripted multi-turn conversation
+# through convsim-core's REST API, and assert a SCORED debrief comes back.
+# Latency is checked against the documented budgets.  Full detail, including
+# the local repro command: docs/real-model-smoke.md.
 #
-# The model file (~2.5 GB) is cached by its registry SHA-256 so repeated
-# nightly runs skip the download.  The first run after a pinned-URL change
-# re-downloads and re-caches the new file.
+# ── Failure classification ───────────────────────────────────────────────────
+# scripts/nightly-model-smoke.py attributes every failure to one class and
+# exits with a distinct code, so a red nightly is triageable from the job
+# summary without reading logs:
 #
-# Timings are measured on GitHub-hosted ubuntu-latest (2 vCPU, ~8 GB RAM,
-# CPU-only inference).  The CI budget is the documented budget multiplied by
-# a hardware-tier factor that accounts for the slower runner:
+#   1  budget    — pipeline worked, latency regressed
+#   2  download  — model could not be fetched (network/HTTP/stall/empty cache)
+#   3  checksum  — SHA-256 drift against the registry
+#   4  runtime   — llama-server or convsim-core crashed, hung, or 5xx'd
+#   5  pipeline  — servers healthy, end-to-end assertion failed
+#   6  timeout   — wall-clock budget exhausted (failing phase named)
+#
+# The script writes the class, the failures, the remedy, per-phase durations
+# and measured latencies to $GITHUB_STEP_SUMMARY, and the full JSON report is
+# uploaded as an artifact.
+#
+# ── Runtime budget: target < 30 min on standard GitHub runners ───────────────
+# timeout-minutes: 30 is the hard ceiling; the script's own
+# --wall-clock-budget-s 1200 (20 min) trips first so the failure is attributed
+# to a phase instead of surfacing as an unattributable job cancellation.  The
+# 10 min gap is deliberate margin, not an accounting of known work: the steps
+# before the smoke (checkout, pip install, cache restore and, on a cache miss,
+# the download plus the cache save) run against the same job clock and are
+# invisible to the script's own deadline, but they measure 31–51 s in total.
+# The margin is sized for the network-bound part of that, which a slow
+# Hugging Face or Actions-cache night can stretch by minutes.  The download is
+# the one of those steps that could stretch without limit, so it carries its own
+# 5 min budget (DOWNLOAD_BUDGET_S in the script) and fails as `download` rather
+# than spending the margin the smoke still needs.
+#
+#   set up job + checkout + setup-python     ~5 s
+#   pip install ×3 (cached wheels)           ~16 s cold, ~14 s warm
+#   model cache restore (2.3 GiB)            ~24 s on a hit, <1 s on a miss
+#   download + SHA-256 verify (cache miss)   ~10 s    — bounded at 5 min
+#   cache save (cache miss only)             ~10 s
+#   SHA-256 re-verify before load            <10 s    — bounded, see below
+#   model load + core start + session        ~8 s
+#   3 scripted turns                         ~6 min   — projected, see below
+#   debrief generation                       ~4 min   — projected, see below
+#   ------------------------------------------------
+#   total                                   ~11 min, warm cache or cold
+#
+# The non-inference figures are measured, from this job's own logs for the
+# twelve nightlies whose logs GitHub still held on 2026-10-03.  Two of them are
+# counter-intuitive and worth knowing before tuning anything: a cache MISS is no
+# more expensive than a cache HIT — ~10 s to fetch and hash the GGUF plus ~10 s
+# to save it, against ~19–28 s to restore the same bytes — so the cache saves
+# the network and not the clock; and everything before the conversation is
+# 31–51 s of a 30 min job, so the budget is inference and nothing else.
+# The re-verify row is the one with no direct measurement — the previous
+# single-turn harness did not re-verify — and is bounded by the cold row above,
+# where downloading 2.3 GiB *and* hashing it together took 10 s.
+#
+# The inference figures are projected from the one latency this job has
+# actually measured — a single behavioral-interview turn, twelve nightlies:
+# 66/71/93/94/98/103/104/111/112/114/115/115 s — rounded up to ~120 s per turn
+# because turns 2 and 3 render a longer transcript than turn 1, with the
+# debrief allowed twice a turn.  Replace both with real per-phase durations
+# from the first green nightly's report artifact.
+#
+# If the smoke step creeps past ~15 min — or the job past ~25 min — reduce
+# SCRIPTED_PLAYER_TURNS rather than raising either ceiling: a nightly that
+# routinely runs near its timeout flaps.
+#
+# ── Latency budget on CI hardware ────────────────────────────────────────────
+# Timings are measured on GitHub-hosted ubuntu-latest, CPU-only inference.
+# The CI budget is the documented budget multiplied by a hardware-tier factor:
 #
 #   CI budget = documented budget × CI_HARDWARE_FACTOR (currently 20)
 #
 # The factor is calibrated empirically: a full behavioral-interview turn on
-# this runner measures ~116 s (two consecutive runs at 117.0 s and 115.3 s —
-# large grammar-constrained prompt, 4B-Q4 model, CPU-only), i.e. ~11.6× the
-# 10 s documented budget.  20× gives a 10 s × 20 = 200 s CI budget and, with
-# the script's 1.20 regression tolerance, a 240 s ceiling — roughly 2× headroom
-# over the observed latency so normal runner-to-runner variance does not flap
-# the nightly, while a genuine >2× regression still fails.  This factor only
+# this runner has measured 66–115 s across the twelve nightlies whose logs
+# GitHub still held on 2026-10-03 (median ~104 s — large grammar-constrained
+# prompt, 4B-Q4 model, CPU-only), i.e. ~11.5× the 10 s documented budget at the
+# slow end, which is the end the factor is set from.  20× gives a
+# 10 s × 20 = 200 s CI budget and, with the script's 1.20 regression tolerance,
+# a 240 s ceiling — roughly 2× headroom over the observed latency so normal
+# runner-to-runner variance does not flap the nightly, while a genuine >2×
+# regression still fails.  The headline full_response_ms is the MEDIAN of the
+# scripted turns, so one unlucky turn cannot flap the job.  This factor only
 # models CI-hardware slowness; the product's 10 s target-hardware SLO is
 # unchanged.  Re-measure and re-tune if the runner class or starter model
 # changes.
@@ -42,11 +109,19 @@ on:
 permissions:
   contents: read
 
+env:
+  # Bump this prefix to abandon a poisoned model cache entry (Actions cache
+  # entries are immutable, so a corrupt GGUF under a given key would otherwise
+  # be restored on every re-run — see the checksum remedy in the script).
+  MODEL_CACHE_PREFIX: model-gguf-v1
+
 jobs:
   model-smoke:
     name: Starter-model end-to-end smoke
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    # Hard ceiling for the documented < 30 min runtime budget.  The script's
+    # own 20 min wall-clock budget trips first and names the failing phase.
+    timeout-minutes: 30
 
     steps:
       - uses: actions/checkout@v5
@@ -65,42 +140,44 @@ jobs:
       - name: Install convsim-core
         run: pip install -e "services/convsim-core[dev]"
 
+      # --only-binary enforces what the step name already promises.  PyPI ships
+      # llama-cpp-python as an sdist only — the prebuilt wheels live on the
+      # extra index — and --extra-index-url does not prefer them: pip resolves
+      # the newest version across both indexes, so a release the wheel index has
+      # not caught up with yet is installed by compiling llama.cpp on the
+      # runner.  That build turns the ~16 s install row in the runtime budget
+      # into 15+ min, which the 30 min job timeout has no room for next to ~11
+      # min of inference: the job would die
+      # with the unattributable cancellation this whole design exists to avoid,
+      # on a night when nothing about the product changed.  Refusing the sdist
+      # makes pip fall back to the newest version that does have a wheel, and if
+      # there is none it fails in seconds against this step's own name.
       - name: Install llama-cpp-python with server extra (CPU-only prebuilt wheel)
         run: |
           pip install "llama-cpp-python[server]" \
+            --only-binary llama-cpp-python \
             --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
 
-      - name: Read starter model metadata from registry
+      # Single source of truth: the cache key, the download URL and the
+      # checksum the smoke verifies against all come from this one lookup, so
+      # they cannot drift apart.
+      - name: Resolve starter model from the registry
         id: registry
         run: |
-          python - <<'EOF'
-          import yaml, json, sys
-          from pathlib import Path
-          reg = yaml.safe_load(Path("model-registry/registry.yaml").read_text())
-          starter = next(m for m in reg["models"] if m["role"] == "starter")
-          url = starter["download"]["url"]
-          sha256 = starter["download"]["sha256"]
-          model_id = starter["id"]
-          print(f"model_id={model_id}")
-          print(f"model_url={url}")
-          print(f"model_sha256={sha256}")
-          # Write to GITHUB_OUTPUT
-          import os
-          out = os.environ.get("GITHUB_OUTPUT", "/dev/stdout")
-          with open(out, "a") as f:
-              f.write(f"model_id={model_id}\n")
-              f.write(f"model_url={url}\n")
-              f.write(f"model_sha256={sha256}\n")
-          EOF
-
-      - name: Cache model file
+          echo "Local: python scripts/nightly-model-smoke.py --print-registry-model starter"
+          python scripts/nightly-model-smoke.py --print-registry-model starter
+
+      # Restore/save are split (rather than actions/cache) so a model that
+      # fails checksum verification is never written to the cache: the save
+      # step is gated on success().
+      - name: Restore cached model file
         id: model-cache
-        uses: actions/cache@v5
+        uses: actions/cache/restore@v5
         with:
           path: ~/.convsim/models/llm
-          key: model-${{ steps.registry.outputs.model_sha256 }}
+          key: ${{ env.MODEL_CACHE_PREFIX }}-${{ steps.registry.outputs.model_sha256 }}
 
-      - name: Download starter model
+      - name: Download starter model (SHA-256 verified)
         if: steps.model-cache.outputs.cache-hit != 'true'
         run: |
           python scripts/nightly-model-smoke.py \
@@ -109,11 +186,24 @@ jobs:
             --model-sha256 "${{ steps.registry.outputs.model_sha256 }}" \
             --model-id "${{ steps.registry.outputs.model_id }}"
 
-      - name: Run end-to-end smoke and check latency budgets
+      - name: Cache the verified model file
+        if: success() && steps.model-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v5
+        with:
+          path: ~/.convsim/models/llm
+          key: ${{ env.MODEL_CACHE_PREFIX }}-${{ steps.registry.outputs.model_sha256 }}
+
+      # --model-sha256 makes the run re-verify the file it is about to load, so
+      # a drifted or truncated CACHE HIT fails loudly (exit 3) instead of
+      # feeding corrupt weights to the smoke.
+      - name: Run real-model smoke (scripted conversation + scored debrief)
         run: |
+          echo "Local: see docs/real-model-smoke.md for the one-line local repro"
           python scripts/nightly-model-smoke.py \
             --model-id "${{ steps.registry.outputs.model_id }}" \
+            --model-sha256 "${{ steps.registry.outputs.model_sha256 }}" \
             --ci-hardware-factor 20 \
+            --wall-clock-budget-s 1200 \
             --report-path /tmp/smoke-report.json
 
       - name: Upload smoke report
diff --git a/ROADMAP.md b/ROADMAP.md
index 5968c4ee..433184a1 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -91,7 +91,7 @@ criteria.
 
 - [x] Real UI screenshots and demo recording, captured from a local-model playthrough — [#455](https://github.com/outrightmental/ConversationSimulator/issues/455)
 - [ ] Desktop app with bundled backend (Tauri sidecar for `convsim-core`) — [#456](https://github.com/outrightmental/ConversationSimulator/issues/456)
-- [ ] Automated real-model CI smoke test — [#457](https://github.com/outrightmental/ConversationSimulator/issues/457)
+- [x] Automated real-model CI smoke test — [#457](https://github.com/outrightmental/ConversationSimulator/issues/457)
 
 ### Post-alpha — Milestone 2+
 
diff --git a/docs-site/src/content/docs/dev/adr-0001-npc-relationship-memory.md b/docs-site/src/content/docs/dev/adr-0001-npc-relationship-memory.md
index dd639cf2..9285d06d 100644
--- a/docs-site/src/content/docs/dev/adr-0001-npc-relationship-memory.md
+++ b/docs-site/src/content/docs/dev/adr-0001-npc-relationship-memory.md
@@ -2,7 +2,7 @@
 title: "ADR-0001: NPC relationship memory"
 description: "Spike go/no-go report on persistent NPC relationship memory, covering the prototype, evaluation results, safety and privacy reviews, and a conditional-go recommendation."
 sidebar:
-  order: 14
+  order: 15
 ---
 
 **Status:** Proposed (spike branch — not merged)  
diff --git a/docs-site/src/content/docs/dev/offline-smoke-tests.md b/docs-site/src/content/docs/dev/offline-smoke-tests.md
index a1409ff0..6befab0f 100644
--- a/docs-site/src/content/docs/dev/offline-smoke-tests.md
+++ b/docs-site/src/content/docs/dev/offline-smoke-tests.md
@@ -131,6 +131,10 @@ Exit codes: `0` = pass, `1` = network violation or pack error, `3` = unexpected
 To verify that the installed llama.cpp or Ollama runtime makes no *unexpected*
 outbound calls, run the smoke test while the real runtime is active.
 
+> This page covers the **network** promise only. For the nightly CI job that
+> plays a scripted conversation on a real model end-to-end and asserts a scored
+> debrief, see [Real-model smoke test](/dev/real-model-smoke/).
+
 ### 1. Start the local runtime
 
 ```bash
diff --git a/docs-site/src/content/docs/dev/post-alpha-issues.md b/docs-site/src/content/docs/dev/post-alpha-issues.md
index bce37ff4..3121404f 100644
--- a/docs-site/src/content/docs/dev/post-alpha-issues.md
+++ b/docs-site/src/content/docs/dev/post-alpha-issues.md
@@ -124,7 +124,19 @@ latency and output quality signals.
 cache-unfriendly in most CI environments. The fake runtime provides full
 structural coverage; real-model CI is a quality-of-life improvement.
 
-**Milestone:** 2 (CI hardening)
+**Milestone:** 2 (CI hardening)  
+**Status:** ✅ **Shipped,** pulled forward into the Milestone-1 polish set
+rather than waiting for Milestone 2.
+[`.github/workflows/model-smoke-nightly.yml`](https://github.com/outrightmental/ConversationSimulator/blob/main/.github/workflows/model-smoke-nightly.yml)
+runs nightly: it resolves the registry's `starter` model from one lookup,
+downloads it with SHA-256 verification (cached between runs and re-verified
+before the weights load, so a drifted cache hit fails loudly), starts
+`llama-server` and `convsim-core` on it, plays a scripted multi-turn
+conversation and asserts a *scored* debrief — inside a documented budget under
+30 min. Every failure is attributed to one of six classes with its own exit
+code, so a red nightly is triageable from the job summary; the harness's own
+decision logic is unit-tested per-PR. See
+[Real-model smoke test](/dev/real-model-smoke/).
 
 ---
 
diff --git a/docs-site/src/content/docs/dev/real-model-smoke.md b/docs-site/src/content/docs/dev/real-model-smoke.md
new file mode 100644
index 00000000..ca99fa74
--- /dev/null
+++ b/docs-site/src/content/docs/dev/real-model-smoke.md
@@ -0,0 +1,496 @@
+---
+title: "Real-model smoke test (nightly)"
+description: "The nightly CI job that plays a scripted conversation on a real local model end-to-end — registry download, llama.cpp, convsim-core, scored debrief — with classified failures and a documented runtime budget."
+sidebar:
+  order: 13
+---
+
+Every other CI job validates Conversation Simulator against the **fake** and
+**scripted** runtimes — deterministic stand-ins that need no model download.
+That keeps pull requests fast, but it means a whole class of regression is
+invisible per-PR: a prompt a real model cannot satisfy, a JSON schema the
+adapter mis-serialises, a debrief that generates prose but never a score.
+
+The nightly real-model smoke closes that gap. It plays a scripted conversation
+on a **real local model** end-to-end — registry download → llama.cpp →
+`convsim-core` → scored debrief — and fails with a classified verdict.
+
+| | |
+|---|---|
+| Workflow | [`.github/workflows/model-smoke-nightly.yml`](https://github.com/outrightmental/ConversationSimulator/blob/main/.github/workflows/model-smoke-nightly.yml) |
+| Harness | [`scripts/nightly-model-smoke.py`](https://github.com/outrightmental/ConversationSimulator/blob/main/scripts/nightly-model-smoke.py) |
+| Unit tests (per-PR) | `tests/scripts/test_nightly_model_smoke.py` |
+| Schedule | 04:00 UTC daily, plus **Run workflow** (`workflow_dispatch`) |
+| Runner | GitHub-hosted `ubuntu-latest`, CPU-only inference |
+| Model | registry role `starter` — Qwen3 4B Instruct Q4\_K\_M, ~2.5 GB, Apache-2.0 |
+| Runtime budget | **< 30 min** (`timeout-minutes: 30`); the harness self-limits to 20 min |
+
+> Nightly, not per-PR, on purpose: a 2.5 GB download plus ~11 min of CPU-only
+> inference cannot sit in the PR path. The fake-runtime counterpart is the
+> playthrough in `tests/e2e/test_scripted_playthrough.py` — run at release time
+> by `scripts/release-smoke.sh`, not on pull requests — which uses the *same
+> scripted player turns*, so the two cover the same conversation shape against
+> different runtimes. What *does* run per-PR is this harness's own unit tests.
+
+---
+
+## What the nightly proves
+
+1. The pinned starter model downloads and its bytes match the registry SHA-256.
+2. `llama-server` loads the GGUF and `convsim-core` reports
+   `runtime_id == "llama_cpp"` — the smoke refuses to run on a fake runtime.
+3. A real model drives a multi-turn conversation on the built-in
+   `behavioral_interview` scenario, producing NPC turns that satisfy
+   `turn-output.schema.json`.
+4. The session ends and produces a **scored** debrief: at least one rubric
+   dimension scored, a numeric `overall_score` in `[0, 100]`, and a non-empty
+   summary. The scores come from the rubric observations the model volunteered
+   on each turn, so this asserts the scoring path; the debrief *narrative* is a
+   separate model call, and a run where it fell back to the deterministic
+   template still passes — with a warning that says so, because the scores are
+   unaffected and one unlucky generation must not red the nightly. Recurring
+   across nightlies is a regression in the debrief prompt or its schema.
+5. End-to-end turn latency stays within the documented budget, scaled for CI
+   hardware.
+
+The NPC *opening* line is authored scenario text, not a generation — so the
+scripted player turns, not the opening, are what prove the model is working.
+
+A turn that is real model output is not automatically a *reply*, and the
+per-turn parse flags cannot tell the difference. All twelve nightly runs of the
+previous single-turn harness whose logs GitHub still held on 2026-10-03 logged
+an NPC answer whose leading characters were byte-identical to the scenario's
+`opening_npc_says` — the model reciting the opening question back instead of
+answering it. `used_fallback` stays false for those turns (the utterance is
+neither empty nor the canned safe one), so the run went green on a conversation
+that never happened. The harness therefore records `replayed_opening` per turn
+and applies the same policy it applies to fallbacks: one such turn warns,
+*every* generated turn doing it fails, because then the model answered nothing
+and the run proves nothing.
+
+Read that twelve-for-twelve with its cause attached, because it is not evidence
+that the starter model always recites. The single player turn those runs
+submitted was `"Reply with exactly one sentence: Hello, I am ready."` — an instruction
+addressed to the model rather than something an interview candidate says — and
+this job replaces it with the three in-character turns in
+`SCRIPTED_PLAYER_TURNS`, which the real starter model answered properly in local
+verification. So the recital is a known failure mode of the *old* prompt, and
+the baseline rate under the new script is simply not yet known: expect zero, and
+treat the first recital warning as new information rather than as the documented
+status quo.
+
+The two policies also compose, because a conversation can be *split* between
+them — one turn recites the opening, the others fall back — and trip neither
+"all of them" check while still containing no reply at all. Each check warns
+rather than fails on its own only because a run that recovers from one bad turn
+is the product working; when nothing recovered there is nothing to credit, so a
+run with no turn that is both real model output *and* an answer fails as well.
+
+What counts as a recital is a *leading prefix* match, `REPLAY_PREFIX_CHARS`
+(40 collapsed characters) of the authored opening reproduced at the head of the
+reply — not the whole opening. Those nightly logs printed only `npc_text[:80]`
+against a 122-character opening, so what they establish is that the reply
+*began* as the opening; whether the copy ran to the end or veered into the
+model's own words after a sentence or two was never recorded, and a check that
+demanded the entire line would have gone unreachable on the second case. Forty
+characters of the scenario's own text at the head of an answer is not something
+a reply arrives at by chance — the transcript the turn prompt renders is the
+only place it can come from — and the threshold is capped at the opening's own
+length, so a scenario with a terser opening still has to have it recited in
+full.
+
+If the NPC closes the conversation before the script runs out
+(`session_control.continue_session`), the run still passes on the turns it did
+play — that is the product working as designed — but it records
+`scripted_turns_played` and warns in the step summary, so a green run on one
+turn is not mistaken for a green run on three.
+
+---
+
+## Failure classification
+
+A red nightly must be triageable from the job summary alone, so every failure is
+attributed to exactly one class, with its own exit code, its own banner in the
+log, and a remedy printed next to it:
+
+| Exit | Class | Means | First thing to do |
+|---|---|---|---|
+| 1 | `budget` | Pipeline worked; latency regressed past the CI ceiling | Compare `measured_ms` in the report artifact against recent nightlies |
+| 2 | `download` | Model could not be fetched — network, HTTP, a transfer that stalled past its 5 min budget, or an empty cache | `python scripts/validate-registry.py --url-check`, then re-run |
+| 3 | `checksum` | **SHA-256 drift**: on-disk bytes ≠ `model-registry/registry.yaml` | See [Checksum drift](#checksum-drift) — never relax the check |
+| 4 | `runtime` | `llama-server` or `convsim-core` crashed, hung, or returned 5xx — or one of the two ports was already taken, so neither could be started | Read the child stderr tail printed above the banner — but only if a child actually exited. The banner says which case it is and carries the matching remedy: a port conflict started no child (see [Running it locally](#running-it-locally)), and a 5xx from a server that is still running is usually [a cut-off generation](#a-5xx-from-a-live-server), not a crash |
+| 5 | `pipeline` | Servers healthy, but an end-to-end assertion failed | Inspect the per-turn `used_fallback` and `replayed_opening` flags in the report artifact — except for an unscored debrief, see [Unscored debrief](#unscored-debrief) |
+| 6 | `timeout` | Wall-clock budget exhausted; the failing phase is named | Check `phase_durations_s` before raising the budget |
+
+The distinction that matters most in practice is **2/3 vs 4 vs 5**: a download or
+checksum failure says nothing about the app, a runtime failure is a crash, and a
+pipeline failure means the model ran and produced output the product rejected.
+
+Anything the harness could not carry out — a `registry.yaml` that is malformed,
+names no usable `starter` model, or still carries the schema's `PENDING`
+placeholder instead of a pinned URL and checksum; a bad command line; a disk
+that fills up mid-run; a bug in the harness itself — is also reported as exit 5,
+with a message and a remedy that say so, so that **exit 1 only ever means a
+latency regression**. Nothing exits with an unclassified traceback, and nothing
+borrows a class's exit code without correcting the advice printed beneath it: a
+broken registry entry is *not* exit 2, because `download`'s remedy ends "re-run
+the job" and no re-run will repair a file in the repository.
+
+When more than one class could apply, the harness reports the strongest evidence
+rather than the symptom the client happened to see:
+
+1. **A crashed child wins.** A dead `convsim-core` reads as `runtime`, not as the
+   connection reset it caused downstream.
+2. **An exhausted wall clock beats a transport error.** Each request timeout is
+   capped to the budget that remains, so the last call before the deadline dies
+   client-side and *looks* like an unresponsive server. It is reported as
+   `timeout`, whose remedy points at `phase_durations_s`.
+
+Because `timeout` outranks `budget`, a *uniform* slowdown normally surfaces as
+exit 6 rather than exit 1: three turns at the 240 s CI ceiling plus a debrief
+allowed twice that very nearly fill the 20 min budget, so a ~2x regression
+exhausts the clock before the budget phase runs. The class stays `timeout` — the
+clock really did run out — but when the turns that *did* complete already have a
+median past the ceiling, the verdict says so and names the measurement, so exit 6
+is not mistaken for a hang. Exit 1 remains the verdict when the conversation
+finishes inside the budget but too slowly.
+
+Each run writes the verdict, failure class, remedy, measured latencies and
+per-phase durations to the GitHub **step summary**, and uploads the full JSON
+report as the `model-smoke-report` artifact (30-day retention).
+
+### A 5xx from a live server {#a-5xx-from-a-live-server}
+
+A 5xx is a `runtime` failure because the transport said so, but `runtime`'s
+stock advice — read the child stderr tail, suspect a runner OOM or a bad
+llama-cpp-python wheel — only fits a failure where a child actually died. Often
+none has, and then the tail above the banner shows a perfectly healthy server.
+The harness checks `poll()` on both children and, when neither has exited, prints
+a different remedy: *"Neither child process exited, so nothing crashed."*
+
+The cause worth knowing about is a generation the adapter cut off. `convsim-core`
+answers **504 `TURN_TIMEOUT`** when `CONVSIM_LLAMA_CPP_CHAT_TIMEOUT` runs out
+mid-reply. The harness deliberately sets that budget *above* the latency ceiling
+it is about to judge the turn on — `DEBRIEF_SLOWDOWN_FACTOR` × the ceiling, 540 s
+at `--ci-hardware-factor 20` — so that a slow turn is measured and reported as
+`budget` rather than killed and misreported as a crash. But that moves the cliff
+rather than removing it: a turn slower than 540 s is still cut off, and with
+1200 s of wall clock still unspent neither `timeout` nor `budget` takes the class
+over. So exit 4 with both children alive and a 504 in the message means inference
+roughly 5× slower than the nightlies have ever measured — a latency regression.
+Start from `measured_ms` and `phase_durations_s` in the report artifact, not from
+an OOM.
+
+The other way to land here with both children alive is a server that is up but
+wedged: `llama-server` still loading the GGUF when readiness ran out, or a
+request it refused. The stderr tails are the right place for that one.
+
+### Unscored debrief {#unscored-debrief}
+
+Exit 5 with *"Debrief has no rubric dimension scores"* is the one `pipeline`
+failure whose cause is ambiguous from the class alone, so the harness resolves it
+for you: each NPC turn's `rubric_observation_count` is recorded in the report
+artifact, totalled as `rubric_observations_seen`, and the failure message names
+which of the causes below applies.
+
+The debrief's dimension scores are accumulated entirely from
+`rubric_observations` that the model volunteers on each NPC turn. Nothing asks
+it for them: no prompt layer names any rubric dimensions, and the only hint the
+model gets is the bare `rubric_observations` array in the embedded output
+schema — whose empty list the schema accepts. Nor could a scenario supply one.
+`ScenarioData` (in `convsim_prompt.types`) has no rubric field at all, so the
+official `job-interview-basic` pack's `behavioral_interview` — which *does*
+define a rubric — would compose the identical turn prompt; and the full edition
+resolves the built-in catalogue before installed packs, so this job always plays
+the rubric-less built-in scenario regardless. The fake runtime always returns
+`[]`, which is why the release-time playthrough asserts only that `scores` *is*
+a dict.
+
+That thin prompt coverage is the standing weakness that makes an unscored
+debrief reachable at all. It is *not*, however, the expected outcome: in local
+verification against the real starter model — Qwen3 4B Q4\_K\_M, the same pin
+the nightly uses, under the turn-output schema — every one of six scripted turns
+across two runs volunteered at least one observation, and the debrief scored
+three to four dimensions with an `overall_score` near 50. Treat zero
+observations as a signal, not as the resting state.
+
+So an unscored debrief means one of four quite different things:
+
+- **`rubric_observations_seen` is 0 and every generated turn fell back.** Then
+  this failure is downstream of the fallback failure reported beside it and
+  carries no independent signal: a turn that fell back returns the canned safe
+  utterance and an empty `rubric_observations` list whatever the prompt asked
+  for, so the debrief would have had nothing to accumulate even with a rubric
+  layer in place. Start from the per-turn `parse_events` in the `/debug`
+  payload — they say why the model's output was rejected — and judge the
+  scoring path on the next run. The harness detects this case and says so
+  rather than printing the next bullet's note, which would be the wrong
+  investigation.
+- **`rubric_observations_seen` is 0 with turns that did not fall back — the
+  model volunteered nothing.** Only the prompt's bare schema hint was ever
+  asking, so this is reachable by design —
+  but since the starter model does normally answer that hint, first check what
+  changed about what reaches the model: the `OUTPUT_SCHEMA` prompt layer, the
+  registry's `starter` pin, the adapter's JSON-schema constraint, sampling
+  settings. Then fix the weakness itself in the product — the turn prompt needs
+  a rubric layer *and* a rubric to put in it, since `ScenarioData` carries none
+  — not by relaxing the assertion, and not by pointing the smoke at a
+  rubric-defining scenario, which would compose the same prompt.
+- **`rubric_observations_seen` is above 0 — a real regression.** The turns
+  returned observations and the debrief scored none of them, so the model did
+  its part. The debrief engine does not score the validated observations the
+  turn pipeline handed it: `_parse_rubric_observations` in
+  `services/convsim-core/.../debrief_engine.py` re-reads each NPC turn's *stored
+  raw model output* with a plain `json.loads`, so output that needed repair — or
+  that arrived pre-parsed from the runtime adapter — loses its observations on
+  the way to the debrief. Compare `raw_npc_output` from the `/debug` payload
+  against the per-turn `rubric_observation_count` in the artifact.
+- **`rubric_observations_seen` is `null` — the harness could not tell.** At
+  least one generated turn carried no readable `rubric_observations` list, and
+  the turns it *could* read carried nothing between them, so the total cannot be
+  trusted as a total and the message quotes the two causes it cannot tell
+  apart — volunteered nothing, or volunteered something the debrief lost —
+  rather than picking one. Check `rubric_observation_count` per turn in the report to see
+  which turns were unreadable (`null`) and which really were zero; `null` is not
+  "every turn was unreadable". `convsim-core` always sends that list today, so
+  any `null` is itself worth chasing: the turn response contract changed. The
+  harness deliberately does not round an unreadable payload down to zero — that
+  would have the verdict assert the model volunteered nothing when the turn it
+  could not read may have carried plenty.
+
+In every case the run is red: *"scored debrief"* is the acceptance criterion for
+[#457](https://github.com/outrightmental/ConversationSimulator/issues/457).
+No cause is a reason to weaken the check. The one that is *also* a standing
+product gap is **zero observations with turns that did not fall back**: that one
+is worth a tracking issue for the missing rubric prompt layer *and* an
+investigation of the run that hit it. The all-fell-back case is not — it says
+nothing about rubric prompting, and the harness says so rather than sending you
+after the prompt layer.
+
+---
+
+## Runtime budget
+
+Target: **under 30 minutes on a standard GitHub-hosted runner.** The job sets
+`timeout-minutes: 30` as a hard ceiling, and the harness runs with
+`--wall-clock-budget-s 1200` (20 min) so *it* fails first and names the phase
+that ran long — a GitHub-side timeout would only say "the operation was
+canceled".
+
+The 10-minute gap between the two is deliberate margin, not an accounting of
+known work. `timeout-minutes` covers the whole job, and the harness only starts
+after checkout, three `pip install` steps, the model cache restore and — on a
+cache miss — the download and the cache save that follows it, which together
+measure **31–51 s** on the nightlies run so far, not the several
+minutes a 2.5 GB transfer sounds like. The gap is sized for the part of that
+work that is network-bound rather than for its measured cost: a slow
+Hugging Face or Actions-cache night can cost minutes that the harness's own
+clock never sees, and the whole design exists so that a slow night still
+produces an attributed verdict rather than a job cancellation.
+
+On `ubuntu-latest` (CPU-only, 4B Q4\_K\_M):
+
+| Phase | Cold cache | Warm cache |
+|---|---|---|
+| Set up job + checkout + `setup-python` | ~5 s | ~5 s |
+| `pip install` ×3 (prompt-composer, convsim-core, llama-cpp-python wheel) | ~16 s | ~14 s |
+| Model cache restore (2.3 GiB) | — (miss: <1 s) | ~24 s |
+| Model download from Hugging Face + SHA-256 verify (2.3 GiB) ◊ | ~10 s | — |
+| Cache save (2.3 GiB, its own step before the smoke) | ~10 s | — |
+| SHA-256 re-verify before the weights are loaded ‡ | <10 s | <10 s |
+| `llama-server` model load + `convsim-core` start + session create | ~8 s | ~8 s |
+| 3 scripted turns † | ~6 min | ~6 min |
+| Debrief generation † | ~4 min | ~4 min |
+| **Total** | **~11 min** | **~11 min** |
+
+The non-inference rows are measured, from the job's own logs for the twelve
+nightlies whose logs GitHub still held on 2026-10-03 — the cold-cache rows from
+the one of those that missed the cache. Two things in them are worth knowing
+before tuning anything:
+
+- **A cache miss costs no more than a cache hit.** Fetching the GGUF from
+  Hugging Face and hashing it takes ~10 s, and saving it to the Actions cache
+  another ~10 s; restoring the same bytes on a hit takes 19–28 s. Caching the
+  model saves the *network*, not the clock, and abandoning a poisoned cache
+  entry costs nothing.
+- **Everything before the conversation is noise.** All of it together measured
+  31–51 s across those twelve runs, in a 30 min job. The budget is inference,
+  and only inference.
+
+◊ The download is the one pre-smoke step that could stretch without limit, and
+it runs where the harness's own deadline cannot see it, so it carries its own
+5 min budget (`DOWNLOAD_BUDGET_S`) with a 2 min cap on any single quiet stretch
+(`DOWNLOAD_READ_TIMEOUT_S`). `urlopen`'s timeout is per socket read, not per
+transfer, so a connection that goes quiet repeatedly — or merely crawls — would
+otherwise spend the whole margin above and then hand the smoke a job clock with
+no room in it, which is the one way left to reach an unattributable
+cancellation. Exceeding either bound is exit 2 with its own remedy. 5 min is
+~30× the measured 10 s, so a transfer has to be pathologically slow to hit it.
+
+‡ The harness re-verifies the on-disk file at the start of every run, which the
+previous single-turn harness did not, so this row has no direct measurement.
+The bound comes from the cold-cache row above: downloading 2.3 GiB *and*
+hashing it took 10 s in total, so the hash alone is a few seconds.
+
+† The two inference rows are *projected*, because the multi-turn conversation
+and the debrief have never run on a runner. The projection comes from the only
+latency this job has measured — one behavioral-interview turn, twelve
+nightlies: 66 / 71 / 93 / 94 / 98 / 103 / 104 / 111 / 112 / 114 / 115 / 115 s —
+rounded up to ~120 s per turn, since turns 2 and 3 render a longer transcript
+than turn 1, and a
+debrief allowed twice a turn (`DEBRIEF_SLOWDOWN_FACTOR`). Replace both rows with
+the real `phase_durations_s` from the first green nightly's report artifact.
+
+At ~11 min projected against a 20 min harness budget there is roughly 2×
+headroom, and the harness's clock covers only the rows from the re-verify
+downwards.
+
+The model is cached between runs under the key
+`model-gguf-v1-<registry-sha256>`, so the download only recurs when the registry
+pin changes. Cache *restore* and *save* are separate steps and the save is gated
+on `success()`, so a file that fails verification is never written to the cache.
+
+If the smoke step creeps past ~15 min — or the job past ~25 min — shorten
+`SCRIPTED_PLAYER_TURNS` rather than raising either ceiling: a nightly that
+routinely runs near its timeout flaps.
+
+### Latency budget on CI hardware
+
+CPU-only CI hardware is far slower than the mid-spec reference machine the
+product budgets target ([Performance and hardware tiers](/play/performance/)), so the harness
+scales the documented budget:
+
+```
+CI ceiling = documented budget × CI_HARDWARE_FACTOR × REGRESSION_TOLERANCE
+           = 10 000 ms        × 20                 × 1.20   = 240 000 ms
+```
+
+`CI_HARDWARE_FACTOR` (20) is calibrated empirically: a full behavioral-interview
+turn on this runner has measured 66–115 s across the twelve nightlies whose
+logs GitHub still held on 2026-10-03 (median ~104 s), i.e. ~11.5× the 10 s
+documented budget at the slow end. The factor is set from that slow end, not the
+median, and still leaves
+roughly 2× headroom — note the 66 s run against the 115 s one, which is the
+night-to-night spread a single-sample calibration would have missed. A genuine
+>2× regression still fails. The headline `full_response_ms` is the **median** of
+the scripted turns, so one unlucky turn cannot flap the job either.
+
+This factor models CI slowness only — the product's 10 s target-hardware budget
+is unchanged. Re-measure and re-tune it if the runner class or the starter model
+changes.
+
+`session_start_ms` and `debrief_ms` are measured and reported but **not**
+budget-checked: the NPC opening is authored text rather than inference, and
+debrief generation has no documented SLO.
+
+---
+
+## Checksum drift
+
+The checksum is verified in two places: right after a fresh download, and again
+at the start of every run (including cache hits) before the weights are loaded.
+A cache hit with drifted bytes therefore fails loudly instead of silently
+feeding a corrupt model to the smoke.
+
+Exit 3 means one of two things:
+
+- **The pinned upstream file was replaced.** All registry URLs are pinned to a
+  specific Hugging Face revision, so this should not happen silently — if it
+  did, review the new file and re-pin with `scripts/pin-model.py`.
+- **The cached download is corrupt or truncated.** The harness deletes the bad
+  local file, but Actions cache entries are immutable, so a poisoned entry would
+  be restored again on the next run. Delete it
+  (`gh cache delete model-gguf-v1-<sha>`) or bump `MODEL_CACHE_PREFIX` in the
+  workflow.
+
+Never "fix" a drift by updating the expected hash to whatever is on disk — that
+is exactly the check this job exists to perform.
+
+What exit 3 does *not* mean is that the digest it was handed is unusable. The
+expected digest is normalised (case, surrounding whitespace) and then checked to
+be 64 hex characters before anything is hashed; one that is not — a value copied
+one character short out of a terminal, the realistic mistake when running the
+repro below by hand — is exit 5 with its own remedy, and the model file is left
+exactly where it was. Nothing no file's hash could ever match is allowed to cost
+a 2.5 GB download or to be reported as upstream drift.
+
+---
+
+## Running it locally
+
+You need the starter model on disk and `llama-cpp-python[server]` installed.
+Nothing here touches the network except the one-time model download.
+
+```bash
+# 1. Install the harness dependencies
+pip install -e "packages/prompt-composer[dev]"
+pip install -e "services/convsim-core[dev]"
+pip install "llama-cpp-python[server]" \
+  --only-binary llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+
+# 2. Resolve the starter model's id / url / sha256 from the registry
+python scripts/nightly-model-smoke.py --print-registry-model starter
+
+# 3. Download it (~2.5 GB) into ~/.convsim/models/llm/, verifying the checksum
+python scripts/nightly-model-smoke.py --download-only \
+  --model-id qwen3-4b-instruct-q4_k_m \
+  --model-url "<model_url from step 2>" \
+  --model-sha256 "<model_sha256 from step 2>"
+
+# 4. Run the smoke. --ci-hardware-factor 1 holds your machine to the real
+#    product budget; use 20 to reproduce what CI accepts.
+python scripts/nightly-model-smoke.py \
+  --model-id qwen3-4b-instruct-q4_k_m \
+  --model-sha256 "<model_sha256 from step 2>" \
+  --ci-hardware-factor 1 \
+  --report-path /tmp/smoke-report.json
+```
+
+Useful extras:
+
+- `--verify-only` — re-verify an already-downloaded model and exit. Like every
+  other checksum check here, a mismatch **deletes** the file (exit 3), so a
+  re-run re-downloads rather than re-verifying the same bad bytes — do not
+  reach for it as a read-only inspection of a 2.5 GB download you want to keep.
+  A `--model-sha256` that is not 64 hex characters is rejected as exit 5 before
+  the file is touched, so a mistyped digest costs you nothing.
+- `--models-dir <dir>` — look for `<model-id>.gguf` somewhere other than
+  `~/.convsim/models/llm/`.
+- `--wall-clock-budget-s` — the self-imposed deadline (default 1200 s).
+
+The harness binds `llama-server` on port 7356 and `convsim-core` on port 7399,
+and refuses to start (exit 4) if either is already taken. That is deliberate
+rather than fussy: readiness is a URL poll, so a server the harness did not
+start would answer it, the child that lost the bind would die unnoticed, and the
+run would report a **pass** for a model it never checksum-verified. Stop whatever
+owns the port — do not work around it.
+
+To exercise the harness's own logic without a model:
+
+```bash
+python -m pytest tests/scripts/ -v
+```
+
+---
+
+## Privacy
+
+The conversation is 100 % scripted (`SCRIPTED_PLAYER_TURNS` in the harness) and
+contains no user data, so short NPC excerpts are printed to the log and recorded
+in the report artifact — they are the evidence that a real model, rather than a
+canned fallback, drove the conversation. Sessions are created with
+`save_transcript: false`, so no transcript file is written, and `convsim-core`
+runs against a throwaway data directory that is deleted when the run ends.
+
+---
+
+## Related
+
+- [Offline smoke tests](/dev/offline-smoke-tests/) — proves no cloud service is
+  contacted during play (fake runtime, per-PR).
+- [Voice smoke tests](/dev/voice-smoke-tests/) — STT/TTS sidecar checks.
+- [Performance and hardware tiers](/play/performance/) — where the latency budgets
+  come from.
+- [Release checklist](/dev/release-checklist/) — Part F covers manual real-model
+  verification of a packaged build.
diff --git a/docs-site/src/content/docs/dev/screenshots.md b/docs-site/src/content/docs/dev/screenshots.md
index 18fc9b64..b80650af 100644
--- a/docs-site/src/content/docs/dev/screenshots.md
+++ b/docs-site/src/content/docs/dev/screenshots.md
@@ -2,7 +2,7 @@
 title: "Screenshots & demo assets"
 description: "What each captured screenshot and the demo recording in docs/assets/ show, the machine they were captured on, their alt text and licence, and the command that remakes them."
 sidebar:
-  order: 13
+  order: 14
 ---
 
 Every image in `docs/assets/` is a capture of the app running a real session against a
diff --git a/docs-site/src/content/docs/play/performance.md b/docs-site/src/content/docs/play/performance.md
index 2cad3474..ab2def3e 100644
--- a/docs-site/src/content/docs/play/performance.md
+++ b/docs-site/src/content/docs/play/performance.md
@@ -9,12 +9,13 @@ ConversationSimulator runs a local LLM, optionally local STT (Whisper) and TTS (
 
 ## Latency budgets
 
-These are the official latency budgets for the **mid-spec reference machine** (Apple M2 / NVIDIA RTX 3060 equivalent). In-app performance warnings fire when a measurement exceeds the corresponding budget. Nightly CI smoke tests flag any regression greater than 20 % against these values.
+These are the official latency budgets for the **mid-spec reference machine** (Apple M2 / NVIDIA RTX 3060 equivalent). In-app performance warnings fire when a measurement exceeds the corresponding budget. Of these, only **full NPC response** is enforced automatically: the [nightly real-model smoke test](/dev/real-model-smoke/) scales that budget for CPU-only CI hardware and fails when a measurement exceeds the scaled budget by more than 20 %. The others are verified by hand — cold start, TTS and STT are not exercised by that job, and TTFT needs the token-streaming gateway it does not start.
 
 | Metric | Budget | Condition |
 |--------|--------|-----------|
 | Cold start → interactive Home | < 10 s | Model already downloaded; startup to playable state |
 | Time-to-first-token (TTFT) | < 2.5 s | Starter model (4 B Q4\_K\_M) on recommended-tier hardware |
+| Full NPC response | < 10 s | Starter model, complete reply returned; the one budget the nightly CI smoke enforces |
 | TTS first-audio chunk | < 1.5 s | Kokoro sidecar, sentence-level streaming |
 | STT round-trip | < 2 s | 10-word utterance, whisper.cpp small model |
 
diff --git a/docs-site/src/content/docs/project/roadmap.md b/docs-site/src/content/docs/project/roadmap.md
index 162c546f..0049a6b4 100644
--- a/docs-site/src/content/docs/project/roadmap.md
+++ b/docs-site/src/content/docs/project/roadmap.md
@@ -77,7 +77,7 @@ criteria.
 
 - [x] Real UI screenshots and demo recording, captured from a local-model playthrough
 - [ ] Desktop app with bundled backend (Tauri sidecar for `convsim-core`)
-- [ ] Automated real-model CI smoke test
+- [x] Automated real-model CI smoke test
 
 ### Post-alpha — Milestone 2+
 
diff --git a/docs/offline-smoke-tests.md b/docs/offline-smoke-tests.md
index 98c397b1..50b9c18b 100644
--- a/docs/offline-smoke-tests.md
+++ b/docs/offline-smoke-tests.md
@@ -127,6 +127,10 @@ Exit codes: `0` = pass, `1` = network violation or pack error, `3` = unexpected
 To verify that the installed llama.cpp or Ollama runtime makes no *unexpected*
 outbound calls, run the smoke test while the real runtime is active.
 
+> This page covers the **network** promise only. For the nightly CI job that
+> plays a scripted conversation on a real model end-to-end and asserts a scored
+> debrief, see [Real-model smoke test](real-model-smoke.md).
+
 ### 1. Start the local runtime
 
 ```bash
diff --git a/docs/performance.md b/docs/performance.md
index eea01726..2be60b69 100644
--- a/docs/performance.md
+++ b/docs/performance.md
@@ -4,12 +4,13 @@ ConversationSimulator runs a local LLM, optionally local STT (Whisper) and TTS (
 
 ## Latency budgets
 
-These are the official latency budgets for the **mid-spec reference machine** (Apple M2 / NVIDIA RTX 3060 equivalent). In-app performance warnings fire when a measurement exceeds the corresponding budget. Nightly CI smoke tests flag any regression greater than 20 % against these values.
+These are the official latency budgets for the **mid-spec reference machine** (Apple M2 / NVIDIA RTX 3060 equivalent). In-app performance warnings fire when a measurement exceeds the corresponding budget. Of these, only **full NPC response** is enforced automatically: the [nightly real-model smoke test](real-model-smoke.md) scales that budget for CPU-only CI hardware and fails when a measurement exceeds the scaled budget by more than 20 %. The others are verified by hand — cold start, TTS and STT are not exercised by that job, and TTFT needs the token-streaming gateway it does not start.
 
 | Metric | Budget | Condition |
 |--------|--------|-----------|
 | Cold start → interactive Home | < 10 s | Model already downloaded; startup to playable state |
 | Time-to-first-token (TTFT) | < 2.5 s | Starter model (4 B Q4\_K\_M) on recommended-tier hardware |
+| Full NPC response | < 10 s | Starter model, complete reply returned; the one budget the nightly CI smoke enforces |
 | TTS first-audio chunk | < 1.5 s | Kokoro sidecar, sentence-level streaming |
 | STT round-trip | < 2 s | 10-word utterance, whisper.cpp small model |
 
diff --git a/docs/post-alpha-issues.md b/docs/post-alpha-issues.md
index 0d4ddc2c..c9f319e1 100644
--- a/docs/post-alpha-issues.md
+++ b/docs/post-alpha-issues.md
@@ -130,6 +130,17 @@ cache-unfriendly in most CI environments. The fake runtime provides full
 structural coverage; real-model CI is a quality-of-life improvement.
 
 **Milestone:** 2 (CI hardening)  
+**Status:** ✅ **Shipped** ([#457](https://github.com/outrightmental/ConversationSimulator/issues/457)),
+pulled forward into the Milestone-1 polish set rather than waiting for
+Milestone 2. `.github/workflows/model-smoke-nightly.yml` runs nightly: it
+resolves the registry's `starter` model from one lookup, downloads it with
+SHA-256 verification (cached between runs and re-verified before the weights
+load, so a drifted cache hit fails loudly), starts `llama-server` and
+`convsim-core` on it, plays a scripted multi-turn conversation and asserts a
+*scored* debrief — inside a documented budget under 30 min. Every failure is
+attributed to one of six classes with its own exit code, so a red nightly is
+triageable from the job summary; the harness's own decision logic is unit-tested
+per-PR. See [real-model-smoke.md](real-model-smoke.md).  
 **Tracking:** [#457](https://github.com/outrightmental/ConversationSimulator/issues/457)
 
 ---
diff --git a/docs/real-model-smoke.md b/docs/real-model-smoke.md
new file mode 100644
index 00000000..2c124f08
--- /dev/null
+++ b/docs/real-model-smoke.md
@@ -0,0 +1,492 @@
+<!-- SPDX-License-Identifier: CC-BY-4.0 -->
+# Real-model smoke test (nightly)
+
+Every other CI job validates Conversation Simulator against the **fake** and
+**scripted** runtimes — deterministic stand-ins that need no model download.
+That keeps pull requests fast, but it means a whole class of regression is
+invisible per-PR: a prompt a real model cannot satisfy, a JSON schema the
+adapter mis-serialises, a debrief that generates prose but never a score.
+
+The nightly real-model smoke closes that gap. It plays a scripted conversation
+on a **real local model** end-to-end — registry download → llama.cpp →
+`convsim-core` → scored debrief — and fails with a classified verdict.
+
+| | |
+|---|---|
+| Workflow | [`.github/workflows/model-smoke-nightly.yml`](../.github/workflows/model-smoke-nightly.yml) |
+| Harness | [`scripts/nightly-model-smoke.py`](../scripts/nightly-model-smoke.py) |
+| Unit tests (per-PR) | `tests/scripts/test_nightly_model_smoke.py` |
+| Schedule | 04:00 UTC daily, plus **Run workflow** (`workflow_dispatch`) |
+| Runner | GitHub-hosted `ubuntu-latest`, CPU-only inference |
+| Model | registry role `starter` — Qwen3 4B Instruct Q4\_K\_M, ~2.5 GB, Apache-2.0 |
+| Runtime budget | **< 30 min** (`timeout-minutes: 30`); the harness self-limits to 20 min |
+
+> Nightly, not per-PR, on purpose: a 2.5 GB download plus ~11 min of CPU-only
+> inference cannot sit in the PR path. The fake-runtime counterpart is the
+> playthrough in `tests/e2e/test_scripted_playthrough.py` — run at release time
+> by `scripts/release-smoke.sh`, not on pull requests — which uses the *same
+> scripted player turns*, so the two cover the same conversation shape against
+> different runtimes. What *does* run per-PR is this harness's own unit tests.
+
+---
+
+## What the nightly proves
+
+1. The pinned starter model downloads and its bytes match the registry SHA-256.
+2. `llama-server` loads the GGUF and `convsim-core` reports
+   `runtime_id == "llama_cpp"` — the smoke refuses to run on a fake runtime.
+3. A real model drives a multi-turn conversation on the built-in
+   `behavioral_interview` scenario, producing NPC turns that satisfy
+   `turn-output.schema.json`.
+4. The session ends and produces a **scored** debrief: at least one rubric
+   dimension scored, a numeric `overall_score` in `[0, 100]`, and a non-empty
+   summary. The scores come from the rubric observations the model volunteered
+   on each turn, so this asserts the scoring path; the debrief *narrative* is a
+   separate model call, and a run where it fell back to the deterministic
+   template still passes — with a warning that says so, because the scores are
+   unaffected and one unlucky generation must not red the nightly. Recurring
+   across nightlies is a regression in the debrief prompt or its schema.
+5. End-to-end turn latency stays within the documented budget, scaled for CI
+   hardware.
+
+The NPC *opening* line is authored scenario text, not a generation — so the
+scripted player turns, not the opening, are what prove the model is working.
+
+A turn that is real model output is not automatically a *reply*, and the
+per-turn parse flags cannot tell the difference. All twelve nightly runs of the
+previous single-turn harness whose logs GitHub still held on 2026-10-03 logged
+an NPC answer whose leading characters were byte-identical to the scenario's
+`opening_npc_says` — the model reciting the opening question back instead of
+answering it. `used_fallback` stays false for those turns (the utterance is
+neither empty nor the canned safe one), so the run went green on a conversation
+that never happened. The harness therefore records `replayed_opening` per turn
+and applies the same policy it applies to fallbacks: one such turn warns,
+*every* generated turn doing it fails, because then the model answered nothing
+and the run proves nothing.
+
+Read that twelve-for-twelve with its cause attached, because it is not evidence
+that the starter model always recites. The single player turn those runs
+submitted was `"Reply with exactly one sentence: Hello, I am ready."` — an instruction
+addressed to the model rather than something an interview candidate says — and
+this job replaces it with the three in-character turns in
+`SCRIPTED_PLAYER_TURNS`, which the real starter model answered properly in local
+verification. So the recital is a known failure mode of the *old* prompt, and
+the baseline rate under the new script is simply not yet known: expect zero, and
+treat the first recital warning as new information rather than as the documented
+status quo.
+
+The two policies also compose, because a conversation can be *split* between
+them — one turn recites the opening, the others fall back — and trip neither
+"all of them" check while still containing no reply at all. Each check warns
+rather than fails on its own only because a run that recovers from one bad turn
+is the product working; when nothing recovered there is nothing to credit, so a
+run with no turn that is both real model output *and* an answer fails as well.
+
+What counts as a recital is a *leading prefix* match, `REPLAY_PREFIX_CHARS`
+(40 collapsed characters) of the authored opening reproduced at the head of the
+reply — not the whole opening. Those nightly logs printed only `npc_text[:80]`
+against a 122-character opening, so what they establish is that the reply
+*began* as the opening; whether the copy ran to the end or veered into the
+model's own words after a sentence or two was never recorded, and a check that
+demanded the entire line would have gone unreachable on the second case. Forty
+characters of the scenario's own text at the head of an answer is not something
+a reply arrives at by chance — the transcript the turn prompt renders is the
+only place it can come from — and the threshold is capped at the opening's own
+length, so a scenario with a terser opening still has to have it recited in
+full.
+
+If the NPC closes the conversation before the script runs out
+(`session_control.continue_session`), the run still passes on the turns it did
+play — that is the product working as designed — but it records
+`scripted_turns_played` and warns in the step summary, so a green run on one
+turn is not mistaken for a green run on three.
+
+---
+
+## Failure classification
+
+A red nightly must be triageable from the job summary alone, so every failure is
+attributed to exactly one class, with its own exit code, its own banner in the
+log, and a remedy printed next to it:
+
+| Exit | Class | Means | First thing to do |
+|---|---|---|---|
+| 1 | `budget` | Pipeline worked; latency regressed past the CI ceiling | Compare `measured_ms` in the report artifact against recent nightlies |
+| 2 | `download` | Model could not be fetched — network, HTTP, a transfer that stalled past its 5 min budget, or an empty cache | `python scripts/validate-registry.py --url-check`, then re-run |
+| 3 | `checksum` | **SHA-256 drift**: on-disk bytes ≠ `model-registry/registry.yaml` | See [Checksum drift](#checksum-drift) — never relax the check |
+| 4 | `runtime` | `llama-server` or `convsim-core` crashed, hung, or returned 5xx — or one of the two ports was already taken, so neither could be started | Read the child stderr tail printed above the banner — but only if a child actually exited. The banner says which case it is and carries the matching remedy: a port conflict started no child (see [Running it locally](#running-it-locally)), and a 5xx from a server that is still running is usually [a cut-off generation](#a-5xx-from-a-live-server), not a crash |
+| 5 | `pipeline` | Servers healthy, but an end-to-end assertion failed | Inspect the per-turn `used_fallback` and `replayed_opening` flags in the report artifact — except for an unscored debrief, see [Unscored debrief](#unscored-debrief) |
+| 6 | `timeout` | Wall-clock budget exhausted; the failing phase is named | Check `phase_durations_s` before raising the budget |
+
+The distinction that matters most in practice is **2/3 vs 4 vs 5**: a download or
+checksum failure says nothing about the app, a runtime failure is a crash, and a
+pipeline failure means the model ran and produced output the product rejected.
+
+Anything the harness could not carry out — a `registry.yaml` that is malformed,
+names no usable `starter` model, or still carries the schema's `PENDING`
+placeholder instead of a pinned URL and checksum; a bad command line; a disk
+that fills up mid-run; a bug in the harness itself — is also reported as exit 5,
+with a message and a remedy that say so, so that **exit 1 only ever means a
+latency regression**. Nothing exits with an unclassified traceback, and nothing
+borrows a class's exit code without correcting the advice printed beneath it: a
+broken registry entry is *not* exit 2, because `download`'s remedy ends "re-run
+the job" and no re-run will repair a file in the repository.
+
+When more than one class could apply, the harness reports the strongest evidence
+rather than the symptom the client happened to see:
+
+1. **A crashed child wins.** A dead `convsim-core` reads as `runtime`, not as the
+   connection reset it caused downstream.
+2. **An exhausted wall clock beats a transport error.** Each request timeout is
+   capped to the budget that remains, so the last call before the deadline dies
+   client-side and *looks* like an unresponsive server. It is reported as
+   `timeout`, whose remedy points at `phase_durations_s`.
+
+Because `timeout` outranks `budget`, a *uniform* slowdown normally surfaces as
+exit 6 rather than exit 1: three turns at the 240 s CI ceiling plus a debrief
+allowed twice that very nearly fill the 20 min budget, so a ~2x regression
+exhausts the clock before the budget phase runs. The class stays `timeout` — the
+clock really did run out — but when the turns that *did* complete already have a
+median past the ceiling, the verdict says so and names the measurement, so exit 6
+is not mistaken for a hang. Exit 1 remains the verdict when the conversation
+finishes inside the budget but too slowly.
+
+Each run writes the verdict, failure class, remedy, measured latencies and
+per-phase durations to the GitHub **step summary**, and uploads the full JSON
+report as the `model-smoke-report` artifact (30-day retention).
+
+### A 5xx from a live server
+
+A 5xx is a `runtime` failure because the transport said so, but `runtime`'s
+stock advice — read the child stderr tail, suspect a runner OOM or a bad
+llama-cpp-python wheel — only fits a failure where a child actually died. Often
+none has, and then the tail above the banner shows a perfectly healthy server.
+The harness checks `poll()` on both children and, when neither has exited, prints
+a different remedy: *"Neither child process exited, so nothing crashed."*
+
+The cause worth knowing about is a generation the adapter cut off. `convsim-core`
+answers **504 `TURN_TIMEOUT`** when `CONVSIM_LLAMA_CPP_CHAT_TIMEOUT` runs out
+mid-reply. The harness deliberately sets that budget *above* the latency ceiling
+it is about to judge the turn on — `DEBRIEF_SLOWDOWN_FACTOR` × the ceiling, 540 s
+at `--ci-hardware-factor 20` — so that a slow turn is measured and reported as
+`budget` rather than killed and misreported as a crash. But that moves the cliff
+rather than removing it: a turn slower than 540 s is still cut off, and with
+1200 s of wall clock still unspent neither `timeout` nor `budget` takes the class
+over. So exit 4 with both children alive and a 504 in the message means inference
+roughly 5× slower than the nightlies have ever measured — a latency regression.
+Start from `measured_ms` and `phase_durations_s` in the report artifact, not from
+an OOM.
+
+The other way to land here with both children alive is a server that is up but
+wedged: `llama-server` still loading the GGUF when readiness ran out, or a
+request it refused. The stderr tails are the right place for that one.
+
+### Unscored debrief
+
+Exit 5 with *"Debrief has no rubric dimension scores"* is the one `pipeline`
+failure whose cause is ambiguous from the class alone, so the harness resolves it
+for you: each NPC turn's `rubric_observation_count` is recorded in the report
+artifact, totalled as `rubric_observations_seen`, and the failure message names
+which of the causes below applies.
+
+The debrief's dimension scores are accumulated entirely from
+`rubric_observations` that the model volunteers on each NPC turn. Nothing asks
+it for them: no prompt layer names any rubric dimensions, and the only hint the
+model gets is the bare `rubric_observations` array in the embedded output
+schema — whose empty list the schema accepts. Nor could a scenario supply one.
+`ScenarioData` (in `convsim_prompt.types`) has no rubric field at all, so the
+official `job-interview-basic` pack's `behavioral_interview` — which *does*
+define a rubric — would compose the identical turn prompt; and the full edition
+resolves the built-in catalogue before installed packs, so this job always plays
+the rubric-less built-in scenario regardless. The fake runtime always returns
+`[]`, which is why the release-time playthrough asserts only that `scores` *is*
+a dict.
+
+That thin prompt coverage is the standing weakness that makes an unscored
+debrief reachable at all. It is *not*, however, the expected outcome: in local
+verification against the real starter model — Qwen3 4B Q4\_K\_M, the same pin
+the nightly uses, under the turn-output schema — every one of six scripted turns
+across two runs volunteered at least one observation, and the debrief scored
+three to four dimensions with an `overall_score` near 50. Treat zero
+observations as a signal, not as the resting state.
+
+So an unscored debrief means one of four quite different things:
+
+- **`rubric_observations_seen` is 0 and every generated turn fell back.** Then
+  this failure is downstream of the fallback failure reported beside it and
+  carries no independent signal: a turn that fell back returns the canned safe
+  utterance and an empty `rubric_observations` list whatever the prompt asked
+  for, so the debrief would have had nothing to accumulate even with a rubric
+  layer in place. Start from the per-turn `parse_events` in the `/debug`
+  payload — they say why the model's output was rejected — and judge the
+  scoring path on the next run. The harness detects this case and says so
+  rather than printing the next bullet's note, which would be the wrong
+  investigation.
+- **`rubric_observations_seen` is 0 with turns that did not fall back — the
+  model volunteered nothing.** Only the prompt's bare schema hint was ever
+  asking, so this is reachable by design —
+  but since the starter model does normally answer that hint, first check what
+  changed about what reaches the model: the `OUTPUT_SCHEMA` prompt layer, the
+  registry's `starter` pin, the adapter's JSON-schema constraint, sampling
+  settings. Then fix the weakness itself in the product — the turn prompt needs
+  a rubric layer *and* a rubric to put in it, since `ScenarioData` carries none
+  — not by relaxing the assertion, and not by pointing the smoke at a
+  rubric-defining scenario, which would compose the same prompt.
+- **`rubric_observations_seen` is above 0 — a real regression.** The turns
+  returned observations and the debrief scored none of them, so the model did
+  its part. The debrief engine does not score the validated observations the
+  turn pipeline handed it: `_parse_rubric_observations` in
+  `services/convsim-core/.../debrief_engine.py` re-reads each NPC turn's *stored
+  raw model output* with a plain `json.loads`, so output that needed repair — or
+  that arrived pre-parsed from the runtime adapter — loses its observations on
+  the way to the debrief. Compare `raw_npc_output` from the `/debug` payload
+  against the per-turn `rubric_observation_count` in the artifact.
+- **`rubric_observations_seen` is `null` — the harness could not tell.** At
+  least one generated turn carried no readable `rubric_observations` list, and
+  the turns it *could* read carried nothing between them, so the total cannot be
+  trusted as a total and the message quotes the two causes it cannot tell
+  apart — volunteered nothing, or volunteered something the debrief lost —
+  rather than picking one. Check `rubric_observation_count` per turn in the report to see
+  which turns were unreadable (`null`) and which really were zero; `null` is not
+  "every turn was unreadable". `convsim-core` always sends that list today, so
+  any `null` is itself worth chasing: the turn response contract changed. The
+  harness deliberately does not round an unreadable payload down to zero — that
+  would have the verdict assert the model volunteered nothing when the turn it
+  could not read may have carried plenty.
+
+In every case the run is red: *"scored debrief"* is the acceptance criterion for
+[#457](https://github.com/outrightmental/ConversationSimulator/issues/457).
+No cause is a reason to weaken the check. The one that is *also* a standing
+product gap is **zero observations with turns that did not fall back**: that one
+is worth a tracking issue for the missing rubric prompt layer *and* an
+investigation of the run that hit it. The all-fell-back case is not — it says
+nothing about rubric prompting, and the harness says so rather than sending you
+after the prompt layer.
+
+---
+
+## Runtime budget
+
+Target: **under 30 minutes on a standard GitHub-hosted runner.** The job sets
+`timeout-minutes: 30` as a hard ceiling, and the harness runs with
+`--wall-clock-budget-s 1200` (20 min) so *it* fails first and names the phase
+that ran long — a GitHub-side timeout would only say "the operation was
+canceled".
+
+The 10-minute gap between the two is deliberate margin, not an accounting of
+known work. `timeout-minutes` covers the whole job, and the harness only starts
+after checkout, three `pip install` steps, the model cache restore and — on a
+cache miss — the download and the cache save that follows it, which together
+measure **31–51 s** on the nightlies run so far, not the several
+minutes a 2.5 GB transfer sounds like. The gap is sized for the part of that
+work that is network-bound rather than for its measured cost: a slow
+Hugging Face or Actions-cache night can cost minutes that the harness's own
+clock never sees, and the whole design exists so that a slow night still
+produces an attributed verdict rather than a job cancellation.
+
+On `ubuntu-latest` (CPU-only, 4B Q4\_K\_M):
+
+| Phase | Cold cache | Warm cache |
+|---|---|---|
+| Set up job + checkout + `setup-python` | ~5 s | ~5 s |
+| `pip install` ×3 (prompt-composer, convsim-core, llama-cpp-python wheel) | ~16 s | ~14 s |
+| Model cache restore (2.3 GiB) | — (miss: <1 s) | ~24 s |
+| Model download from Hugging Face + SHA-256 verify (2.3 GiB) ◊ | ~10 s | — |
+| Cache save (2.3 GiB, its own step before the smoke) | ~10 s | — |
+| SHA-256 re-verify before the weights are loaded ‡ | <10 s | <10 s |
+| `llama-server` model load + `convsim-core` start + session create | ~8 s | ~8 s |
+| 3 scripted turns † | ~6 min | ~6 min |
+| Debrief generation † | ~4 min | ~4 min |
+| **Total** | **~11 min** | **~11 min** |
+
+The non-inference rows are measured, from the job's own logs for the twelve
+nightlies whose logs GitHub still held on 2026-10-03 — the cold-cache rows from
+the one of those that missed the cache. Two things in them are worth knowing
+before tuning anything:
+
+- **A cache miss costs no more than a cache hit.** Fetching the GGUF from
+  Hugging Face and hashing it takes ~10 s, and saving it to the Actions cache
+  another ~10 s; restoring the same bytes on a hit takes 19–28 s. Caching the
+  model saves the *network*, not the clock, and abandoning a poisoned cache
+  entry costs nothing.
+- **Everything before the conversation is noise.** All of it together measured
+  31–51 s across those twelve runs, in a 30 min job. The budget is inference,
+  and only inference.
+
+◊ The download is the one pre-smoke step that could stretch without limit, and
+it runs where the harness's own deadline cannot see it, so it carries its own
+5 min budget (`DOWNLOAD_BUDGET_S`) with a 2 min cap on any single quiet stretch
+(`DOWNLOAD_READ_TIMEOUT_S`). `urlopen`'s timeout is per socket read, not per
+transfer, so a connection that goes quiet repeatedly — or merely crawls — would
+otherwise spend the whole margin above and then hand the smoke a job clock with
+no room in it, which is the one way left to reach an unattributable
+cancellation. Exceeding either bound is exit 2 with its own remedy. 5 min is
+~30× the measured 10 s, so a transfer has to be pathologically slow to hit it.
+
+‡ The harness re-verifies the on-disk file at the start of every run, which the
+previous single-turn harness did not, so this row has no direct measurement.
+The bound comes from the cold-cache row above: downloading 2.3 GiB *and*
+hashing it took 10 s in total, so the hash alone is a few seconds.
+
+† The two inference rows are *projected*, because the multi-turn conversation
+and the debrief have never run on a runner. The projection comes from the only
+latency this job has measured — one behavioral-interview turn, twelve
+nightlies: 66 / 71 / 93 / 94 / 98 / 103 / 104 / 111 / 112 / 114 / 115 / 115 s —
+rounded up to ~120 s per turn, since turns 2 and 3 render a longer transcript
+than turn 1, and a
+debrief allowed twice a turn (`DEBRIEF_SLOWDOWN_FACTOR`). Replace both rows with
+the real `phase_durations_s` from the first green nightly's report artifact.
+
+At ~11 min projected against a 20 min harness budget there is roughly 2×
+headroom, and the harness's clock covers only the rows from the re-verify
+downwards.
+
+The model is cached between runs under the key
+`model-gguf-v1-<registry-sha256>`, so the download only recurs when the registry
+pin changes. Cache *restore* and *save* are separate steps and the save is gated
+on `success()`, so a file that fails verification is never written to the cache.
+
+If the smoke step creeps past ~15 min — or the job past ~25 min — shorten
+`SCRIPTED_PLAYER_TURNS` rather than raising either ceiling: a nightly that
+routinely runs near its timeout flaps.
+
+### Latency budget on CI hardware
+
+CPU-only CI hardware is far slower than the mid-spec reference machine the
+product budgets target ([docs/performance.md](performance.md)), so the harness
+scales the documented budget:
+
+```
+CI ceiling = documented budget × CI_HARDWARE_FACTOR × REGRESSION_TOLERANCE
+           = 10 000 ms        × 20                 × 1.20   = 240 000 ms
+```
+
+`CI_HARDWARE_FACTOR` (20) is calibrated empirically: a full behavioral-interview
+turn on this runner has measured 66–115 s across the twelve nightlies whose
+logs GitHub still held on 2026-10-03 (median ~104 s), i.e. ~11.5× the 10 s
+documented budget at the slow end. The factor is set from that slow end, not the
+median, and still leaves
+roughly 2× headroom — note the 66 s run against the 115 s one, which is the
+night-to-night spread a single-sample calibration would have missed. A genuine
+>2× regression still fails. The headline `full_response_ms` is the **median** of
+the scripted turns, so one unlucky turn cannot flap the job either.
+
+This factor models CI slowness only — the product's 10 s target-hardware budget
+is unchanged. Re-measure and re-tune it if the runner class or the starter model
+changes.
+
+`session_start_ms` and `debrief_ms` are measured and reported but **not**
+budget-checked: the NPC opening is authored text rather than inference, and
+debrief generation has no documented SLO.
+
+---
+
+## Checksum drift
+
+The checksum is verified in two places: right after a fresh download, and again
+at the start of every run (including cache hits) before the weights are loaded.
+A cache hit with drifted bytes therefore fails loudly instead of silently
+feeding a corrupt model to the smoke.
+
+Exit 3 means one of two things:
+
+- **The pinned upstream file was replaced.** All registry URLs are pinned to a
+  specific Hugging Face revision, so this should not happen silently — if it
+  did, review the new file and re-pin with `scripts/pin-model.py`.
+- **The cached download is corrupt or truncated.** The harness deletes the bad
+  local file, but Actions cache entries are immutable, so a poisoned entry would
+  be restored again on the next run. Delete it
+  (`gh cache delete model-gguf-v1-<sha>`) or bump `MODEL_CACHE_PREFIX` in the
+  workflow.
+
+Never "fix" a drift by updating the expected hash to whatever is on disk — that
+is exactly the check this job exists to perform.
+
+What exit 3 does *not* mean is that the digest it was handed is unusable. The
+expected digest is normalised (case, surrounding whitespace) and then checked to
+be 64 hex characters before anything is hashed; one that is not — a value copied
+one character short out of a terminal, the realistic mistake when running the
+repro below by hand — is exit 5 with its own remedy, and the model file is left
+exactly where it was. Nothing no file's hash could ever match is allowed to cost
+a 2.5 GB download or to be reported as upstream drift.
+
+---
+
+## Running it locally
+
+You need the starter model on disk and `llama-cpp-python[server]` installed.
+Nothing here touches the network except the one-time model download.
+
+```bash
+# 1. Install the harness dependencies
+pip install -e "packages/prompt-composer[dev]"
+pip install -e "services/convsim-core[dev]"
+pip install "llama-cpp-python[server]" \
+  --only-binary llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+
+# 2. Resolve the starter model's id / url / sha256 from the registry
+python scripts/nightly-model-smoke.py --print-registry-model starter
+
+# 3. Download it (~2.5 GB) into ~/.convsim/models/llm/, verifying the checksum
+python scripts/nightly-model-smoke.py --download-only \
+  --model-id qwen3-4b-instruct-q4_k_m \
+  --model-url "<model_url from step 2>" \
+  --model-sha256 "<model_sha256 from step 2>"
+
+# 4. Run the smoke. --ci-hardware-factor 1 holds your machine to the real
+#    product budget; use 20 to reproduce what CI accepts.
+python scripts/nightly-model-smoke.py \
+  --model-id qwen3-4b-instruct-q4_k_m \
+  --model-sha256 "<model_sha256 from step 2>" \
+  --ci-hardware-factor 1 \
+  --report-path /tmp/smoke-report.json
+```
+
+Useful extras:
+
+- `--verify-only` — re-verify an already-downloaded model and exit. Like every
+  other checksum check here, a mismatch **deletes** the file (exit 3), so a
+  re-run re-downloads rather than re-verifying the same bad bytes — do not
+  reach for it as a read-only inspection of a 2.5 GB download you want to keep.
+  A `--model-sha256` that is not 64 hex characters is rejected as exit 5 before
+  the file is touched, so a mistyped digest costs you nothing.
+- `--models-dir <dir>` — look for `<model-id>.gguf` somewhere other than
+  `~/.convsim/models/llm/`.
+- `--wall-clock-budget-s` — the self-imposed deadline (default 1200 s).
+
+The harness binds `llama-server` on port 7356 and `convsim-core` on port 7399,
+and refuses to start (exit 4) if either is already taken. That is deliberate
+rather than fussy: readiness is a URL poll, so a server the harness did not
+start would answer it, the child that lost the bind would die unnoticed, and the
+run would report a **pass** for a model it never checksum-verified. Stop whatever
+owns the port — do not work around it.
+
+To exercise the harness's own logic without a model:
+
+```bash
+python -m pytest tests/scripts/ -v
+```
+
+---
+
+## Privacy
+
+The conversation is 100 % scripted (`SCRIPTED_PLAYER_TURNS` in the harness) and
+contains no user data, so short NPC excerpts are printed to the log and recorded
+in the report artifact — they are the evidence that a real model, rather than a
+canned fallback, drove the conversation. Sessions are created with
+`save_transcript: false`, so no transcript file is written, and `convsim-core`
+runs against a throwaway data directory that is deleted when the run ends.
+
+---
+
+## Related
+
+- [Offline smoke tests](offline-smoke-tests.md) — proves no cloud service is
+  contacted during play (fake runtime, per-PR).
+- [Voice smoke tests](voice-smoke-tests.md) — STT/TTS sidecar checks.
+- [Performance and hardware tiers](performance.md) — where the latency budgets
+  come from.
+- [Release checklist](release-checklist.md) — Part F covers manual real-model
+  verification of a packaged build.
diff --git a/scripts/nightly-model-smoke.py b/scripts/nightly-model-smoke.py
old mode 100644
new mode 100755
index 2e43bba7..1bb67b4b
--- a/scripts/nightly-model-smoke.py
+++ b/scripts/nightly-model-smoke.py
@@ -1,42 +1,98 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
-"""Nightly real-model smoke test for latency budget regression.
-
-Downloads the pinned starter model (if not already cached), starts a local
-llama-server (llama-cpp-python's OpenAI-compatible server) against it, starts
-convsim-core wired to that server, runs a scripted end-to-end session through
-convsim-core's REST API, measures full-response latency, and fails if it
-exceeds the documented budget multiplied by ``--ci-hardware-factor``.
-
-convsim-core's turn endpoint is synchronous — it returns the completed NPC
+"""Nightly real-model smoke test (issue #457).
+
+Exercises the whole play pipeline against a **real local model** — no fake
+runtime anywhere — and fails with a *classified* verdict so a red nightly is
+triageable from the job summary alone.
+
+What a full run does
+--------------------
+1. **download** — fetch the smallest registry model (role ``starter``:
+   Qwen3 4B Instruct Q4_K_M) unless the CI cache already holds it.
+2. **checksum** — hash the on-disk file and compare it to the registry
+   ``sha256``.  This runs on cache hits too, so a drifted or truncated cached
+   file fails loudly instead of silently feeding a corrupt model to the smoke.
+3. **runtime** — start llama-cpp-python's OpenAI-compatible server on the GGUF
+   and start ``convsim-core`` wired to it (``CONVSIM_RUNTIME_ID=llama_cpp``).
+4. **conversation** — create and start a session on the built-in
+   ``behavioral_interview`` scenario (the NPC opening is authored scenario text,
+   not a generation), then submit a scripted multi-turn player conversation.
+   Every NPC *reply* is generated by the real model under the turn-output JSON
+   schema.
+5. **debrief** — end the session and generate the debrief, asserting a *scored*
+   debrief comes back (non-empty rubric dimensions, numeric overall score,
+   non-empty summary).
+6. **budget** — compare measured latencies against the documented budgets
+   scaled by ``--ci-hardware-factor``.
+
+``convsim-core``'s turn endpoint is synchronous — it returns the completed NPC
 turn in a single response rather than streaming ``npc.token`` events (token
 streaming is added by the ``apps/api`` gateway, which is not exercised here).
 The smoke therefore measures full end-to-end turn latency (which subsumes
-time-to-first-token) against the ``FULL_RESPONSE_MS`` budget.
-
-A budget regression is defined as measured_ms > ci_budget_ms * 1.20.
+time-to-first-token) against the ``full_response_ms`` budget.
+
+Failure classification
+----------------------
+Every failure is attributed to one class, printed in a banner, written to the
+JSON report (``failure_class``) and the GitHub step summary, and mapped to a
+distinct exit code:
+
+===========  ====  =======================================================
+class        exit  meaning
+===========  ====  =======================================================
+``budget``      1  latency regression: the pipeline worked, just too slowly
+``download``    2  model could not be fetched (network / HTTP / stalled
+                   transfer / missing cache)
+``checksum``    3  SHA-256 drift: on-disk bytes ≠ registry ``sha256``
+``runtime``     4  llama-server or convsim-core crashed, hung, or 5xx'd
+``pipeline``    5  servers healthy but an end-to-end assertion failed
+``timeout``     6  wall-clock budget exhausted (phase named in the message)
+===========  ====  =======================================================
+
+Exit 5 is also the catch-all for everything the harness could not carry out —
+a bad command line, an unusable ``registry.yaml``, a full disk, a bug in this
+file — so that exit 1 never means anything but a latency regression.  Those
+failures carry their own remedy instead of ``pipeline``'s, because "inspect the
+per-turn used_fallback flags" is the wrong first move for any of them.
+
+``timeout`` outranks ``budget``, and the two thresholds are close together by
+construction (three turns at the CI ceiling plus a debrief very nearly fill the
+wall-clock budget), so a uniform latency regression usually exhausts the clock
+before the budget phase runs.  The class stays ``timeout``, but the verdict then
+names the measured turn latencies that say it was a regression rather than a
+hang — see ``timeout_latency_evidence``.
+
+Transcript content is fully scripted (see ``SCRIPTED_PLAYER_TURNS``) and holds
+no user data, so short NPC excerpts are logged and reported — they are the
+evidence that a real model, not a canned fallback, drove the conversation.
+``save_transcript`` is false, so no transcript file is written.
 
 Usage
 -----
-Download only (called from CI before the model cache is populated)::
+Resolve the starter model from the registry (writes ``$GITHUB_OUTPUT``)::
 
-    python scripts/nightly-model-smoke.py \\
-        --download-only \\
-        --model-url <url> \\
-        --model-sha256 <hex> \\
-        --model-id <id>
+    python scripts/nightly-model-smoke.py --print-registry-model starter
+
+Download only (cache miss)::
 
-Full smoke (model already on disk)::
+    python scripts/nightly-model-smoke.py --download-only \\
+        --model-id <id> --model-url <url> --model-sha256 <hex>
+
+Verify an already-cached file without running the smoke (a mismatch *deletes*
+the file — see ``verify_model_checksum``)::
+
+    python scripts/nightly-model-smoke.py --verify-only \\
+        --model-id <id> --model-sha256 <hex>
+
+Full smoke::
 
     python scripts/nightly-model-smoke.py \\
         --model-id qwen3-4b-instruct-q4_k_m \\
-        --ci-hardware-factor 6 \\
+        --model-sha256 <hex> \\
+        --ci-hardware-factor 20 \\
+        --wall-clock-budget-s 1200 \\
         --report-path /tmp/smoke-report.json
-
-Exit codes
-----------
-0  All latency budgets met (within 20 % CI tolerance).
-1  One or more budgets exceeded, or the server failed to start / respond.
 """
 
 from __future__ import annotations
@@ -45,6 +101,9 @@
 import hashlib
 import json
 import os
+import shutil
+import socket
+import statistics
 import subprocess
 import sys
 import tempfile
@@ -54,20 +113,27 @@
 import urllib.request
 from collections import deque
 from pathlib import Path
-from typing import Optional
+from typing import Any, Dict, List, NoReturn, Optional, Sequence
 
 REPO_ROOT = Path(__file__).parent.parent
-
-# Documented latency budgets (ms) from packages/shared/src/types/metrics.ts.
-# Keep in sync manually; the script is intentionally not importing TS source.
-# Only full_response_ms is measurable through convsim-core's synchronous REST
-# turn API; TTFT requires token streaming (apps/api gateway), which this smoke
-# does not stand up.
+REGISTRY_PATH = REPO_ROOT / "model-registry" / "registry.yaml"
+
+# Documented latency budgets (ms) from docs/performance.md and the
+# LATENCY_BUDGETS constant in packages/shared/src/types/metrics.ts.
+# Keep in sync manually; the script intentionally does not import TS source.
+#
+# Only full_response_ms is budget-checked here:
+#   - TTFT needs token streaming (the apps/api gateway), which this smoke does
+#     not stand up, and full_response_ms subsumes it;
+#   - session_start_ms and debrief_ms are measured and reported but NOT checked.
+#     The NPC opening is authored scenario text, not a generation, so session
+#     start is not inference-bound and the CI hardware factor does not apply to
+#     it; debrief generation has no documented budget (docs/performance.md).
 BUDGETS_MS = {
     "full_response_ms": 10_000,
 }
 
-# Regression tolerance: measured may exceed CI budget by at most this factor.
+# Regression tolerance: measured may exceed the CI budget by at most this factor.
 REGRESSION_TOLERANCE = 1.20
 
 MODELS_DIR = Path.home() / ".convsim" / "models" / "llm"
@@ -76,65 +142,718 @@
 LLAMA_SERVER_PORT = 7356  # convsim-core's default CONVSIM_LLAMA_CPP_BASE_URL port
 CORE_PORT = 7399
 
-# A simple scripted player turn that any instruct model can answer in <30 tokens.
-SMOKE_PLAYER_TURN = "Reply with exactly one sentence: Hello, I am ready."
+# Wall-clock budget for the whole run, in seconds.  The script — which knows
+# which phase it is in — must fail first with an attributed verdict instead of
+# GitHub killing the job with an unattributable "The operation was canceled",
+# so this has to clear the job's timeout-minutes *minus the steps that run
+# before the script does*: checkout, pip install, cache restore and (on a cache
+# miss) the download plus the cache save that follows it.  Those measure ~1 min
+# in total on the nightlies run so far — not the several minutes a 2.5 GB
+# transfer sounds like — so 20 min inside a 30 min job leaves ~9 min of real
+# margin for the network-bound part of them, and leaves this deadline the first
+# one to trip.  Against the ~11 min the run itself is projected to take, 20 min
+# is also roughly 2× headroom.
+# See docs/real-model-smoke.md for the measured breakdown.
+DEFAULT_WALL_CLOCK_BUDGET_S = 1200.0  # 20 min
+
+# Budget for the whole model transfer, in seconds, and for one quiet stretch of
+# it.  The download runs in its own `--download-only` step, *before* the smoke,
+# so it burns the job clock where the Deadline above cannot see it — which makes
+# an unbounded transfer the one remaining route to the unattributable "The
+# operation was canceled" that every other budget here exists to replace.
+#
+# urllib's timeout is per socket operation, not per transfer, so the 600 s it
+# used to be carried bounded nothing: a connection that went quiet repeatedly,
+# or merely crawled, could spend the entire ~9 min of margin between the
+# harness's 20 min and the job's 30 min and then hand the smoke a job clock with
+# no room left in it.  Both numbers are therefore sized against that margin
+# rather than against the transfer: 300 s is ~33× the 9 s the measured
+# nightlies take to fetch *and* hash 2.3 GiB, and leaves ~4 min of the margin
+# untouched, while 120 s of silence is already far past anything a live
+# Hugging Face connection does.
+#
+# Exceeding either is a `download` failure, whose remedy — the registry URL
+# check, then re-run — is the right advice for a transfer that stalled.
+DOWNLOAD_BUDGET_S = 300.0
+DOWNLOAD_READ_TIMEOUT_S = 120.0
+
+# Scripted player turns — deliberately the same script as the fake-runtime
+# playthrough in tests/e2e/test_scripted_playthrough.py (run at release time by
+# scripts/release-smoke.sh), so the two smokes exercise the same conversation
+# shape against different runtimes.
+SCRIPTED_PLAYER_TURNS = (
+    "I have five years of experience in software development.",
+    "My biggest achievement was building a cross-team API platform.",
+    "I believe in clear documentation and knowledge sharing.",
+)
+
+SCENARIO_ID = "behavioral_interview"
+
+# Longest NPC excerpt copied into the log / report (scripted content only).
+EXCERPT_CHARS = 160
+
+# How much of the download has to complete before another progress line is
+# printed.  A terminal collapses the "\r" updates in place, so per-megabyte
+# progress costs nothing there — but CI captures this script through a pipe with
+# no terminal, where every flush becomes its own log line.  The one nightly in
+# retention that missed the model cache logged 2381 of them for a 2.3 GiB GGUF,
+# which is the wall of output a reader has to scroll past to reach the
+# `download` or `checksum` verdict printed directly underneath it.  Eleven lines
+# still show a stalled transfer; 2381 hide the verdict they precede.
+PROGRESS_LOG_STEP_PCT = 10
+
+# A SHA-256 digest is 64 hex characters.  Anything else is not a digest the
+# on-disk bytes could ever match, so comparing against it is not a check — see
+# verify_model_checksum for why that distinction is worth 2.5 GB.
+SHA256_HEX_CHARS = 64
+_HEX_DIGITS = frozenset("0123456789abcdef")
+
+# A debrief summary shorter than this is not a usable coaching summary.
+MIN_SUMMARY_CHARS = 20
+
+# How much of the authored opening an NPC reply has to recite, leading, before
+# we stop calling it a reply.  See _replays_opening.
+#
+# Not "the whole opening": the evidence this check was built from is the old
+# harness's log line, which printed `npc_text[:80]` against a 122-character
+# opening, so what the nightlies actually establish is that the reply's first
+# 80 characters were the opening's first 80.  Whether the recital ran all the
+# way to "…interested in this role." or veered off into the model's own words
+# after the first sentence or two was never logged, and a check that demanded
+# the entire opening would have quietly failed to fire on the latter — a dead
+# guard on the exact behaviour it exists to catch.
+#
+# 40 collapsed characters is ~7 words of the scenario's own text reproduced
+# verbatim at the head of a reply ("Thanks for coming in today. I'm Alex Che"),
+# which no answer to a player turn arrives at by chance; the transcript the
+# turn prompt renders is the only place it can come from.  The threshold is
+# capped at the opening's own length below, so a scenario whose opening is
+# shorter than this keeps the strict whole-opening semantics.
+REPLAY_PREFIX_CHARS = 40
+
+# Appended to the "no scores" failure.  The debrief's dimension scores are
+# accumulated purely from rubric_observations the *model* volunteers, and
+# nothing asks it for them: no prompt layer names a rubric dimension (see
+# packages/prompt-composer/.../layers.py — the model's only hint is the bare
+# rubric_observations array in the embedded output schema, whose empty list the
+# schema accepts).  Nor is there any route by which a scenario could supply
+# one: convsim_prompt.types.ScenarioData carries no rubric field at all, so the
+# official job-interview-basic pack's behavioral_interview — which *does*
+# define a rubric — would compose the identical turn prompt, and the full
+# edition resolves the catalogue first anyway, so this job always plays the
+# rubric-less built-in.  Switching scenarios is therefore not a fix, and the
+# remedy must not offer it as one.  So `pipeline`'s stock advice — inspect the
+# per-turn used_fallback flags — would send triage looking for a parse failure
+# that did not happen.
+#
+# That unguided array is NOT normally empty, though, which is why this note
+# does not simply write the failure off: in local verification against the real
+# starter model (Qwen3 4B Q4_K_M under the turn-output schema) every one of six
+# scripted turns volunteered at least one observation, and the debrief scored
+# three to four dimensions.  Zero observations across a whole conversation is
+# therefore itself a change worth chasing — the schema layer no longer reaching
+# the model, a sampling or quantisation change, a different starter pin — not
+# the expected resting state.  Say both halves: the thin prompt coverage is the
+# standing weakness that makes this reachable at all, and a run that actually
+# hits it is a signal.
+#
+# Used when the run observed no rubric observations at all, and again when it
+# could not tell; when the turns *did* carry some, the cause is not ambiguous
+# and UNSCORED_WITH_OBSERVATIONS_NOTE applies instead.  The note therefore
+# explains the weakness without itself asserting that the array was empty —
+# the caller's own prefix says what the run actually observed, and a note that
+# announced "No NPC turn volunteered a single rubric observation" contradicted
+# the prefix on the branch that had just said it could not tell.
+UNSCORED_DEBRIEF_NOTE = (
+    "Nothing asks the NPC for rubric observations — "
+    "no prompt layer names the rubric dimensions, so an empty array satisfies "
+    "the turn schema — which is the standing product weakness that makes this "
+    "failure reachable. But an empty array is not the normal outcome: the real "
+    "starter model volunteered observations on every scripted turn in local "
+    "verification. So check what changed about what reaches the model (the "
+    "OUTPUT_SCHEMA layer, the starter model pin, sampling) before writing this "
+    "off, and fix the weakness itself in the product: the turn prompt needs a "
+    "rubric layer AND a rubric to put in it (ScenarioData carries no rubric "
+    "field, so pointing the smoke at a scenario that defines one composes the "
+    "same prompt). See the 'Unscored debrief' section of "
+    "docs/real-model-smoke.md."
+)
+
+# The other cause of an unscored debrief, and the one that *is* a regression.
+# The debrief engine does not score the validated observations the turn pipeline
+# returned: it re-parses each NPC turn's stored raw model output with a plain
+# json.loads (_parse_rubric_observations in debrief_engine.py) and reads
+# rubric_observations out of that.  So observations the turn accepted — because
+# the adapter handed back pre-parsed structured output, or the parser repaired
+# the text — are silently dropped when the stored raw text is not itself valid
+# JSON.  Saying so beats sending triage to diff last night's artifact.
+UNSCORED_WITH_OBSERVATIONS_NOTE = (
+    "This one IS a regression, and not an ambiguous one: the NPC turns returned "
+    "rubric observations, so the model did its part. The debrief engine scores "
+    "by re-parsing each NPC turn's stored raw model output rather than the "
+    "validated observations it was handed, so output that needed repair (or "
+    "arrived pre-parsed from the runtime) loses its observations on the way to "
+    "the debrief. Start at _parse_rubric_observations in "
+    "services/convsim-core/.../debrief_engine.py and compare raw_npc_output in "
+    "the /debug payload against the per-turn rubric_observation_count in the "
+    "report artifact."
+)
+
+# The third cause, and the one where the zero says nothing about rubrics at all.
+# A turn that fell back carries the canned safe utterance and an empty
+# rubric_observations list by construction, so a conversation in which *every*
+# generated turn fell back reaches the debrief with nothing to score no matter
+# how well the rubric were prompted.  UNSCORED_DEBRIEF_NOTE would send triage
+# after "the OUTPUT_SCHEMA layer, the starter model pin, sampling" — a different
+# investigation from the one the fallback failure in the same verdict is already
+# pointing at — and its "an empty array is not the normal outcome" is beside the
+# point when no array was ever produced.  evaluate_turns has already failed the
+# run for the fallbacks; this note says the unscored debrief is downstream of
+# that and carries no independent signal.
+UNSCORED_AFTER_FALLBACK_NOTE = (
+    "This is a consequence of the fallback failure above, not a second finding: "
+    "a turn that fell back to the canned safe utterance carries an empty "
+    "rubric_observations list whatever the prompt asked for, so with every "
+    "generated turn falling back the debrief had nothing to accumulate and would "
+    "have had nothing to accumulate even with a rubric layer in place. Fix the "
+    "fallbacks first — the per-turn parse_events in the /debug payload say why "
+    "the model's output was rejected — and judge the scoring path on the next "
+    "run. Do not start from the rubric prompting; this run says nothing about it."
+)
+
+# How much slower the debrief generation is allowed to be than one NPC turn
+# before we stop calling it slow and start calling it hung.  Only used to size
+# request timeouts (see run_smoke) — debrief latency itself is not budget-checked.
+DEBRIEF_SLOWDOWN_FACTOR = 2.0
+
+
+# ── Failure classification ────────────────────────────────────────────────────
+
+
+class FailureClass:
+    """Mutually exclusive failure classes, each with its own exit code."""
+
+    BUDGET = "budget"
+    DOWNLOAD = "download"
+    CHECKSUM = "checksum"
+    RUNTIME = "runtime"
+    PIPELINE = "pipeline"
+    TIMEOUT = "timeout"
+
+
+EXIT_CODES = {
+    FailureClass.BUDGET: 1,
+    FailureClass.DOWNLOAD: 2,
+    FailureClass.CHECKSUM: 3,
+    FailureClass.RUNTIME: 4,
+    FailureClass.PIPELINE: 5,
+    FailureClass.TIMEOUT: 6,
+}
 
+# What a human should do first for each class.  Printed in the failure banner
+# and the step summary so triage does not require reading this file.
+REMEDIES = {
+    FailureClass.BUDGET: (
+        "Latency regression, not a broken pipeline. Compare measured_ms in the "
+        "report artifact against recent nightlies. If the runner class or the "
+        "starter model changed, re-calibrate --ci-hardware-factor; otherwise "
+        "look for a prompt-size or sampling-parameter regression."
+    ),
+    FailureClass.DOWNLOAD: (
+        "The model could not be fetched. Check that the registry URL is still "
+        "live (scripts/validate-registry.py --url-check) and that Hugging Face "
+        "is reachable. Nothing about the app changed — re-run the job."
+    ),
+    FailureClass.CHECKSUM: (
+        "SHA-256 drift: the bytes on disk do not match model-registry/registry.yaml. "
+        "Either the pinned upstream file was replaced (re-pin with "
+        "scripts/pin-model.py and review the new file) or the cached download is "
+        "corrupt. The local copy has been deleted, but the Actions cache entry has "
+        "not: delete it too (`gh cache delete <key>`, or bump the cache-key prefix "
+        "in model-smoke-nightly.yml) or every re-run restores the same bad bytes. "
+        "Do NOT relax this check."
+    ),
+    FailureClass.RUNTIME: (
+        "llama-server or convsim-core crashed, hung, or returned 5xx. The child "
+        "stderr tail is printed above the banner. Likely causes: runner OOM on "
+        "model load, an incompatible llama-cpp-python wheel, or an unhandled "
+        "exception in the turn pipeline."
+    ),
+    FailureClass.PIPELINE: (
+        "Both servers were healthy but an end-to-end assertion failed: the model "
+        "ran and produced output the product rejects. Inspect the per-turn "
+        "used_fallback, native-structured-output and replayed_opening flags in "
+        "the report artifact — except for an unscored debrief, where the flags "
+        "are not where to start: the failure text above has already attributed "
+        "it, so read that and follow it. See the 'Unscored debrief' section of "
+        "docs/real-model-smoke.md."
+    ),
+    FailureClass.TIMEOUT: (
+        "The run exhausted its wall-clock budget in the phase named above. "
+        "Either inference got much slower or a phase hung. Check phase_durations_s "
+        "in the report artifact before raising --wall-clock-budget-s."
+    ),
+}
 
-# ── Download helpers ──────────────────────────────────────────────────────────
 
+class SmokeFailure(Exception):
+    """A failure with a known class, so CI can distinguish *why* the smoke failed.
 
-def _download_with_progress(url: str, dest: Path) -> None:
-    dest.parent.mkdir(parents=True, exist_ok=True)
-    print(f"  Downloading {url[:80]}…")
-    req = urllib.request.Request(url, headers={"User-Agent": "convsim-smoke/1.0"})
-    with urllib.request.urlopen(req, timeout=600) as resp, open(dest, "wb") as f:
-        total = int(resp.headers.get("Content-Length", 0)) or None
-        downloaded = 0
-        chunk = 1 << 20  # 1 MB
-        while True:
-            buf = resp.read(chunk)
-            if not buf:
-                break
-            f.write(buf)
-            downloaded += len(buf)
-            if total:
-                pct = int(downloaded / total * 100)
-                print(f"\r  {pct}% ({downloaded // 1_048_576} / {total // 1_048_576} MB)", end="", flush=True)
-    print()
+    ``remedy`` overrides the class default for the failures that borrow a
+    class's exit code without matching its usual cause.  Printing the class
+    remedy under those sends triage the wrong way — the whole point of the
+    classification is that the advice next to the banner is actionable.
+    """
 
+    def __init__(
+        self,
+        failure_class: str,
+        message: str,
+        *,
+        phase: Optional[str] = None,
+        remedy: Optional[str] = None,
+    ) -> None:
+        super().__init__(message)
+        self.failure_class = failure_class
+        self.phase = phase
+        self._remedy = remedy
+
+    @property
+    def exit_code(self) -> int:
+        return EXIT_CODES[self.failure_class]
+
+    @property
+    def remedy(self) -> str:
+        return self._remedy or REMEDIES[self.failure_class]
+
+
+# ── Wall-clock budget ─────────────────────────────────────────────────────────
+
+
+class Deadline:
+    """Tracks the run's wall-clock budget and which phase is consuming it."""
+
+    def __init__(self, budget_s: float) -> None:
+        self.budget_s = budget_s
+        self._start = time.monotonic()
+        self._phase = "startup"
+        self._phase_start = self._start
+        self.phase_durations_s: Dict[str, float] = {}
+
+    @property
+    def elapsed_s(self) -> float:
+        return time.monotonic() - self._start
+
+    @property
+    def remaining_s(self) -> float:
+        return self.budget_s - self.elapsed_s
+
+    @property
+    def phase(self) -> str:
+        return self._phase
+
+    def enter(self, phase: str) -> None:
+        """Check the budget, then close out this phase's duration and start ``phase``.
+
+        The check comes first, while ``self._phase`` is still the phase that has
+        been running: it is the one that consumed the budget, and so the one a
+        triager needs named.  Switching phases first would attribute the overrun
+        to a phase that has not executed a single instruction — reporting
+        "exhausted during phase 'debrief'" for a conversation that ran long.
+        """
+        self.check()
+        self._close_phase(phase)
+
+    def finish(self) -> Dict[str, float]:
+        """Close the books without checking the budget.
+
+        Called from ``run_smoke``'s ``finally``, where an exhausted budget has
+        already been classified: raising here would escape the handler and throw
+        the verdict away, turning a reportable timeout into a bare traceback.
+        """
+        self._close_phase("done")
+        return self.phase_durations_s
+
+    def _close_phase(self, next_phase: str) -> None:
+        now = time.monotonic()
+        self.phase_durations_s[self._phase] = round(
+            self.phase_durations_s.get(self._phase, 0.0) + (now - self._phase_start), 1
+        )
+        self._phase = next_phase
+        self._phase_start = now
+
+    def check(self) -> None:
+        if self.remaining_s <= 0:
+            raise SmokeFailure(
+                FailureClass.TIMEOUT,
+                f"Wall-clock budget of {self.budget_s:.0f} s exhausted during phase "
+                f"{self._phase!r} (elapsed {self.elapsed_s:.0f} s)",
+                phase=self._phase,
+            )
 
-def _verify_sha256(path: Path, expected: str) -> None:
+    def cap(self, timeout_s: float) -> float:
+        """Clamp a request timeout to what is left of the budget."""
+        self.check()
+        return max(5.0, min(timeout_s, self.remaining_s))
+
+
+# ── Registry ──────────────────────────────────────────────────────────────────
+
+
+# A registry that does not describe a usable model is a repository problem, not
+# a network one.  It borrows `pipeline`'s exit code — the class this harness
+# uses for everything it could not carry out — rather than `download`, whose
+# remedy ends "Nothing about the app changed — re-run the job": no number of
+# re-runs makes a missing `role: starter` entry appear, and the checksum the
+# cache key is built from never existed to drift.
+REGISTRY_REMEDY = (
+    "model-registry/registry.yaml does not describe a usable model for the "
+    "requested role, so the job never got as far as fetching anything. "
+    "Re-running cannot fix it: repair the registry entry (validate with "
+    "scripts/validate-registry.py) and merge."
+)
+
+# The placeholder the registry schema allows in `download.url` and
+# `download.sha256` for an entry that has been added but not yet pinned
+# (model-registry.schema.json documents both: "or 'PENDING' for pre-release
+# entries").  `scripts/validate-registry.py` fails on it — but only in
+# registry-nightly.yml, which does not run per-PR, so a `role: starter` entry
+# can reach main un-pinned.
+#
+# It has to be rejected here rather than passed along as a value, because both
+# fields carry it into a verdict that blames the wrong thing:
+#
+#   * sha256 "PENDING" makes the cache key `…-PENDING`, downloads 2.5 GB, and
+#     then fails `checksum` (exit 3) — whose remedy says the pinned upstream
+#     file was replaced and "do NOT relax this check" — while deleting the
+#     file it just fetched;
+#   * url "PENDING" raises out of urllib's Request constructor, which sits
+#     outside _download_with_progress's handler, so it lands in main's
+#     catch-all as "most likely a bug in the smoke harness" (exit 5).
+#
+# An un-pinned entry is neither. It is precisely the "registry does not
+# describe a usable model" case REGISTRY_REMEDY exists for, and catching it at
+# the one lookup means the `--print-registry-model` step fails first, before
+# any cache key is built or any byte is fetched.
+REGISTRY_PENDING = "PENDING"
+
+
+def resolve_registry_model(role: str, registry_path: Path = REGISTRY_PATH) -> Dict[str, Any]:
+    """Return ``{id, url, sha256, size_gb}`` for the registry model with ``role``.
+
+    Single source of truth for the workflow: the cache key, the download and the
+    checksum verification all come from this one lookup, so they cannot drift
+    apart. ``yaml`` is imported lazily so the module imports without PyYAML.
+    """
+    import yaml  # lazy: keeps this module importable for unit tests without PyYAML
+
+    registry = yaml.safe_load(registry_path.read_text(encoding="utf-8"))
+    matches = [m for m in registry.get("models", []) if m.get("role") == role]
+    if not matches:
+        raise SmokeFailure(
+            FailureClass.PIPELINE,
+            f"No model with role {role!r} in {registry_path}",
+            remedy=REGISTRY_REMEDY,
+        )
+    if len(matches) > 1:
+        raise SmokeFailure(
+            FailureClass.PIPELINE,
+            f"Registry has {len(matches)} models with role {role!r}; expected exactly one",
+            remedy=REGISTRY_REMEDY,
+        )
+    model = matches[0]
+    download = model.get("download", {})
+    sha256 = download.get("sha256")
+    url = download.get("url")
+    if not sha256 or not url:
+        raise SmokeFailure(
+            FailureClass.PIPELINE,
+            f"Registry model {model.get('id')!r} is missing download.url or download.sha256",
+            remedy=REGISTRY_REMEDY,
+        )
+    unpinned = [
+        field
+        for field, value in (("url", url), ("sha256", sha256))
+        if value == REGISTRY_PENDING
+    ]
+    if unpinned:
+        raise SmokeFailure(
+            FailureClass.PIPELINE,
+            f"Registry model {model.get('id')!r} is not pinned yet: "
+            f"{', '.join('download.' + f for f in unpinned)} "
+            f"still reads {REGISTRY_PENDING!r}. There is nothing to download or "
+            "verify against, so the smoke cannot run on this model.",
+            remedy=REGISTRY_REMEDY,
+        )
+    return {
+        "id": model["id"],
+        "url": url,
+        "sha256": sha256,
+        "size_gb": model.get("size_gb"),
+    }
+
+
+def _emit_github_output(values: Dict[str, Any]) -> None:
+    """Write ``key=value`` lines to $GITHUB_OUTPUT (and always to stdout)."""
+    lines = [f"{k}={v}" for k, v in values.items()]
+    for line in lines:
+        print(line)
+    out = os.environ.get("GITHUB_OUTPUT")
+    if out:
+        with open(out, "a", encoding="utf-8") as f:
+            f.write("\n".join(lines) + "\n")
+
+
+# ── Download and checksum ─────────────────────────────────────────────────────
+
+
+def _download_with_progress(
+    url: str, dest: Path, *, budget_s: float = DOWNLOAD_BUDGET_S
+) -> None:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    print(f"  Downloading {url[:80]}…")
+    try:
+        req = urllib.request.Request(url, headers={"User-Agent": "convsim-smoke/1.0"})
+    except ValueError as exc:
+        # urllib raises out of the *constructor* for a string it cannot read as
+        # a URL ("unknown url type"), i.e. before the handler below exists. That
+        # escaped to main's catch-all and was reported as "most likely a bug in
+        # the smoke harness", with a remedy telling the reader to re-run with
+        # the same arguments — when the argument is the whole problem.
+        #
+        # Not `download` either: nothing was fetched and no re-run or registry
+        # URL check changes the outcome. This is the harness being handed an
+        # input it cannot act on, which is what `pipeline` covers for everything
+        # else of that kind, with a remedy naming both places the string can
+        # come from. The realistic source is the local repro in
+        # docs/real-model-smoke.md, whose --model-url is a placeholder the
+        # reader is meant to substitute.
+        raise SmokeFailure(
+            FailureClass.PIPELINE,
+            f"Not a usable download URL: {url[:120]!r} ({exc})",
+            remedy=(
+                "Nothing was fetched — the string is not a URL urllib can "
+                "request. If it came from --model-url on the command line, "
+                "substitute the real URL printed by "
+                "`--print-registry-model starter` (the repro block in "
+                "docs/real-model-smoke.md shows a placeholder, not a value). "
+                "If it came from the registry, repair download.url in "
+                "model-registry/registry.yaml and validate with "
+                "scripts/validate-registry.py --url-check."
+            ),
+        ) from exc
+    started = time.monotonic()
+    try:
+        with urllib.request.urlopen(
+            req, timeout=DOWNLOAD_READ_TIMEOUT_S
+        ) as resp, open(dest, "wb") as f:
+            total = int(resp.headers.get("Content-Length", 0)) or None
+            downloaded = 0
+            chunk = 1 << 20  # 1 MB
+            last_logged_pct = -PROGRESS_LOG_STEP_PCT
+            while True:
+                buf = resp.read(chunk)
+                if not buf:
+                    break
+                f.write(buf)
+                downloaded += len(buf)
+                elapsed = time.monotonic() - started
+                if elapsed > budget_s:
+                    # Checked per chunk rather than left to urllib's per-read
+                    # timeout, which a transfer that merely crawls never trips.
+                    # See DOWNLOAD_BUDGET_S: past this the job clock, not the
+                    # transfer, is what runs out next.
+                    rate = downloaded / elapsed / 1_048_576 if elapsed else 0.0
+                    raise SmokeFailure(
+                        FailureClass.DOWNLOAD,
+                        f"Download exceeded its {budget_s:.0f} s budget after "
+                        f"{downloaded // 1_048_576} MB"
+                        + (f" of {total // 1_048_576} MB" if total else "")
+                        + f" ({rate:.1f} MB/s)",
+                        remedy=(
+                            "The transfer stalled or crawled rather than failing. "
+                            "It is bounded because it runs before the smoke and on "
+                            "the same job clock, which the harness's own deadline "
+                            "cannot see: letting it run on would get the job "
+                            "cancelled with no attributed verdict at all. Nothing "
+                            "about the app changed — check that Hugging Face is "
+                            "reachable (scripts/validate-registry.py --url-check) "
+                            "and re-run the job."
+                        ),
+                    )
+                if total:
+                    pct = int(downloaded / total * 100)
+                    if pct - last_logged_pct >= PROGRESS_LOG_STEP_PCT:
+                        last_logged_pct = pct
+                        print(
+                            f"\r  {pct}% ({downloaded // 1_048_576} / {total // 1_048_576} MB)",
+                            end="",
+                            flush=True,
+                        )
+        print()
+    except SmokeFailure:
+        # The exhausted-budget failure above, which already carries its own
+        # message and remedy.  The partial file still has to go, but re-wrapping
+        # it in the generic verdict below would replace both with "Download
+        # failed for <url>: SmokeFailure(...)".
+        dest.unlink(missing_ok=True)
+        raise
+    except Exception as exc:  # noqa: BLE001 — see below
+        # Network failure, HTTP error, or a full disk — all "could not fetch the
+        # model", none of which say anything about the app under test.  Remove the
+        # partial file so a re-run does not checksum-fail on a truncated download.
+        #
+        # Deliberately broader than OSError: a cut-off 2.5 GB response raises
+        # http.client.IncompleteRead, which is NOT an OSError, so an OSError-only
+        # handler let a truncated download escape as a bare traceback — exit 1,
+        # which EXIT_CODES reads as `budget` — and left the partial file on disk
+        # for the next run to checksum-fail on.
+        dest.unlink(missing_ok=True)
+        raise SmokeFailure(
+            FailureClass.DOWNLOAD,
+            f"Download failed for {url[:120]}: {exc!r}",
+        ) from exc
+
+
+def sha256_file(path: Path) -> str:
     h = hashlib.sha256()
     with open(path, "rb") as f:
         for chunk in iter(lambda: f.read(1 << 20), b""):
             h.update(chunk)
-    actual = h.hexdigest()
-    if actual != expected:
-        raise ValueError(f"SHA-256 mismatch: expected {expected}, got {actual}")
-    print(f"  SHA-256 verified: {actual[:16]}…")
+    return h.hexdigest()
+
 
+def _require_sha256_digest(expected: str) -> str:
+    """Normalise ``expected`` and reject anything that is not a SHA-256 digest.
 
-def download_model(url: str, sha256: str, model_id: str) -> Path:
-    dest = MODELS_DIR / f"{model_id}.gguf"
+    Separate from ``verify_model_checksum`` so it can also run *before* a
+    transfer: the digest is an argument, and nothing about the 2.5 GB that
+    would be fetched can make a malformed one match.  See ``download_model``.
+    """
+    expected = expected.strip().lower()
+    if len(expected) != SHA256_HEX_CHARS or not set(expected) <= _HEX_DIGITS:
+        raise SmokeFailure(
+            FailureClass.PIPELINE,
+            f"Not a SHA-256 digest: {expected!r} ({len(expected)} characters, "
+            f"expected {SHA256_HEX_CHARS} hex). Nothing was fetched, nothing was "
+            "verified and nothing was deleted — no file's hash could match this.",
+            remedy=(
+                "The expected digest itself is malformed, so this says nothing "
+                "about the bytes on disk and the model file has been left in "
+                "place. If it came from --model-sha256 on the command line, "
+                "re-copy all 64 characters printed by "
+                "`--print-registry-model starter` (the repro block in "
+                "docs/real-model-smoke.md shows a placeholder, not a value). If "
+                "it came from the registry, repair download.sha256 in "
+                "model-registry/registry.yaml (re-pin with scripts/pin-model.py) "
+                "and validate with scripts/validate-registry.py."
+            ),
+        )
+    return expected
+
+
+def verify_model_checksum(path: Path, expected: str, *, delete_on_mismatch: bool = True) -> str:
+    """Hash ``path`` and compare to ``expected``; raise on drift.
+
+    A mismatch deletes the file by default: leaving a known-bad GGUF in the CI
+    cache would make every subsequent run fail on the same corrupt bytes, since
+    the cache key is the expected hash and a cache hit skips the download.
+
+    ``expected`` is compared as a hex *value*, not as a string: a digest that
+    differs only in case or surrounding whitespace is the same digest, and
+    string equality would call it drift.  That matters because of what a
+    mismatch does — it deletes a 2.5 GB file and reports "the pinned upstream
+    file was replaced … do NOT relax this check", which is an expensive and
+    thoroughly misleading verdict to hand someone who pasted an upper-case
+    hash into the local repro in docs/real-model-smoke.md.
+
+    For the same reason, ``expected`` has to *be* a SHA-256 digest before it is
+    compared to one.  A digest short by a character — the ordinary outcome of
+    copying 64 hex characters out of a terminal by hand — can never match any
+    file's hash, so the comparison is not a check at all; but it fails exactly
+    like drift does, which is to say it deletes the 2.5 GB file and reports
+    that the pinned upstream file was replaced and must not be re-pinned
+    lightly.  A malformed digest is the harness being handed an input it cannot
+    act on, which is ``pipeline``, and the file is left alone.  ``download_model``
+    applies the same guard before it fetches anything, so the verdict does not
+    cost a transfer either.
+    """
+    expected = _require_sha256_digest(expected)
+    if not path.exists():
+        raise SmokeFailure(
+            FailureClass.DOWNLOAD,
+            f"Model file not found: {path}. The cache restore produced no model and no "
+            "download ran — check the cache key and the download step.",
+        )
+    actual = sha256_file(path)
+    if actual != expected:
+        size_mb = path.stat().st_size // 1_048_576
+        if delete_on_mismatch:
+            path.unlink(missing_ok=True)
+        raise SmokeFailure(
+            FailureClass.CHECKSUM,
+            f"SHA-256 drift on {path.name} ({size_mb} MB): registry expects "
+            f"{expected}, on-disk bytes hash to {actual}"
+            + (". The bad file was deleted." if delete_on_mismatch else ""),
+        )
+    print(f"  SHA-256 verified: {actual[:16]}… ({path.stat().st_size // 1_048_576} MB)")
+    return actual
+
+
+def download_model(url: str, sha256: str, model_id: str, models_dir: Path = MODELS_DIR) -> Path:
+    # The digest is checked for *shape* before the transfer, not after it.
+    # verify_model_checksum below rejects a malformed digest without hashing
+    # anything, but reaching it means 2.5 GB has already been fetched to be
+    # compared against a value no file could ever match — and on the realistic
+    # route here, the local repro's hand-pasted --model-sha256, the reader waits
+    # out the whole download to be told the argument was wrong. There is no
+    # verdict the bytes could have changed, so spend nothing to reach it.
+    sha256 = _require_sha256_digest(sha256)
+    dest = models_dir / f"{model_id}.gguf"
     if dest.exists():
         print(f"  Model already on disk: {dest}")
-        return dest
-    _download_with_progress(url, dest)
-    _verify_sha256(dest, sha256)
+    else:
+        _download_with_progress(url, dest)
+    verify_model_checksum(dest, sha256)
     return dest
 
 
 # ── Server helpers ────────────────────────────────────────────────────────────
 
 
-def _find_model_path(model_id: str) -> Path:
-    path = MODELS_DIR / f"{model_id}.gguf"
-    if not path.exists():
-        raise FileNotFoundError(
-            f"Model file not found: {path}\n"
-            "Run with --download-only first, or check that the model cache is populated."
-        )
-    return path
+def _assert_port_free(port: int, label: str) -> None:
+    """Refuse to run when something already listens on ``port``.
+
+    Readiness is a URL poll, and a squatter answers it.  The child we spawned
+    loses the bind and exits, but it is still starting up when the first poll
+    runs — ``llama_cpp.server`` loads the whole GGUF before it binds — so the
+    poll succeeds against the *foreign* server and ``_wait_for_http`` returns
+    happy.  Nothing downstream catches it either: ``_crashed_child`` is only
+    consulted once a ``SmokeFailure`` has been raised, so a run that otherwise
+    goes green never notices that the process it started is dead.
+
+    The result is the one verdict this job must never return: a pass earned by
+    a model whose bytes it never checksum-verified, on a runtime it did not
+    choose.  CI runners are clean, so this is about the local repro in
+    docs/real-model-smoke.md — which is also how the real-model path gets
+    verified by hand, since no PR job can exercise it.
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.settimeout(1.0)
+        if sock.connect_ex(("127.0.0.1", port)) != 0:
+            return
+    raise SmokeFailure(
+        FailureClass.RUNTIME,
+        f"Port {port} is already in use, so {label} cannot be started on it",
+        # Not the class remedy: it says to read the child's stderr tail, and no
+        # child has been started yet.
+        remedy=(
+            f"Stop whatever is listening on 127.0.0.1:{port}, then re-run. The smoke "
+            f"has to own {label}: readiness is a URL poll, so a server it did not "
+            "start answers it and the run would pass on a model these bytes were "
+            "never verified against."
+        ),
+    )
 
 
 def _start_llama_server(model_path: Path, port: int) -> subprocess.Popen:
@@ -143,7 +862,7 @@ def _start_llama_server(model_path: Path, port: int) -> subprocess.Popen:
     convsim-core's ``llama_cpp`` runtime adapter talks to this server over
     ``/v1/chat/completions`` (see services/convsim-core/.../runtime/llama_cpp.py).
     """
-    proc = subprocess.Popen(
+    return subprocess.Popen(
         [sys.executable, "-m", "llama_cpp.server",
          "--model", str(model_path),
          "--host", "127.0.0.1", "--port", str(port),
@@ -151,7 +870,6 @@ def _start_llama_server(model_path: Path, port: int) -> subprocess.Popen:
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
-    return proc
 
 
 def _start_core(data_dir: Path, port: int, llama_port: int, llama_timeout_s: float) -> subprocess.Popen:
@@ -163,11 +881,13 @@ def _start_core(data_dir: Path, port: int, llama_port: int, llama_timeout_s: flo
         # Point the adapter at the llama-server started above.
         "CONVSIM_LLAMA_CPP_BASE_URL": f"http://127.0.0.1:{llama_port}",
         "CONVSIM_LLAMA_CPP_CONTEXT_LENGTH": "8192",
-        # The adapter's generation timeout must outlast the CI ceiling we
-        # measure against, or a slow-but-passing turn on the 2-vCPU CPU-only
-        # runner is killed before we can time it, failing the nightly spuriously
-        # instead of reporting the real latency. CHAT_TIMEOUT is the one that
-        # bounds /v1/chat/completions; TIMEOUT only covers model listing, and is
+        # The adapter's generation timeout must outlast the slowest call we
+        # intend to measure (the debrief), or a slow-but-reportable generation
+        # on the 2-vCPU CPU-only runner is killed before we can time it and
+        # convsim-core answers 500 — a latency problem misfiled as a RUNTIME
+        # crash, failing the nightly spuriously instead of reporting the real
+        # latency.  See run_smoke.  CHAT_TIMEOUT is the one that bounds
+        # /v1/chat/completions; TIMEOUT only covers model listing, and is
         # raised alongside it so a cold first listing on a loaded runner is not
         # cut short either.
         "CONVSIM_LLAMA_CPP_CHAT_TIMEOUT": str(int(llama_timeout_s)),
@@ -179,23 +899,32 @@ def _start_core(data_dir: Path, port: int, llama_port: int, llama_timeout_s: flo
         "CONVSIM_PORT": str(port),
         "CONVSIM_HOST": "127.0.0.1",
     })
-    proc = subprocess.Popen(
+    return subprocess.Popen(
         [sys.executable, "-m", "uvicorn", "convsim_core.main:app",
          "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"],
         env=env,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
-    return proc
 
 
-def _wait_for_http(url: str, timeout_s: float, label: str) -> None:
-    """Poll ``url`` until it returns any HTTP response, or raise on timeout."""
+def _wait_for_http(url: str, timeout_s: float, label: str, proc: Optional[subprocess.Popen] = None) -> None:
+    """Poll ``url`` until it answers, or raise a RUNTIME failure.
+
+    When ``proc`` is given, an exited child short-circuits the wait: a crashed
+    server is a runtime failure we can name immediately rather than something to
+    keep polling for until the timeout.
+    """
     deadline = time.monotonic() + timeout_s
     last_err: Optional[Exception] = None
     while time.monotonic() < deadline:
+        if proc is not None and proc.poll() is not None:
+            raise SmokeFailure(
+                FailureClass.RUNTIME,
+                f"{label} exited with code {proc.returncode} before becoming ready",
+            )
         try:
-            with urllib.request.urlopen(url, timeout=2):
+            with urllib.request.urlopen(url, timeout=5):
                 return
         except urllib.error.HTTPError:
             # A response (even 4xx) means the server is up and routing.
@@ -203,47 +932,715 @@ def _wait_for_http(url: str, timeout_s: float, label: str) -> None:
         except Exception as exc:  # noqa: BLE001 — connection refused while booting
             last_err = exc
             time.sleep(1)
-    raise TimeoutError(f"{label} did not become ready within {timeout_s:.0f} s ({last_err})")
+    raise SmokeFailure(
+        FailureClass.RUNTIME,
+        # !r, like every other error this file reports: str() on an exception
+        # raised with no arguments is the empty string, so a readiness failure
+        # whose last error was e.g. a bare ConnectionRefusedError rendered as
+        # "did not become ready within 300 s ()" — no diagnostic at all in the
+        # one message a `runtime` verdict leaves behind.
+        f"{label} did not become ready within {timeout_s:.0f} s ({last_err!r})",
+    )
 
 
-# ── Session helpers ───────────────────────────────────────────────────────────
+# ── HTTP helpers (every transport-level failure is a RUNTIME failure) ──────────
 
 
-def _post_json(url: str, payload: dict, timeout: float = 30.0) -> dict:
-    data = json.dumps(payload).encode()
+def _request_json(
+    url: str,
+    *,
+    payload: Optional[dict] = None,
+    timeout: float,
+    expect: Sequence[int] = (200, 201),
+) -> dict:
+    data = json.dumps(payload).encode() if payload is not None else None
     req = urllib.request.Request(
         url,
         data=data,
-        headers={"Content-Type": "application/json"},
-        method="POST",
+        headers={"Content-Type": "application/json"} if data is not None else {},
+        method="POST" if data is not None else "GET",
     )
-    with urllib.request.urlopen(req, timeout=timeout) as resp:
-        return json.loads(resp.read())
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            body = json.loads(resp.read())
+            if resp.status not in expect:
+                raise SmokeFailure(
+                    FailureClass.PIPELINE,
+                    f"{url} returned HTTP {resp.status}, expected one of {list(expect)}",
+                )
+            return body
+    except urllib.error.HTTPError as exc:
+        detail = exc.read(2048).decode(errors="replace")
+        # 5xx is the server breaking; 4xx is the server rejecting what we asked
+        # for, i.e. an end-to-end assertion about the API contract.
+        cls = FailureClass.RUNTIME if exc.code >= 500 else FailureClass.PIPELINE
+        raise SmokeFailure(cls, f"{url} returned HTTP {exc.code}: {detail[:500]}") from exc
+    except SmokeFailure:
+        raise
+    except Exception as exc:  # noqa: BLE001 — connection reset, timeout, bad JSON
+        raise SmokeFailure(
+            FailureClass.RUNTIME,
+            f"{url} did not answer with usable JSON after {timeout:.0f} s: {exc!r}",
+        ) from exc
+
+
+def _npc_turn_payload(events: Sequence[dict]) -> Dict[str, Any]:
+    """Return the NPC event's payload from a turn/start response, or ``{}``."""
+    for ev in events:
+        if ev.get("event_type") in ("npc_turn", "npc_opening"):
+            payload = ev.get("payload")
+            return payload if isinstance(payload, dict) else {}
+    return {}
 
 
-def _get_json(url: str, timeout: float = 30.0) -> dict:
-    with urllib.request.urlopen(url, timeout=timeout) as resp:
-        return json.loads(resp.read())
+def _npc_turn_content(events: Sequence[dict]) -> str:
+    """Return the NPC utterance from a turn/start response's events, or ''."""
+    content = _npc_turn_payload(events).get("content", "")
+    return content if isinstance(content, str) else ""
+
+
+def _rubric_observation_count(events: Sequence[dict]) -> Optional[int]:
+    """Count the rubric observations the model volunteered on this NPC turn.
+
+    The turn response carries the *validated* observations, which is the only
+    way to tell the two causes of an unscored debrief apart: the debrief engine
+    re-derives its dimension scores from each NPC turn's stored raw model output
+    (``_parse_rubric_observations`` in debrief_engine.py), so observations that
+    were accepted on the turn but never scored mean the debrief engine lost
+    them, while no observations anywhere means the model simply never
+    volunteered any.  See ``evaluate_debrief``.
+
+    Returns ``None`` — not 0 — when the payload carries no readable
+    ``rubric_observations`` list, the same way the per-turn parse flags warn
+    rather than pass when the debug endpoint gives us nothing: "the model
+    volunteered none" and "we could not read what it volunteered" lead to
+    opposite triage, and the whole point of recording this is that the
+    unscored-debrief verdict names a cause it has actually established.  An
+    empty list is a real observation of zero and still counts as 0.
+    """
+    observations = _npc_turn_payload(events).get("rubric_observations")
+    return len(observations) if isinstance(observations, list) else None
 
 
-def _npc_turn_content(events: list) -> str:
-    """Return the NPC utterance from a turn/start response's events, or ''."""
-    for ev in events:
-        if ev.get("event_type") in ("npc_turn", "npc_opening"):
-            return ev.get("payload", {}).get("content", "")
-    return ""
+def _total_rubric_observations(turns: Sequence[Dict[str, Any]]) -> Optional[int]:
+    """Total the per-turn rubric observation counts, or ``None`` if unknowable.
+
+    ``None`` means *the sum cannot be trusted as a total*, which is not the same
+    as "nothing was readable":
+
+    * a total above zero is conclusive even with an unreadable turn among the
+      readable ones — observations *were* seen, so the debrief losing them is
+      the cause whatever the missing turn carried;
+    * a sum of zero is only conclusive when every generated turn was readable.
+      With counts of ``[0, None, 0]`` two turns really did carry nothing, but
+      reporting 0 would have the verdict assert the model volunteered nothing
+      across the conversation when the turn it could not read may have carried
+      plenty — so that case is ``None`` too, and the report's per-turn
+      ``rubric_observation_count`` shows which turn was the unreadable one;
+    * no generated turn at all is ``None``: there was nothing to read.
+
+    Only the authored opening is skipped: it is not a generation and so has no
+    count to miss.
+    """
+    counts = [
+        t.get("rubric_observation_count") for t in turns if t.get("model_generated")
+    ]
+    total = sum(c for c in counts if isinstance(c, int))
+    if total > 0:
+        return total
+    zero_is_unreliable = not counts or any(not isinstance(c, int) for c in counts)
+    return None if zero_is_unreliable else 0
+
+
+def _collapse(text: str) -> str:
+    """Normalise whitespace so two utterances can be compared as written."""
+    return " ".join(text.split())
+
+
+def _excerpt(text: str) -> str:
+    collapsed = _collapse(text)
+    return collapsed[:EXCERPT_CHARS] + ("…" if len(collapsed) > EXCERPT_CHARS else "")
+
+
+def _excerpt_any(value: Any) -> str:
+    """Excerpt a server-supplied value that *should* be a string, whatever it is.
+
+    ``_excerpt`` calls ``str.split``, so handing it a non-string raises
+    ``AttributeError``.  Where that value came out of a convsim-core response —
+    the debrief summary is the one such field the harness both excerpts into the
+    report and asserts on — the raise escapes into ``run_smoke``'s catch-all and
+    reports a changed product response contract as "most likely a bug in the
+    smoke harness", which is the one thing it is certainly not.
+
+    ``repr`` rather than ``str``, so the artifact shows the shape that arrived
+    instead of a coerced empty string.  Telling the run it is *wrong* stays
+    ``evaluate_debrief``'s job: a dict's repr is easily long enough to satisfy
+    the summary length check, so nothing here may be mistaken for acceptance.
+    """
+    if value is None:
+        return ""
+    return _excerpt(value if isinstance(value, str) else repr(value))
+
+
+def _replays_opening(npc_text: str, opening_text: str) -> bool:
+    """True when an NPC reply opens by reciting the authored opening verbatim.
+
+    The scenario's opening line sits in the transcript the turn prompt renders,
+    and the starter model can answer a player turn by copying it back: all
+    twelve nightly runs of the previous single-turn harness whose logs GitHub
+    still held on 2026-10-03 logged an NPC reply whose leading characters
+    matched ``opening_npc_says`` exactly.  That is not a reply, but nothing
+    downstream notices — the utterance is non-empty and it is not
+    ``SAFE_FALLBACK_UTTERANCE``, so ``used_fallback`` stays false and the
+    per-turn parse flags report a healthy turn.  Proving the turns are real
+    therefore needs this check as well as those flags.
+
+    Those twelve runs are evidence that the failure mode exists, not that it is
+    the norm: the player turn they submitted was "Reply with exactly one
+    sentence: Hello, I am ready." — an instruction to the model, not something an
+    interview candidate says — and this harness replaces it with the
+    in-character ``SCRIPTED_PLAYER_TURNS``, which the real starter model
+    answered properly in local verification.  So the expected rate here is zero
+    and the warning is worth reading when it fires, rather than a known
+    condition to scroll past.
+
+    A *leading prefix* match, not equality and not the whole opening: a recital
+    that then carries on into fresh prose is the same regurgitation, and a
+    recital that veers off partway through the opening is too — see
+    ``REPLAY_PREFIX_CHARS`` for why the latter matters and where the length
+    comes from.  No legitimate reply begins by reading the opening question
+    back out.
+    """
+    opening = _collapse(opening_text)
+    if not opening:
+        return False
+    threshold = min(len(opening), REPLAY_PREFIX_CHARS)
+    return _collapse(npc_text)[:threshold] == opening[:threshold]
+
+
+# ── Assertions (pure, unit-tested) ────────────────────────────────────────────
+
+
+def _all_generated_turns_fell_back(turns: Sequence[Dict[str, Any]]) -> bool:
+    """True when every model-generated turn returned the canned safe utterance.
+
+    Two verdicts turn on this and they must agree: ``evaluate_turns`` fails the
+    run for it, and ``evaluate_debrief`` uses it to say that an unscored debrief
+    is downstream of those fallbacks rather than evidence about rubric
+    prompting.  One of them deciding "all of them" while the other decides "not
+    quite all" would put both notes in the same verdict, each telling the reader
+    the other is the wrong place to start.
+
+    False when there are no generated turns (``evaluate_turns`` reports that
+    separately) and when the parse flags were unavailable, since an absent
+    ``used_fallback`` is not a fallback.
+    """
+    generated = [t for t in turns if t.get("model_generated")]
+    return bool(generated) and all(t.get("used_fallback") for t in generated)
+
+
+def evaluate_turns(turns: List[Dict[str, Any]]) -> tuple[List[str], List[str]]:
+    """Check the conversation really came from the model.  Returns (failures, warnings).
+
+    The fallback policy is applied only to turns marked ``model_generated``: the
+    NPC opening is authored scenario text, so counting it would make the
+    "every turn fell back" check unreachable.
+
+    An empty NPC utterance is always a failure.  A single fallback turn is a
+    warning — a 4 B model does occasionally emit output the validator rejects,
+    and the product's job is to recover from that — but if *every* generated turn
+    fell back to the canned safe utterance then no real NPC turns were produced
+    and the smoke proved nothing, so that is a failure.
+
+    Turns carrying no ``used_fallback`` flag — the debug endpoint is
+    best-effort — warn that the check did not run, so an absent proof is not
+    mistaken for a passed one.  Counted per turn rather than all-or-nothing: a
+    payload that keys only some of the turns leaves the same hole on the ones it
+    missed, and an absent flag is falsy, so those turns would otherwise be
+    credited as real replies by the checks below without any evidence.
+
+    ``replayed_opening`` gets the identical policy for the other way a turn can
+    pass every flag without being a reply: reciting the authored opening back at
+    the player (see ``_replays_opening``).  Unlike the parse flags this is
+    computed from the response the harness already holds, so it is never
+    missing.
+
+    The two policies also have to compose: a conversation split between them —
+    one turn recites, the rest fall back — trips neither "all of them" check
+    while still containing no reply, so a run with no turn that is both real
+    output *and* an answer fails as well.
+    """
+    failures: List[str] = []
+    warnings: List[str] = []
+    if not turns:
+        return (["No turns were recorded; the conversation never ran"], warnings)
+
+    for turn in turns:
+        label = turn.get("label", f"turn {turn.get('turn_number')}")
+        if not turn.get("npc_excerpt"):
+            failures.append(f"{label}: completed but returned no NPC utterance")
+
+    generated = [t for t in turns if t.get("model_generated")]
+    if not generated:
+        failures.append("No model-generated NPC turns were recorded")
+        return (failures, warnings)
+
+    # The flags come from the debug endpoint, which is best-effort. Without
+    # them a canned fallback is indistinguishable from real model output — it is
+    # a non-empty NPC utterance too — so the checks below silently pass. Say so,
+    # rather than let the report imply a proof we do not have.
+    #
+    # Counted per turn, not all-or-nothing: _debug_flags_by_turn drops any entry
+    # it cannot key by turn number, so a changed /debug contract can cover some
+    # turns and not others. On the turns it missed the hole is exactly the same,
+    # and worse than harmless — an absent used_fallback is falsy, so those turns
+    # count towards "at least one generated turn was a real reply" in the checks
+    # below and can keep a conversation of fallbacks from failing at all. The
+    # rubric path already refuses to round an unreadable turn down to zero (see
+    # _total_rubric_observations); this is the same refusal.
+    unflagged = [t for t in generated if "used_fallback" not in t]
+    if len(unflagged) == len(generated):
+        warnings.append(
+            "Per-turn parse flags were unavailable, so the fallback check did not "
+            "run: this run cannot prove the NPC turns were model output rather "
+            "than the canned safe utterance"
+        )
+    elif unflagged:
+        missing = ", ".join(
+            str(t.get("label") or f"turn {t.get('turn_number')}") for t in unflagged
+        )
+        warnings.append(
+            f"Per-turn parse flags were missing for {len(unflagged)} of "
+            f"{len(generated)} model-generated NPC turns ({missing}), so the "
+            "fallback check did not run for those: they count as real replies "
+            "below on no evidence at all. The /debug payload keyed only some of "
+            "the turns, which is itself worth a look at the turn response contract"
+        )
+
+    fallbacks = [t for t in generated if t.get("used_fallback")]
+    all_fell_back = _all_generated_turns_fell_back(turns)
+    if all_fell_back:
+        failures.append(
+            f"All {len(generated)} model-generated NPC turns fell back to the canned "
+            "safe utterance; the real model produced no usable turn output"
+        )
+    elif fallbacks:
+        warnings.append(
+            f"{len(fallbacks)} of {len(generated)} model-generated NPC turns fell back "
+            "to the safe utterance"
+        )
+
+    # Same policy as the fallback check above, for the other way a turn can look
+    # real without being one: reciting the authored opening back instead of
+    # answering.  See _replays_opening — the parse flags cannot see this, so
+    # without it "the model produced real NPC turns" rests on the utterance
+    # merely being non-empty.
+    echoes = [t for t in generated if t.get("replayed_opening")]
+    if echoes and len(echoes) == len(generated):
+        failures.append(
+            f"All {len(generated)} model-generated NPC turns began by reciting the "
+            "authored opening line back at the player; the model answered no player "
+            "turn, so this run proves nothing about real NPC replies"
+        )
+    elif echoes:
+        warnings.append(
+            f"{len(echoes)} of {len(generated)} model-generated NPC turns began by "
+            "reciting the authored opening line back at the player instead of "
+            "answering them: real output, but not a reply. Recurring in the same turn "
+            "position every night is a turn-prompt problem, not an unlucky sample"
+        )
+
+    # Neither check above fires when the generated turns are *split* between the
+    # two anomalies — one recites the authored opening, the others fall back —
+    # and that conversation contains no NPC reply at all.  Each check warns
+    # rather than fails on its own because a run that recovers from one bad turn
+    # is the product working; the union of them leaves nothing that recovered,
+    # which is the failure both checks exist to catch, so it has to fail too.
+    # Reachable in practice: the previous harness logged a recited opening on
+    # the first player turn every night, and a 4 B model does occasionally emit
+    # output the validator rejects on the rest.
+    mixed = not all_fell_back and len(echoes) != len(generated)
+    if mixed and not [
+        t
+        for t in generated
+        if not t.get("used_fallback") and not t.get("replayed_opening")
+    ]:
+        failures.append(
+            f"Not one of the {len(generated)} model-generated NPC turns was a reply: "
+            f"{len(fallbacks)} fell back to the canned safe utterance and "
+            f"{len(echoes)} began by reciting the authored opening line. Neither "
+            "count alone would fail the run, but between them they account for "
+            "every generated turn, so no player turn was answered and this run "
+            "proves nothing about real NPC replies"
+        )
+    return (failures, warnings)
+
+
+def evaluate_debrief(
+    debrief: Optional[Dict[str, Any]],
+    *,
+    rubric_observations_seen: Optional[int] = None,
+    turns_all_fell_back: bool = False,
+) -> tuple[List[str], List[str]]:
+    """Check that a *scored* debrief was produced.  Returns (failures, warnings).
+
+    "Scored" is the #457 acceptance criterion, so an unscored debrief is a
+    failure — but on its own that failure does not say which of several very
+    different things happened, and the remedy differs completely.
+    ``rubric_observations_seen`` (how many rubric observations the NPC turns
+    actually returned, ``None`` when the run could not read any — see
+    ``_total_rubric_observations``) and ``turns_all_fell_back`` resolve it:
+
+    * **0, every generated turn fell back** — the turns carried no observations
+      because they carried no model output at all. Not evidence about rubric
+      prompting; a consequence of the fallback failure ``evaluate_turns``
+      already reported: ``UNSCORED_AFTER_FALLBACK_NOTE``.
+    * **0, turns that did not fall back** — the model volunteered nothing to
+      score, which nothing asked it to. Reachable because of the thin prompt
+      coverage, but not the normal outcome (the real starter model volunteers
+      observations on every turn), so still worth chasing:
+      ``UNSCORED_DEBRIEF_NOTE``.
+    * **> 0** — the turns carried observations and the debrief lost them. A real
+      regression with a named starting point: ``UNSCORED_WITH_OBSERVATIONS_NOTE``.
+
+    Without that count the failure can only quote both possibilities and send
+    the reader to the previous nightly's artifact to tell them apart, which is
+    the one piece of triage this harness is meant to have already done.
+    """
+    failures: List[str] = []
+    warnings: List[str] = []
+    if not debrief:
+        return (["No debrief was produced"], warnings)
+
+    scores = debrief.get("scores")
+    if not isinstance(scores, dict) or not scores:
+        if rubric_observations_seen:
+            cause = (
+                f"the NPC turns returned {rubric_observations_seen} rubric "
+                "observation(s), so the debrief engine had something to accumulate "
+                "and did not. "
+            ) + UNSCORED_WITH_OBSERVATIONS_NOTE
+        elif rubric_observations_seen == 0 and turns_all_fell_back:
+            cause = (
+                "no NPC turn carried a rubric_observation, so the debrief engine "
+                "had nothing to accumulate — but none of them carried any model "
+                "output either. "
+            ) + UNSCORED_AFTER_FALLBACK_NOTE
+        elif rubric_observations_seen == 0:
+            cause = (
+                "no NPC turn carried a rubric_observation, so the debrief engine "
+                "had nothing to accumulate. "
+            ) + UNSCORED_DEBRIEF_NOTE
+        else:
+            # Neither cause can be ruled out, so quote both rather than pick
+            # one: asserting the model volunteered nothing, on the one branch
+            # that just said it could not tell, is the triage this harness is
+            # supposed to have done, done wrongly.
+            cause = (
+                "at least one NPC turn's rubric_observations list was "
+                "unreadable and the turns that were readable carried none, so "
+                "the run cannot claim the model volunteered nothing and the two "
+                "causes below cannot be told apart from this message — read "
+                "rubric_observation_count per turn in the report artifact (null "
+                "marks the unreadable turns, which is itself worth a look at the "
+                "turn response contract). " + UNSCORED_DEBRIEF_NOTE
+                + " If the turns did carry observations, it is the other cause "
+                "instead: " + UNSCORED_WITH_OBSERVATIONS_NOTE
+            )
+        failures.append(f"Debrief has no rubric dimension scores: {cause}")
+
+    overall = debrief.get("overall_score")
+    if not isinstance(overall, (int, float)) or isinstance(overall, bool):
+        failures.append(f"Debrief overall_score is not numeric (got {overall!r})")
+    elif not 0 <= overall <= 100:
+        failures.append(f"Debrief overall_score {overall} is outside [0, 100]")
+
+    summary = debrief.get("summary")
+    if summary is not None and not isinstance(summary, str):
+        # Checked as a type before it is measured as a length, for the same
+        # reason _rubric_observation_count returns None rather than 0 for an
+        # unreadable list: len(summary.strip()) raises AttributeError here, and
+        # an AttributeError out of the assertion layer lands in run_smoke's
+        # catch-all as "most likely a bug in the smoke harness" — the one
+        # verdict that is certainly wrong when it is the product's own response
+        # contract that changed.  A debrief whose summary is not text is a
+        # `pipeline` failure like any other output the product rejects.
+        failures.append(
+            f"Debrief summary is not a string (got {type(summary).__name__}): "
+            f"{_excerpt_any(summary)}"
+        )
+    elif len((summary or "").strip()) < MIN_SUMMARY_CHARS:
+        failures.append(
+            f"Debrief summary is shorter than {MIN_SUMMARY_CHARS} characters "
+            f"({len((summary or '').strip())} chars)"
+        )
+
+    if debrief.get("used_fallback"):
+        # Not a failure: the scores — the thing #457 asks this job to assert —
+        # come from the per-turn rubric observations, not from this generation,
+        # so a template narrative degrades the debrief rather than invalidating
+        # the run, and one unlucky generation must not red the nightly.
+        #
+        # But the debrief is generated exactly once, so a fallback means *no*
+        # model-written debrief prose was produced at all, and this is the only
+        # job that would ever notice.  Say what the otherwise-green verdict
+        # therefore does not prove, rather than just naming the flag.
+        warnings.append(
+            "Debrief narrative used the deterministic fallback, not model prose: "
+            "the model's debrief output failed validation and the repair pass, so "
+            "this run exercised the scoring path but proves nothing about "
+            "real-model debrief generation. Recurring across nightlies means a "
+            "regression in the debrief prompt or DEBRIEF_NARRATIVE_SCHEMA rather "
+            "than one unlucky generation"
+        )
+    if not debrief.get("turning_points"):
+        warnings.append("Debrief contains no turning points")
+    return (failures, warnings)
+
+
+def evaluate_budgets(
+    measured_ms: Dict[str, float],
+    ci_hardware_factor: float,
+    budgets_ms: Dict[str, int] = BUDGETS_MS,
+    tolerance: float = REGRESSION_TOLERANCE,
+) -> tuple[List[str], List[str]]:
+    """Compare measurements to scaled budgets.  Returns (failures, report_lines)."""
+    failures: List[str] = []
+    lines: List[str] = []
+    for key, budget_ms in budgets_ms.items():
+        measured = measured_ms.get(key)
+        if measured is None:
+            continue
+        ci_budget = budget_ms * ci_hardware_factor
+        ceiling = ci_budget * tolerance
+        status = "PASS" if measured <= ceiling else "FAIL"
+        lines.append(
+            f"{key}: {measured:.0f} ms "
+            f"(CI budget {ci_budget:.0f} ms × {tolerance} = {ceiling:.0f} ms) → {status}"
+        )
+        if status == "FAIL":
+            failures.append(
+                f"{key} = {measured:.0f} ms exceeds CI ceiling {ceiling:.0f} ms "
+                f"(documented budget {budget_ms} ms × {ci_hardware_factor} × {tolerance})"
+            )
+    return (failures, lines)
+
+
+def timeout_latency_evidence(
+    turns: Sequence[Dict[str, Any]], ceiling_ms: float
+) -> Optional[str]:
+    """Name the latency regression hiding inside a ``timeout`` verdict, if there is one.
+
+    The wall-clock budget and the latency ceiling sit close together by
+    construction: three scripted turns at the CI ceiling (240 s each) plus a
+    debrief allowed twice that very nearly fill the 20 min budget.  So a uniform
+    slowdown large enough to fail the budget check usually exhausts the clock
+    *before* the budget phase runs, and TIMEOUT — which outranks everything but
+    a crashed child — is what gets reported.  That class is honest, the clock
+    did run out, but its remedy ("inference got much slower or a phase hung")
+    leaves the reader to work out which.  When the turns that *did* complete are
+    already past the ceiling, we know which, so say so.
+
+    Returns ``None`` when the completed turns were inside the ceiling (a genuine
+    hang, or a phase other than the conversation ran long) or when no
+    model-generated turn finished at all.
+    """
+    latencies = [
+        float(t["latency_ms"])
+        for t in turns
+        if t.get("model_generated") and isinstance(t.get("latency_ms"), (int, float))
+    ]
+    if not latencies:
+        return None
+    median_ms = statistics.median(latencies)
+    if median_ms <= ceiling_ms:
+        return None
+    return (
+        f"The {len(latencies)} NPC turn(s) that did complete have a median latency of "
+        f"{median_ms:.0f} ms, already past the {ceiling_ms:.0f} ms CI ceiling: this is "
+        "most likely the latency regression `budget` exists to catch rather than a "
+        "hung phase. Compare measured_ms in the report artifact against recent "
+        "nightlies before raising --wall-clock-budget-s."
+    )
+
+
+def runtime_without_crash_remedy(llama_timeout_s: float) -> str:
+    """The ``runtime`` advice for a failure where neither child actually died.
+
+    ``runtime``'s class remedy opens with "The child stderr tail is printed above
+    the banner" and lists crash causes — runner OOM on model load, an
+    incompatible wheel, an unhandled exception.  All of that assumes a child
+    exited.  Often none has: every transport-level failure is a ``runtime``
+    failure, so a *live* convsim-core that answered 5xx, or a live llama-server
+    still loading when readiness ran out, lands in this class with both children
+    running and a stderr tail that shows a healthy server.
+
+    The 5xx is the one worth naming, because the cause behind it is usually not
+    a crash at all.  ``_start_core`` raises the adapter's generation budget
+    above the latency ceiling precisely so a slow turn is *measured* rather than
+    killed — but that moves the cliff, it does not remove it.  A turn slower
+    than that budget is still cut off adapter-side and convsim-core answers
+    504 ``TURN_TIMEOUT``, and since the budget is well inside the wall clock
+    (540 s against 1200 s at ``--ci-hardware-factor 20``) the run still has time
+    left, so neither ``timeout`` nor ``budget`` takes the class over.  The
+    verdict is then honest about the transport and wrong about the cause, with a
+    remedy sending triage after an OOM that did not happen — which is the same
+    misclassification ``_start_core``'s raised timeout exists to prevent, one
+    cliff further out.
+    """
+    return (
+        "Neither child process exited, so nothing crashed: the stderr tails above are "
+        "from servers that were still running when the run gave up. A live server "
+        "either never became ready or did not answer one request. If convsim-core "
+        "answered 504 TURN_TIMEOUT, that is the adapter's own generation budget "
+        f"({llama_timeout_s:.0f} s, CONVSIM_LLAMA_CPP_CHAT_TIMEOUT, which this harness "
+        "sets above the latency ceiling for exactly this reason) running out: "
+        "inference slower than that is a latency regression, not a crash, so compare "
+        "measured_ms and phase_durations_s in the report artifact against recent "
+        "nightlies instead of looking for an OOM. Otherwise read the tails for a "
+        "server that is alive but wedged — a model still loading, a request refused."
+    )
+
+
+# ── Reporting ─────────────────────────────────────────────────────────────────
+
+
+def render_step_summary(results: Dict[str, Any]) -> str:
+    """Render the GitHub step summary: verdict, class, timings, what to do next."""
+    verdict = results.get("verdict", "unknown")
+    icon = {"pass": "✅", "fail": "❌"}.get(verdict, "⚠️")
+    lines = [
+        f"## {icon} Real-model smoke: {verdict.upper()}",
+        "",
+    ]
+    # Same reasoning as the wall clock below: `--print-registry-model` runs
+    # before any --model-id exists, and it is the mode that fails when the
+    # registry names no usable `starter` entry or has not pinned one yet. Its
+    # summary led with "**Model:** `None`" — a field the run never resolved,
+    # presented as the model it ran on, directly above the verdict.
+    if results.get("model_id"):
+        lines.append(f"- **Model:** `{results['model_id']}`")
+    # Only the smoke run keeps a clock.  The pre-run modes (registry lookup,
+    # download, verify) write a summary too, and defaulting their missing
+    # timings to zero printed "Wall clock: 0 s (budget 0 s)" at the top of a
+    # download or checksum verdict — a measurement the run never took, in the
+    # first thing a triager reads.
+    if results.get("wall_clock_s") is not None:
+        lines.append(
+            f"- **Wall clock:** {results['wall_clock_s']:.0f} s "
+            f"(budget {results.get('wall_clock_budget_s', 0):.0f} s)"
+        )
+    if verdict == "fail":
+        cls = results.get("failure_class", "unknown")
+        lines += [
+            f"- **Failure class:** `{cls}` (exit {results.get('exit_code')})",
+            "",
+            "### Failures",
+            *[f"- {f}" for f in results.get("failures", [])],
+            "",
+            "### What to do",
+            results.get("remedy") or REMEDIES.get(cls, "No remedy recorded."),
+        ]
+    measured = results.get("measured_ms") or {}
+    if measured:
+        lines += ["", "### Measured latency", "", "| Metric | Measured | CI ceiling |", "|---|---|---|"]
+        factor = results.get("ci_hardware_factor", 1.0)
+        for key, value in measured.items():
+            budget = BUDGETS_MS.get(key)
+            ceiling = (
+                f"{budget * factor * REGRESSION_TOLERANCE:.0f} ms" if budget else "—"
+            )
+            lines.append(f"| `{key}` | {value:.0f} ms | {ceiling} |")
+    phases = results.get("phase_durations_s") or {}
+    if phases:
+        lines += ["", "### Phase durations", "", "| Phase | Seconds |", "|---|---|"]
+        lines += [f"| `{k}` | {v:.0f} |" for k, v in phases.items()]
+    if results.get("warnings"):
+        lines += ["", "### Warnings", *[f"- {w}" for w in results["warnings"]]]
+    return "\n".join(lines) + "\n"
+
+
+def _write_step_summary(results: Dict[str, Any]) -> None:
+    path = os.environ.get("GITHUB_STEP_SUMMARY")
+    if not path:
+        return
+    try:
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(render_step_summary(results))
+    except OSError as exc:  # never let reporting break the verdict
+        print(f"[smoke] Could not write step summary: {exc}", file=sys.stderr)
+
+
+def _print_banner(
+    failure_class: str, failures: Sequence[str], *, remedy: Optional[str] = None
+) -> None:
+    """Print the failure banner.  ``remedy`` overrides the class default.
+
+    Some failures borrow a class's exit code without matching its usual cause —
+    a harness bug reported as ``pipeline``, say — and printing "inspect the
+    per-turn used_fallback flags" under those sends triage the wrong way.
+    """
+    bar = "━" * 72
+    print(f"\n{bar}", file=sys.stderr)
+    print(f"[smoke] FAILED — class: {failure_class.upper()} "
+          f"(exit {EXIT_CODES[failure_class]})", file=sys.stderr)
+    print(bar, file=sys.stderr)
+    for f in failures:
+        print(f"  ✗ {f}", file=sys.stderr)
+    print(f"\n  → {remedy or REMEDIES[failure_class]}", file=sys.stderr)
+    print(f"{bar}\n", file=sys.stderr)
+
+
+def _write_report(report_path: Optional[Path], results: Dict[str, Any]) -> None:
+    if not report_path:
+        return
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    report_path.write_text(json.dumps(results, indent=2), encoding="utf-8")
+    print(f"[smoke] Report written to {report_path}")
 
 
 # ── Smoke run ─────────────────────────────────────────────────────────────────
 
 
-def _drain(stream: object, sink: Optional["deque"] = None) -> None:
+class StderrTail:
+    """The most recent lines of a child's stderr, safe to read while it writes.
+
+    A bare ``deque`` is the obvious container and the wrong one.  The draining
+    thread keeps appending for as long as the child is alive, and the tails are
+    dumped from ``run_smoke``'s *except* handler — before the ``finally`` block
+    stops either child — so the reader and the writer genuinely overlap.
+    Iterating a deque another thread is appending to raises
+    ``RuntimeError: deque mutated during iteration``, which would escape the
+    handler and throw away the classified verdict it had just assembled: the
+    same way an already-exhausted ``Deadline.finish()`` once turned a
+    reportable timeout into a bare traceback.  And it would do it on exactly
+    the failures where the child is still talking — a timeout whose abandoned
+    generation is still running, a 5xx being logged.
+
+    So appends and reads take a lock, and readers get a snapshot rather than a
+    live view of the buffer.
+    """
+
+    def __init__(self, maxlen: int = 200) -> None:
+        self._lines: deque = deque(maxlen=maxlen)
+        self._lock = threading.Lock()
+
+    def append(self, line: str) -> None:
+        with self._lock:
+            self._lines.append(line)
+
+    def snapshot(self) -> List[str]:
+        with self._lock:
+            return list(self._lines)
+
+
+def _drain(stream: object, sink: Optional[StderrTail] = None) -> None:
     """Consume a subprocess pipe in the background to avoid a full-buffer deadlock.
 
-    When ``sink`` (a bounded deque) is given, the most recent lines are retained
-    so they can be surfaced if the smoke fails.  The child's stderr is otherwise
-    discarded, which makes a server-side 500 undiagnosable from the CI logs — the
-    client only ever sees ``HTTPError: 500`` with no server traceback.
+    When ``sink`` is given, the most recent lines are retained so they can be
+    surfaced if the smoke fails.  The child's stderr is otherwise discarded,
+    which makes a server-side 500 undiagnosable from the CI logs — the client
+    only ever sees ``HTTPError: 500`` with no server traceback.
     """
     try:
         for raw in stream:  # type: ignore[attr-defined]
@@ -254,198 +1651,680 @@ def _drain(stream: object, sink: Optional["deque"] = None) -> None:
         pass
 
 
+def _dump_stderr_tails(tails: Sequence[tuple[str, StderrTail]]) -> None:
+    for label, tail in tails:
+        # One snapshot, used for both the count and the lines: reading the
+        # length separately would let the header promise more lines than the
+        # dump prints while the child is still writing.
+        lines = tail.snapshot()
+        if lines:
+            print(f"\n[smoke] ── {label} stderr (last {len(lines)} lines) ──", file=sys.stderr)
+            for line in lines:
+                print(f"  {label[:4]}| {line}", file=sys.stderr)
+
+
+def _crashed_child(procs: Sequence[tuple[str, Optional[subprocess.Popen]]]) -> Optional[str]:
+    """Return a description of the first child that has exited, if any."""
+    for label, proc in procs:
+        if proc is not None and proc.poll() is not None:
+            return f"{label} exited with code {proc.returncode}"
+    return None
+
+
+def _debug_flags_by_turn(base: str, session_id: str, timeout: float) -> Dict[int, Dict[str, Any]]:
+    """Fetch per-turn parse flags from the dev debug endpoint, keyed by game turn.
+
+    Best-effort: the debug drawer is a diagnostic, so a failure here must not
+    turn a healthy smoke red.  That holds for the *shape* of the answer as well
+    as for the transport — an entry without a ``turn_number`` would otherwise
+    raise a KeyError out of this helper, escape to ``run_smoke``'s catch-all and
+    report an otherwise-green run as a harness bug (exit 5).  A payload we
+    cannot key by turn is the same situation as no payload at all: the fallback
+    check does not run, and ``evaluate_turns`` warns that it did not.
+    """
+    try:
+        debug = _request_json(f"{base}/sessions/{session_id}/debug", timeout=timeout)
+    except SmokeFailure as exc:
+        print(f"[smoke] Could not read debug flags ({exc}); continuing.")
+        return {}
+    turns = debug.get("turns")
+    if not isinstance(turns, list):
+        print(f"[smoke] Debug endpoint returned no turns list ({turns!r}); continuing.")
+        return {}
+    flags: Dict[int, Dict[str, Any]] = {}
+    for entry in turns:
+        if isinstance(entry, dict) and isinstance(entry.get("turn_number"), int):
+            flags[entry["turn_number"]] = entry
+        else:
+            print(f"[smoke] Ignoring unkeyable debug turn entry: {entry!r}")
+    return flags
+
+
 def run_smoke(
     model_id: str,
     ci_hardware_factor: float,
     report_path: Optional[Path],
-) -> bool:
-    """Run the scripted smoke session and check budgets.  Returns True on pass."""
-    model_path = _find_model_path(model_id)
-    # The turn must be allowed to run past the CI ceiling before we judge it, so
-    # both the adapter timeout and our own POST timeout are derived from the
-    # full-response ceiling with headroom rather than a fixed 30 s / 180 s.
-    full_ceiling_s = (BUDGETS_MS["full_response_ms"] * ci_hardware_factor * REGRESSION_TOLERANCE) / 1000
-    llama_timeout_s = max(180.0, full_ceiling_s + 60.0)
-    turn_timeout_s = llama_timeout_s + 30.0
-    results: dict = {
+    *,
+    model_sha256: Optional[str] = None,
+    models_dir: Path = MODELS_DIR,
+    wall_clock_budget_s: float = DEFAULT_WALL_CLOCK_BUDGET_S,
+) -> int:
+    """Run the full scripted smoke.  Returns the process exit code."""
+    clock = Deadline(wall_clock_budget_s)
+    # A call must be allowed to run past the latency we are about to judge it
+    # on, or whoever gives up first decides the verdict instead of us: the
+    # adapter's own timeout surfaces as HTTP 500 (class `runtime`, "the server
+    # crashed") and our POST timeout surfaces as a transport error (also
+    # `runtime`), when the truth is a slow model — class `budget`.
+    #
+    # The debrief, not a turn, is the slowest single generation (one long
+    # structured document against one NPC reply), so the adapter ceiling is
+    # sized for it: DEBRIEF_SLOWDOWN_FACTOR × the full-response CI ceiling.
+    # Our own POST timeout then sits above the adapter's, so when something
+    # really does hang it is the adapter that reports it, with a message that
+    # names the knob.  The run's wall-clock deadline — not these timeouts — is
+    # what bounds the job.
+    full_ceiling_s = (
+        BUDGETS_MS["full_response_ms"] * ci_hardware_factor * REGRESSION_TOLERANCE
+    ) / 1000
+    llama_timeout_s = max(300.0, full_ceiling_s * DEBRIEF_SLOWDOWN_FACTOR + 60.0)
+    request_timeout_s = llama_timeout_s + 30.0
+
+    results: Dict[str, Any] = {
         "model_id": model_id,
+        "scenario_id": SCENARIO_ID,
         "ci_hardware_factor": ci_hardware_factor,
-        "budgets_ms": BUDGETS_MS,
+        "budgets_ms": dict(BUDGETS_MS),
         "ci_budgets_ms": {k: v * ci_hardware_factor for k, v in BUDGETS_MS.items()},
         "regression_tolerance": REGRESSION_TOLERANCE,
+        "wall_clock_budget_s": wall_clock_budget_s,
         "measured_ms": {},
+        "turns": [],
+        "debrief": None,
         "verdict": "pending",
+        "failure_class": None,
+        "exit_code": None,
         "failures": [],
+        "warnings": [],
     }
 
-    with tempfile.TemporaryDirectory(prefix="convsim-smoke-") as tmp:
-        data_dir = Path(tmp)
-        # Bounded tails of each child's stderr, surfaced only if the smoke fails
-        # so a server-side error is diagnosable from the CI logs.
-        llama_stderr_tail: deque = deque(maxlen=200)
-        core_stderr_tail: deque = deque(maxlen=200)
+    model_path = models_dir / f"{model_id}.gguf"
+    llama_proc: Optional[subprocess.Popen] = None
+    core_proc: Optional[subprocess.Popen] = None
+    llama_stderr_tail = StderrTail()
+    core_stderr_tail = StderrTail()
+
+    # convsim-core's throwaway data directory (SQLite db, WAL, logs).  Created
+    # and removed by hand rather than with a `with TemporaryDirectory(...)`
+    # inside the try: that block exits *before* the finally below, so the
+    # directory would be deleted while convsim-core is still writing to it, and
+    # an rmtree that lost that race would surface as an unexpected-error
+    # `pipeline` verdict on an otherwise green run.  The finally stops both
+    # children first, then removes the tree with errors ignored — cleanup must
+    # never overturn a verdict.
+    data_dir = Path(tempfile.mkdtemp(prefix="convsim-smoke-"))
+    try:
+        # ── checksum ──────────────────────────────────────────────────────────
+        clock.enter("checksum")
+        if model_sha256:
+            print(f"[smoke] Verifying model checksum before use: {model_path}")
+            verify_model_checksum(model_path, model_sha256)
+        elif not model_path.exists():
+            raise SmokeFailure(
+                FailureClass.DOWNLOAD,
+                f"Model file not found: {model_path}. Run with --download-only first, "
+                "or check that the model cache is populated.",
+            )
+        else:
+            results["warnings"].append(
+                "No --model-sha256 given: checksum drift was not checked before the run"
+            )
 
-        print(f"\n[smoke] Starting llama-server on port {LLAMA_SERVER_PORT} with model: {model_path.name}")
+        # ── runtime ───────────────────────────────────────────────────────────
+        clock.enter("runtime_start")
+        # Both ports before anything is spawned: the smoke has to serve itself,
+        # or its proof that a real pinned model drove the conversation is worth
+        # nothing.
+        _assert_port_free(LLAMA_SERVER_PORT, "llama-server")
+        _assert_port_free(CORE_PORT, "convsim-core")
+        print(f"\n[smoke] Starting llama-server on port {LLAMA_SERVER_PORT} "
+              f"with model: {model_path.name}")
         llama_proc = _start_llama_server(model_path, LLAMA_SERVER_PORT)
         threading.Thread(target=_drain, args=(llama_proc.stderr, llama_stderr_tail), daemon=True).start()
         threading.Thread(target=_drain, args=(llama_proc.stdout,), daemon=True).start()
 
-        core_proc: Optional[subprocess.Popen] = None
-        try:
-            print("[smoke] Waiting for llama-server (model load)…")
-            _wait_for_http(
-                f"http://127.0.0.1:{LLAMA_SERVER_PORT}/v1/models",
-                timeout_s=300.0,
-                label="llama-server",
-            )
-            print("[smoke] llama-server ready.")
-
-            print(f"[smoke] Starting convsim-core on port {CORE_PORT}…")
-            core_proc = _start_core(data_dir, CORE_PORT, LLAMA_SERVER_PORT, llama_timeout_s)
-            threading.Thread(target=_drain, args=(core_proc.stderr, core_stderr_tail), daemon=True).start()
-            threading.Thread(target=_drain, args=(core_proc.stdout,), daemon=True).start()
+        print("[smoke] Waiting for llama-server (model load)…")
+        _wait_for_http(
+            f"http://127.0.0.1:{LLAMA_SERVER_PORT}/v1/models",
+            timeout_s=clock.cap(300.0),
+            label="llama-server",
+            proc=llama_proc,
+        )
+        print("[smoke] llama-server ready.")
+
+        print(f"[smoke] Starting convsim-core on port {CORE_PORT}…")
+        core_proc = _start_core(data_dir, CORE_PORT, LLAMA_SERVER_PORT, llama_timeout_s)
+        threading.Thread(target=_drain, args=(core_proc.stderr, core_stderr_tail), daemon=True).start()
+        threading.Thread(target=_drain, args=(core_proc.stdout,), daemon=True).start()
+
+        base = f"http://127.0.0.1:{CORE_PORT}/api"
+        _wait_for_http(
+            f"{base}/health",
+            timeout_s=clock.cap(120.0),
+            label="convsim-core",
+            proc=core_proc,
+        )
 
-            _wait_for_http(
-                f"http://127.0.0.1:{CORE_PORT}/api/health",
-                timeout_s=120.0,
-                label="convsim-core",
+        # Sanity-check that the real runtime is wired (not the fake default).
+        health = _request_json(f"{base}/health", timeout=clock.cap(30.0))
+        # `or {}`, not a `{}` default: an absent llm_runtime already falls
+        # through to the clean "core reports None" verdict below, but an explicit
+        # JSON null would make .get() raise AttributeError and report a changed
+        # /health contract as a bug in this harness.  Same reason as _excerpt_any.
+        runtime_id = (health.get("llm_runtime") or {}).get("runtime_id")
+        results["runtime_id"] = runtime_id
+        print(f"[smoke] convsim-core ready (runtime_id={runtime_id}).")
+        if runtime_id != "llama_cpp":
+            raise SmokeFailure(
+                FailureClass.PIPELINE,
+                f"Expected runtime_id 'llama_cpp' but core reports {runtime_id!r}. "
+                "Check CONVSIM_RUNTIME_ID wiring — the smoke must not run on a fake runtime.",
             )
-            base = f"http://127.0.0.1:{CORE_PORT}/api"
-
-            # Sanity-check that the real runtime is wired (not the fake default).
-            health = _get_json(f"{base}/health")
-            runtime_id = health.get("llm_runtime", {}).get("runtime_id")
-            print(f"[smoke] convsim-core ready (runtime_id={runtime_id}).")
-            if runtime_id != "llama_cpp":
-                raise RuntimeError(
-                    f"Expected runtime_id 'llama_cpp' but core reports {runtime_id!r}. "
-                    "Check CONVSIM_RUNTIME_ID wiring."
-                )
 
-            # Create → start → turn against the built-in behavioral_interview scenario.
-            session = _post_json(f"{base}/sessions", {
-                "scenario_id": "behavioral_interview",
-                "difficulty": "standard",
-                "player_role_name": "Smoke Tester",
-                "language": "en",
-                "input_mode": "text-only",
-                "tts_enabled": False,
-                "show_state_meters": False,
-                "save_transcript": False,
-                "seed": 42,
-            })
-            session_id = session["session_id"]
-            print(f"[smoke] Session {session_id} created. Starting session…")
-            # /start generates the NPC opening — a real model call, so it needs
-            # the same generous timeout as the turn on slow CI hardware.
-            _post_json(f"{base}/sessions/{session_id}/start", {}, timeout=turn_timeout_s)
-            print("[smoke] NPC opening received. Submitting player turn…")
-
-            # Submit a player turn and time the synchronous end-to-end response.
-            t_start = time.monotonic()
-            turn = _post_json(
+        # ── conversation ──────────────────────────────────────────────────────
+        clock.enter("conversation")
+        session = _request_json(f"{base}/sessions", payload={
+            "scenario_id": SCENARIO_ID,
+            "difficulty": "standard",
+            "player_role_name": "Smoke Tester",
+            "language": "en",
+            "input_mode": "text-only",
+            "tts_enabled": False,
+            "show_state_meters": False,
+            "save_transcript": False,  # no transcript file written during smoke runs
+            "seed": 42,
+        }, timeout=clock.cap(30.0))
+        session_id = session["session_id"]
+        results["session_id"] = session_id
+        print(f"[smoke] Session {session_id} created. Starting session…")
+
+        # /start replays the scenario's authored opening line — no model
+        # call — so it is measured for the documented session-start metric but
+        # needs no inference-sized timeout.
+        t0 = time.monotonic()
+        start = _request_json(
+            f"{base}/sessions/{session_id}/start",
+            payload={},
+            timeout=clock.cap(60.0),
+        )
+        session_start_ms = (time.monotonic() - t0) * 1000
+        opening = _npc_turn_content(start.get("events", []))
+        print(f"[smoke] NPC opening (authored) in {session_start_ms:.0f} ms: "
+              f"{_excerpt(opening)!r}")
+        results["measured_ms"]["session_start_ms"] = round(session_start_ms)
+        results["turns"].append({
+            "label": "npc_opening",
+            "turn_number": 0,
+            "model_generated": False,
+            "latency_ms": round(session_start_ms),
+            "npc_excerpt": _excerpt(opening),
+        })
+
+        turn_latencies: List[float] = []
+        ended_by_scenario = False
+        for i, player_text in enumerate(SCRIPTED_PLAYER_TURNS, start=1):
+            clock.check()
+            print(f"[smoke] Player turn {i}/{len(SCRIPTED_PLAYER_TURNS)}…")
+            t0 = time.monotonic()
+            turn = _request_json(
                 f"{base}/sessions/{session_id}/turn",
-                {"content": SMOKE_PLAYER_TURN},
-                timeout=turn_timeout_s,
+                payload={"content": player_text},
+                timeout=clock.cap(request_timeout_s),
             )
-            full_ms = (time.monotonic() - t_start) * 1000
+            latency_ms = (time.monotonic() - t0) * 1000
+            turn_latencies.append(latency_ms)
             npc_text = _npc_turn_content(turn.get("events", []))
-            print(f"[smoke] Full response: {full_ms:.0f} ms | NPC said: {npc_text[:80]!r}")
-            if not npc_text:
-                raise RuntimeError("Turn completed but no npc_turn content was returned")
-
-            results["measured_ms"] = {"full_response_ms": round(full_ms)}
-
-            # Check budgets
-            failures = []
-            for key, budget_ms in BUDGETS_MS.items():
-                measured = results["measured_ms"].get(key)
-                if measured is None:
-                    continue
-                ci_budget = budget_ms * ci_hardware_factor
-                ceiling = ci_budget * REGRESSION_TOLERANCE
-                status = "PASS" if measured <= ceiling else "FAIL"
-                print(
-                    f"  {key}: {measured:.0f} ms "
-                    f"(CI budget {ci_budget:.0f} ms × {REGRESSION_TOLERANCE} = {ceiling:.0f} ms) "
-                    f"→ {status}"
-                )
-                if status == "FAIL":
-                    failures.append(
-                        f"{key} = {measured:.0f} ms exceeds CI ceiling {ceiling:.0f} ms "
-                        f"(budget {budget_ms} ms × {ci_hardware_factor} × {REGRESSION_TOLERANCE})"
-                    )
+            replayed_opening = _replays_opening(npc_text, opening)
+            print(f"  {latency_ms:.0f} ms | NPC: {_excerpt(npc_text)!r}"
+                  + (" [replays the authored opening]" if replayed_opening else ""))
+            results["turns"].append({
+                "label": f"player_turn_{i}",
+                "turn_number": i,
+                "model_generated": True,
+                "latency_ms": round(latency_ms),
+                "npc_excerpt": _excerpt(npc_text),
+                # Real model output that is not a reply — see _replays_opening.
+                # The parse flags below cannot tell this from a healthy turn.
+                "replayed_opening": replayed_opening,
+                "ending_type": turn.get("ending_type"),
+                # What the model volunteered for the debrief to score.  Recorded
+                # per turn so an unscored debrief can be attributed without
+                # re-running anything — see evaluate_debrief.
+                "rubric_observation_count": _rubric_observation_count(
+                    turn.get("events", [])
+                ),
+            })
+            if turn.get("ending_type"):
+                # The turn pipeline already moved the session to 'Ended', so
+                # POST /end would 409 — see end_session() in the sessions
+                # router.  Record that and skip the explicit end below.
+                ended_by_scenario = True
+                print(f"[smoke] Scenario ended early (ending_type={turn['ending_type']}); "
+                      "stopping the scripted turns.")
+                break
 
-            results["failures"] = failures
-            results["verdict"] = "pass" if not failures else "fail"
-
-        except BaseException as exc:
-            # The client-side error (e.g. HTTP 500 from a turn) says nothing about
-            # what went wrong inside convsim-core.  Surface the captured stderr
-            # tails so the real traceback is visible in the CI logs, then re-raise.
-            print(f"\n[smoke] ERROR during run: {exc!r}", file=sys.stderr)
-            for label, tail in (("convsim-core", core_stderr_tail), ("llama-server", llama_stderr_tail)):
-                if tail:
-                    print(f"\n[smoke] ── {label} stderr (last {len(tail)} lines) ──", file=sys.stderr)
-                    for line in tail:
-                        print(f"  {label[:4]}| {line}", file=sys.stderr)
-            raise
-        finally:
-            for proc in (core_proc, llama_proc):
-                if proc is None:
-                    continue
-                proc.terminate()
-                try:
-                    proc.wait(timeout=10)
-                except subprocess.TimeoutExpired:
-                    proc.kill()
-
-    if report_path:
-        report_path.parent.mkdir(parents=True, exist_ok=True)
-        report_path.write_text(json.dumps(results, indent=2))
-        print(f"\n[smoke] Report written to {report_path}")
-
-    if results["failures"]:
-        print("\n[smoke] FAILED — latency budget regression detected:")
-        for f in results["failures"]:
-            print(f"  ✗ {f}")
-        return False
+        results["scripted_turns_played"] = len(turn_latencies)
+        if len(turn_latencies) < len(SCRIPTED_PLAYER_TURNS):
+            # Not a failure: the model is allowed to close the conversation
+            # (session_control.continue_session), and the product handled it.
+            # But the job exists to play a *multi-turn* conversation, and a run
+            # that played one turn proves less than one that played three — say
+            # so in the summary rather than let a short green run read like a
+            # full one.
+            results["warnings"].append(
+                f"Only {len(turn_latencies)} of {len(SCRIPTED_PLAYER_TURNS)} scripted "
+                f"player turns ran: the NPC ended the session early (ending_type="
+                f"{results['turns'][-1].get('ending_type')!r}), so this run proves "
+                "less about multi-turn behaviour than a full playthrough"
+            )
+
+        # The median is the headline full-response figure: a single unlucky
+        # turn (runner steal, a long NPC answer) must not flap the nightly,
+        # while a real regression moves every turn and so moves the median.
+        results["measured_ms"]["full_response_ms"] = round(statistics.median(turn_latencies))
+        results["measured_ms"]["full_response_max_ms"] = round(max(turn_latencies))
+
+        # Per-turn parse flags tell us whether the model's structured output
+        # was accepted or quietly replaced by the safe fallback.
+        flags = _debug_flags_by_turn(base, session_id, clock.cap(30.0))
+        for turn in results["turns"]:
+            flag = flags.get(turn["turn_number"])
+            if not flag:
+                continue
+            # Only the flags the payload actually carries.  `.get()` would read
+            # an absent flag as False, which is not "the model did not fall
+            # back" — it is "we were not told" — and writing the key at all is
+            # worse than recording the wrong value: evaluate_turns decides
+            # whether the fallback check ran by whether the key is *present*,
+            # so a payload that keys a turn but omits the flag would both
+            # credit the turn as a real reply and suppress the warning that
+            # says nothing checked.  Same hole _debug_flags_by_turn closes for
+            # an entry it cannot key, one level down.
+            for name in ("used_fallback", "used_native_structured_output"):
+                if name in flag:
+                    turn[name] = bool(flag[name])
+
+        # ── debrief ───────────────────────────────────────────────────────────
+        clock.enter("debrief")
+        if ended_by_scenario:
+            print("[smoke] Session already Ended by the scenario; skipping /end.")
+        else:
+            print("[smoke] Ending session…")
+            _request_json(
+                f"{base}/sessions/{session_id}/end", payload={}, timeout=clock.cap(30.0)
+            )
+        print("[smoke] Generating debrief (real model call)…")
+        t0 = time.monotonic()
+        debrief = _request_json(
+            f"{base}/sessions/{session_id}/debrief",
+            payload={},
+            timeout=clock.cap(request_timeout_s),
+        )
+        debrief_ms = (time.monotonic() - t0) * 1000
+        results["measured_ms"]["debrief_ms"] = round(debrief_ms)
+        results["debrief"] = {
+            "outcome": debrief.get("outcome"),
+            "total_turns": debrief.get("total_turns"),
+            "scores": debrief.get("scores"),
+            "overall_score": debrief.get("overall_score"),
+            # _excerpt_any, not _excerpt: this report is assembled *before*
+            # evaluate_debrief runs, so a summary that is not text would raise
+            # here and the classified verdict evaluate_debrief was about to
+            # reach would never be assembled.  See _excerpt_any.
+            "summary_excerpt": _excerpt_any(debrief.get("summary")),
+            "strength_count": len(debrief.get("strengths") or []),
+            "improvement_count": len(debrief.get("improvements") or []),
+            "turning_point_count": len(debrief.get("turning_points") or []),
+            "used_fallback": bool(debrief.get("used_fallback")),
+            "latency_ms": round(debrief_ms),
+        }
+        # Everything the NPC turns offered the debrief to score.  An unscored
+        # debrief means something different depending on whether this is zero —
+        # and something different again when we could not read it, so an
+        # unreadable turn payload yields None rather than a confident zero.
+        # A positive total is still conclusive even with an unreadable turn
+        # among the others: observations *were* seen, so the debrief losing
+        # them is the cause whatever the missing turn carried.
+        observations_seen = _total_rubric_observations(results["turns"])
+        results["rubric_observations_seen"] = observations_seen
+        print(f"[smoke] Debrief in {debrief_ms:.0f} ms | "
+              f"overall_score={debrief.get('overall_score')} "
+              f"scores={debrief.get('scores')} "
+              f"rubric_observations_from_turns="
+              f"{'unknown' if observations_seen is None else observations_seen}")
+
+        # ── assertions ────────────────────────────────────────────────────────
+        clock.enter("assertions")
+        turn_failures, turn_warnings = evaluate_turns(results["turns"])
+        debrief_failures, debrief_warnings = evaluate_debrief(
+            debrief,
+            rubric_observations_seen=observations_seen,
+            # A conversation that fell back on every generated turn reaches the
+            # debrief with nothing to score for a reason that has nothing to do
+            # with rubric prompting — see UNSCORED_AFTER_FALLBACK_NOTE.
+            turns_all_fell_back=_all_generated_turns_fell_back(results["turns"]),
+        )
+        results["warnings"] += turn_warnings + debrief_warnings
+        pipeline_failures = turn_failures + debrief_failures
+        if pipeline_failures:
+            results["failures"] = pipeline_failures
+            raise SmokeFailure(
+                FailureClass.PIPELINE,
+                f"{len(pipeline_failures)} end-to-end assertion(s) failed",
+            )
 
-    print(f"\n[smoke] PASSED — all latency budgets met on model {model_id}.")
-    return True
+        # ── budget ────────────────────────────────────────────────────────────
+        clock.enter("budget")
+        budget_failures, budget_lines = evaluate_budgets(
+            results["measured_ms"], ci_hardware_factor
+        )
+        for line in budget_lines:
+            print(f"  {line}")
+        if budget_failures:
+            results["failures"] = budget_failures
+            raise SmokeFailure(
+                FailureClass.BUDGET, f"{len(budget_failures)} latency budget(s) exceeded"
+            )
+
+        results["verdict"] = "pass"
+        results["exit_code"] = 0
+
+    except SmokeFailure as exc:
+        # Precedence, strongest evidence first:
+        #   1. A crashed child outranks whatever the client saw — "core exited
+        #      with code 1" is the real story, not the connection reset it caused.
+        #   2. An exhausted wall clock outranks the transport error it produced:
+        #      cap() shrinks each request timeout to the budget that is left, so
+        #      the last request before the deadline dies client-side and reports
+        #      itself as RUNTIME ("the server never answered"). The honest
+        #      verdict is TIMEOUT, which is the class whose remedy says to look
+        #      at phase_durations_s.
+        crashed = _crashed_child((("llama-server", llama_proc), ("convsim-core", core_proc)))
+        out_of_time = not crashed and clock.remaining_s <= 0
+        if crashed:
+            failure_class = FailureClass.RUNTIME
+        elif out_of_time:
+            failure_class = FailureClass.TIMEOUT
+        else:
+            failure_class = exc.failure_class
+        results["verdict"] = "fail"
+        results["failure_class"] = failure_class
+        results["exit_code"] = EXIT_CODES[failure_class]
+        results["failed_phase"] = exc.phase or clock.phase
+        # Record the remedy in the artifact too: the failure table in
+        # docs/real-model-smoke.md sends triage to the JSON report for several
+        # classes, and a report that names a class without the advice attached
+        # makes the reader go and look the mapping up again.  The failure's own
+        # remedy only survives while its class does — once a crashed child or an
+        # exhausted clock has overruled the class, its advice describes a
+        # different failure than the one being reported.
+        results["remedy"] = (
+            exc.remedy if failure_class == exc.failure_class else REMEDIES[failure_class]
+        )
+        # A `runtime` verdict with both children still alive did not come from a
+        # crash, and the class remedy's first instruction — read the child stderr
+        # tail — is advice about a tail that shows a healthy server.  The common
+        # cause is a generation the adapter cut off, i.e. a latency regression.
+        # See runtime_without_crash_remedy.  Only when the failure was using the
+        # class default: the port-conflict failure already carries a remedy of
+        # its own, and it is right about a run where no child was ever started.
+        if (
+            failure_class == FailureClass.RUNTIME
+            and not crashed
+            and results["remedy"] == REMEDIES[FailureClass.RUNTIME]
+        ):
+            results["remedy"] = runtime_without_crash_remedy(llama_timeout_s)
+        if not results["failures"]:
+            results["failures"] = [str(exc)]
+        if crashed:
+            results["failures"].insert(0, crashed)
+        elif out_of_time and exc.failure_class != FailureClass.TIMEOUT:
+            results["failures"].insert(
+                0,
+                f"Wall-clock budget of {wall_clock_budget_s:.0f} s exhausted during phase "
+                f"{results['failed_phase']!r} (elapsed {clock.elapsed_s:.0f} s)",
+            )
+        if failure_class == FailureClass.TIMEOUT:
+            # TIMEOUT is reached before the budget phase can run for exactly the
+            # regression `budget` is meant to catch, so attach the measurement
+            # that tells the two apart.  See timeout_latency_evidence.
+            evidence = timeout_latency_evidence(results["turns"], full_ceiling_s * 1000)
+            if evidence:
+                results["failures"].append(evidence)
+        _dump_stderr_tails((("convsim-core", core_stderr_tail), ("llama-server", llama_stderr_tail)))
+        _print_banner(failure_class, results["failures"], remedy=results["remedy"])
+
+    # Exception, not BaseException: a Ctrl-C should still stop the children (the
+    # finally block below) but must not be dressed up as a product failure.
+    except Exception as exc:
+        results["verdict"] = "fail"
+        results["failure_class"] = FailureClass.PIPELINE
+        results["exit_code"] = EXIT_CODES[FailureClass.PIPELINE]
+        results["failed_phase"] = clock.phase
+        results["failures"] = [
+            f"Unexpected error in phase {clock.phase!r}: {exc!r} — this is most likely "
+            "a bug in the smoke harness itself, not in the product "
+            "(tests/scripts/test_nightly_model_smoke.py covers its logic)"
+        ]
+        # Borrowing `pipeline`'s exit code, not its cause: the class remedy
+        # ("inspect the per-turn used_fallback flags") would point triage at
+        # model output when the traceback above is the thing to read.
+        results["remedy"] = (
+            "Read the traceback above and the phase it names. This is a harness "
+            "bug, not a product verdict — add a case to "
+            "tests/scripts/test_nightly_model_smoke.py once you have it."
+        )
+        _dump_stderr_tails((("convsim-core", core_stderr_tail), ("llama-server", llama_stderr_tail)))
+        _print_banner(FailureClass.PIPELINE, results["failures"], remedy=results["remedy"])
+
+    finally:
+        for proc in (core_proc, llama_proc):
+            if proc is None:
+                continue
+            proc.terminate()
+            try:
+                proc.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+        # Only now that convsim-core is stopped is its data directory safe to
+        # remove.  errors ignored: a stray file left behind is a tidiness
+        # problem, not a reason to turn a decided verdict into a traceback.
+        shutil.rmtree(data_dir, ignore_errors=True)
+        results["phase_durations_s"] = clock.finish()
+        results["wall_clock_s"] = round(clock.elapsed_s, 1)
+
+    if results["verdict"] == "pass":
+        generated = sum(1 for t in results["turns"] if t.get("model_generated"))
+        print(f"\n[smoke] PASSED — real model {model_id} produced "
+              f"{generated} NPC turn(s) and a scored debrief "
+              f"(overall_score={(results['debrief'] or {}).get('overall_score')}) "
+              f"in {results['wall_clock_s']:.0f} s.")
+        for warning in results["warnings"]:
+            print(f"  ! {warning}")
+
+    _write_report(report_path, results)
+    _write_step_summary(results)
+    return int(results["exit_code"] or 0)
 
 
 # ── CLI ───────────────────────────────────────────────────────────────────────
 
 
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__,
-                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+class _ClassifiedArgumentParser(argparse.ArgumentParser):
+    """An ``ArgumentParser`` whose usage errors land in a known failure class.
+
+    argparse exits 2 on a bad command line, and EXIT_CODES reads 2 as
+    ``download`` — whose remedy is "nothing about the app changed, re-run the
+    job". That would send a triager round a loop re-running a job that cannot
+    succeed. A malformed invocation is a harness problem, which is exactly what
+    ``pipeline`` means for everything else the harness did not anticipate.
+    """
+
+    def error(self, message: str) -> NoReturn:
+        self.print_usage(sys.stderr)
+        _print_banner(
+            FailureClass.PIPELINE,
+            [f"Invalid command line: {message}"],
+            remedy="Fix the invocation. See the usage above, the examples in this "
+                   "file's docstring, or the smoke step in "
+                   ".github/workflows/model-smoke-nightly.yml.",
+        )
+        sys.exit(EXIT_CODES[FailureClass.PIPELINE])
+
+
+def _use_line_buffered_output() -> None:
+    """Make stdout arrive in step with stderr, line by line.
+
+    CI captures this script's stdout through a pipe, so Python block-buffers it
+    (8 KB) while stderr stays unbuffered.  Two things break as a result, and
+    both are the log a triager actually reads:
+
+    * nothing streams.  A run spends ~11 min inside the conversation and
+      debrief phases, and the "Player turn 2/3…" lines that say where it got to
+      only reach the log when the buffer fills or the process exits.
+    * the failure banner arrives *before* the context it explains.  The banner
+      and the child stderr tails go to stderr; the phase progress, per-turn
+      latencies and budget arithmetic go to stdout, so the merged log shows the
+      verdict first and the evidence afterwards.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(line_buffering=True)  # type: ignore[union-attr]
+        except (AttributeError, ValueError):
+            # Not a TextIOWrapper (pytest's capture, a redirect_stdout buffer).
+            # Buffering is then the caller's concern, not ours.
+            pass
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    _use_line_buffered_output()
+    parser = _ClassifiedArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("--print-registry-model", metavar="ROLE", default=None,
+                        help="Resolve the registry model with this role (e.g. 'starter'), "
+                             "write id/url/sha256 to $GITHUB_OUTPUT, and exit.")
     parser.add_argument("--download-only", action="store_true",
                         help="Download and verify the model file, then exit.")
+    parser.add_argument("--verify-only", action="store_true",
+                        help="Verify the on-disk model against --model-sha256, then exit. "
+                             "A mismatch deletes the file, so a re-run re-downloads it "
+                             "instead of re-verifying the same bad bytes.")
     parser.add_argument("--model-url", default=None,
                         help="Download URL (required for --download-only).")
     parser.add_argument("--model-sha256", default=None,
-                        help="Expected SHA-256 hex (required for --download-only).")
-    parser.add_argument("--model-id", required=True,
+                        help="Expected SHA-256 hex. Required for --download-only/--verify-only; "
+                             "on a full run the model is re-verified before use when given.")
+    parser.add_argument("--model-id", default=None,
                         help="Registry model ID (e.g. qwen3-4b-instruct-q4_k_m).")
+    parser.add_argument("--models-dir", default=None,
+                        help=f"Directory holding <model-id>.gguf (default {MODELS_DIR}).")
     parser.add_argument("--ci-hardware-factor", type=float, default=1.0,
                         help="Multiplier applied to documented budgets for CI hardware (default 1.0).")
-    parser.add_argument("--report-path", default=None,
-                        help="Write JSON report to this path.")
+    parser.add_argument("--wall-clock-budget-s", type=float, default=DEFAULT_WALL_CLOCK_BUDGET_S,
+                        help=f"Fail with exit {EXIT_CODES[FailureClass.TIMEOUT]} if the run "
+                             f"exceeds this many seconds (default {DEFAULT_WALL_CLOCK_BUDGET_S:.0f}).")
+    parser.add_argument("--report-path", default=None, help="Write JSON report to this path.")
 
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
+    models_dir = Path(args.models_dir) if args.models_dir else MODELS_DIR
 
-    if args.download_only:
-        if not args.model_url or not args.model_sha256:
-            parser.error("--model-url and --model-sha256 are required with --download-only")
-        print(f"[smoke] Downloading model {args.model_id}…")
-        download_model(args.model_url, args.model_sha256, args.model_id)
-        return 0
+    try:
+        if args.print_registry_model:
+            model = resolve_registry_model(args.print_registry_model)
+            _emit_github_output({
+                "model_id": model["id"],
+                "model_url": model["url"],
+                "model_sha256": model["sha256"],
+                "model_size_gb": model["size_gb"],
+            })
+            return 0
+
+        if not args.model_id:
+            parser.error("--model-id is required")
+
+        if args.download_only:
+            if not args.model_url or not args.model_sha256:
+                parser.error("--model-url and --model-sha256 are required with --download-only")
+            print(f"[smoke] Downloading model {args.model_id}…")
+            download_model(args.model_url, args.model_sha256, args.model_id, models_dir)
+            return 0
+
+        if args.verify_only:
+            if not args.model_sha256:
+                parser.error("--model-sha256 is required with --verify-only")
+            print(f"[smoke] Verifying cached model {args.model_id}…")
+            verify_model_checksum(models_dir / f"{args.model_id}.gguf", args.model_sha256)
+            return 0
+
+        # Inside the handler, not after it: run_smoke classifies everything in
+        # its own try/finally, but its prologue (the throwaway data directory)
+        # and its epilogue (the report and step-summary writes that follow the
+        # verdict) sit outside that block.  An OSError there — a full disk is
+        # the realistic one, right after a 2.5 GB download and a 2.5 GB cache
+        # save — would escape as a bare traceback and exit 1.
+        return run_smoke(
+            args.model_id,
+            args.ci_hardware_factor,
+            Path(args.report_path) if args.report_path else None,
+            model_sha256=args.model_sha256,
+            models_dir=models_dir,
+            wall_clock_budget_s=args.wall_clock_budget_s,
+        )
 
-    report = Path(args.report_path) if args.report_path else None
-    passed = run_smoke(args.model_id, args.ci_hardware_factor, report)
-    return 0 if passed else 1
+    except SmokeFailure as exc:
+        _print_banner(exc.failure_class, [str(exc)], remedy=exc.remedy)
+        _write_step_summary({
+            "verdict": "fail",
+            "model_id": args.model_id,
+            "failure_class": exc.failure_class,
+            "exit_code": exc.exit_code,
+            "failures": [str(exc)],
+            "remedy": exc.remedy,
+        })
+        return exc.exit_code
+
+    # Nothing may leave main() unclassified.  The pre-run modes (registry
+    # lookup, download, verify) have no run_smoke around them to catch what
+    # they did not anticipate — an unparseable registry.yaml, a missing file, a
+    # bad --models-dir — and run_smoke itself has a prologue and an epilogue
+    # outside its own handlers.  Letting any of that escape exits 1, which
+    # EXIT_CODES reads as `budget`: the one class that says "the product is
+    # fine, it is just slow".  Classify them the way run_smoke classifies its
+    # own surprises instead, so a banner and a step summary are still written
+    # and exit 1 keeps meaning only what it claims to mean.
+    except Exception as exc:  # noqa: BLE001 — anything unanticipated, reported not raised
+        failures = [
+            f"Unexpected error outside the smoke's own classification: {exc!r} — "
+            "this is most likely a bug in the smoke harness, a malformed "
+            "model-registry/registry.yaml or an unusable disk, not a product failure"
+        ]
+        remedy = (
+            "This escaped run_smoke's own error handling, so it says nothing about "
+            "the product. Re-run the harness locally with the same arguments "
+            "(docs/real-model-smoke.md) and fix the registry file, the environment, "
+            "or the harness bug in the traceback above."
+        )
+        _print_banner(FailureClass.PIPELINE, failures, remedy=remedy)
+        _write_step_summary({
+            "verdict": "fail",
+            "model_id": args.model_id,
+            "failure_class": FailureClass.PIPELINE,
+            "exit_code": EXIT_CODES[FailureClass.PIPELINE],
+            "failures": failures,
+            "remedy": remedy,
+        })
+        return EXIT_CODES[FailureClass.PIPELINE]
 
 
 if __name__ == "__main__":
diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py
new file mode 100644
index 00000000..98813136
--- /dev/null
+++ b/tests/scripts/__init__.py
@@ -0,0 +1 @@
+# SPDX-License-Identifier: Apache-2.0
diff --git a/tests/scripts/test_nightly_model_smoke.py b/tests/scripts/test_nightly_model_smoke.py
new file mode 100644
index 00000000..19bb7bab
--- /dev/null
+++ b/tests/scripts/test_nightly_model_smoke.py
@@ -0,0 +1,3554 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Unit tests for the nightly real-model smoke harness (issue #457).
+
+These run on every PR and need no model, no network and no llama-server: they
+cover the harness's decision logic — checksum verification, the end-to-end
+assertions, budget evaluation, failure classification and reporting — so that
+when the nightly goes red, the verdict it prints can be trusted.
+
+The real-model path itself is exercised only by
+.github/workflows/model-smoke-nightly.yml; see docs/real-model-smoke.md.
+"""
+from __future__ import annotations
+
+import hashlib
+import http.client
+import importlib.util
+import io
+import json
+import socket
+import statistics
+import urllib.error
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCRIPT_PATH = REPO_ROOT / "scripts" / "nightly-model-smoke.py"
+
+
+def _load_module():
+    """Import the hyphenated script as a module."""
+    spec = importlib.util.spec_from_file_location("nightly_model_smoke", SCRIPT_PATH)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+smoke = _load_module()
+
+
+# ---------------------------------------------------------------------------
+# Failure classification contract
+# ---------------------------------------------------------------------------
+
+
+class TestFailureClassification:
+    """Each class maps to its own exit code, so CI can tell failures apart."""
+
+    def test_every_class_has_a_unique_exit_code(self) -> None:
+        codes = list(smoke.EXIT_CODES.values())
+        assert len(codes) == len(set(codes)), "exit codes must be distinguishable"
+        assert 0 not in codes, "a failure must never exit 0"
+
+    def test_every_class_has_an_exit_code_and_a_remedy(self) -> None:
+        classes = {
+            value for name, value in vars(smoke.FailureClass).items()
+            if not name.startswith("_") and isinstance(value, str)
+        }
+        assert classes == set(smoke.EXIT_CODES)
+        assert classes == set(smoke.REMEDIES)
+
+    def test_download_and_checksum_are_separate_classes(self) -> None:
+        # The acceptance criterion for #457: a download failure and a checksum
+        # drift must not look alike in CI output.
+        assert smoke.EXIT_CODES[smoke.FailureClass.DOWNLOAD] != smoke.EXIT_CODES[
+            smoke.FailureClass.CHECKSUM
+        ]
+
+    def test_failure_carries_its_exit_code_and_remedy(self) -> None:
+        exc = smoke.SmokeFailure(smoke.FailureClass.RUNTIME, "boom", phase="runtime_start")
+        assert exc.exit_code == smoke.EXIT_CODES[smoke.FailureClass.RUNTIME]
+        assert exc.phase == "runtime_start"
+        assert exc.remedy == smoke.REMEDIES[smoke.FailureClass.RUNTIME]
+
+    def test_a_failure_can_override_its_class_remedy(self) -> None:
+        # Failures that borrow a class's exit code without matching its usual
+        # cause must not print that class's advice.
+        exc = smoke.SmokeFailure(
+            smoke.FailureClass.PIPELINE, "boom", remedy="Do this instead."
+        )
+        assert exc.exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        assert exc.remedy == "Do this instead."
+
+    def test_the_pipeline_remedy_carves_out_the_unscored_debrief(self) -> None:
+        # All pipeline assertions are raised together, so this one remedy is
+        # printed under every one of them and cannot be swapped per failure.
+        # Three of the four unscored-debrief causes are nothing to do with the
+        # per-turn flags it otherwise sends the reader to -- the docs failure
+        # table says so, and the advice beside the banner is what triage
+        # actually reads, so it has to say so too.
+        remedy = smoke.REMEDIES[smoke.FailureClass.PIPELINE]
+        assert "unscored debrief" in remedy.lower()
+        doc = (REPO_ROOT / "docs" / "real-model-smoke.md").read_text(encoding="utf-8")
+        assert "## Unscored debrief" in doc, "the remedy names a section that must exist"
+
+
+# ---------------------------------------------------------------------------
+# Checksum verification
+# ---------------------------------------------------------------------------
+
+
+class TestChecksumVerification:
+    """Checksum drift must fail loudly, and must not poison the next run."""
+
+    @staticmethod
+    def _write(tmp_path: Path, payload: bytes = b"gguf-ish bytes") -> tuple[Path, str]:
+        path = tmp_path / "model.gguf"
+        path.write_bytes(payload)
+        return path, hashlib.sha256(payload).hexdigest()
+
+    def test_matching_checksum_returns_the_digest(self, tmp_path: Path) -> None:
+        path, digest = self._write(tmp_path)
+        assert smoke.verify_model_checksum(path, digest) == digest
+        assert path.exists()
+
+    def test_the_same_digest_in_a_different_case_is_not_drift(self, tmp_path: Path) -> None:
+        # The registry pins lower-case hex, but --model-sha256 is typed by hand
+        # in the local repro. An upper-case paste is the same digest, and
+        # string equality would call it drift -- deleting a 2.5 GB file and
+        # reporting "the pinned upstream file was replaced", which is both
+        # expensive and wrong.
+        path, digest = self._write(tmp_path)
+        assert smoke.verify_model_checksum(path, digest.upper()) == digest
+        assert path.exists()
+
+    def test_a_digest_pasted_with_stray_whitespace_is_not_drift(self, tmp_path: Path) -> None:
+        path, digest = self._write(tmp_path)
+        assert smoke.verify_model_checksum(path, f"  {digest}\n") == digest
+        assert path.exists()
+
+    def test_drift_raises_checksum_class(self, tmp_path: Path) -> None:
+        path, _ = self._write(tmp_path)
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke.verify_model_checksum(path, "0" * 64)
+        assert exc_info.value.failure_class == smoke.FailureClass.CHECKSUM
+        assert exc_info.value.exit_code == 3
+
+    def test_drift_deletes_the_bad_file_so_a_rerun_redownloads(self, tmp_path: Path) -> None:
+        path, _ = self._write(tmp_path)
+        with pytest.raises(smoke.SmokeFailure):
+            smoke.verify_model_checksum(path, "0" * 64)
+        assert not path.exists()
+
+    def test_drift_can_keep_the_file_for_inspection(self, tmp_path: Path) -> None:
+        path, _ = self._write(tmp_path)
+        with pytest.raises(smoke.SmokeFailure):
+            smoke.verify_model_checksum(path, "0" * 64, delete_on_mismatch=False)
+        assert path.exists()
+
+    def test_a_malformed_digest_is_not_drift_and_keeps_the_file(self, tmp_path: Path) -> None:
+        # A digest short by one character is the ordinary outcome of copying 64
+        # hex characters out of a terminal by hand, and it can never match any
+        # file's hash -- so comparing against it is not a check. Treating it as
+        # drift deleted a 2.5 GB download and reported that the pinned upstream
+        # file had been replaced, which is the expensive, misleading verdict the
+        # case/whitespace normalisation above exists to avoid.
+        path, digest = self._write(tmp_path)
+        for malformed in (digest[:-1], digest + "0", "not-a-digest", "", digest[:-1] + "g"):
+            with pytest.raises(smoke.SmokeFailure) as exc_info:
+                smoke.verify_model_checksum(path, malformed)
+            assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE, malformed
+            assert path.exists(), f"{malformed!r} must not delete the model"
+            # The checksum remedy ("the pinned upstream file was replaced ...
+            # do NOT relax this check") is the wrong advice here.
+            assert exc_info.value.remedy != smoke.REMEDIES[smoke.FailureClass.CHECKSUM]
+
+    def test_a_malformed_digest_is_caught_before_the_file_is_hashed(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # Hashing 2.5 GB to compare it against something that cannot match is
+        # pure waste, and a missing file would otherwise be reported as
+        # `download` while the real problem is the argument.
+        def _never(*args: object, **kwargs: object) -> str:
+            raise AssertionError("must not hash the file for a malformed digest")
+
+        monkeypatch.setattr(smoke, "sha256_file", _never)
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke.verify_model_checksum(tmp_path / "absent.gguf", "deadbeef")
+        assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE
+
+    def test_a_malformed_digest_is_rejected_before_anything_is_fetched(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # Rejecting it only at verification time means the reader waits out a
+        # 2.5 GB download to be told the argument was wrong -- and the realistic
+        # source of a malformed digest is exactly the hand-pasted --model-sha256
+        # of the local repro, which runs --download-only. No bytes could have
+        # changed the verdict, and docs/real-model-smoke.md promises the download
+        # is not spent on one.
+        def _never(*args: object, **kwargs: object) -> None:
+            raise AssertionError("must not fetch 2.5 GB to compare it to a non-digest")
+
+        monkeypatch.setattr(smoke, "_download_with_progress", _never)
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke.download_model("https://example.invalid/m.gguf", "deadbeef", "m", tmp_path)
+        assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE
+        assert not (tmp_path / "m.gguf").exists()
+
+    def test_the_registry_pin_is_a_well_formed_digest(self) -> None:
+        # The one digest the nightly actually runs on has to pass the guard
+        # above, or the job fails before it fetches anything.
+        model = smoke.resolve_registry_model("starter")
+        assert len(model["sha256"]) == smoke.SHA256_HEX_CHARS
+        assert set(model["sha256"].lower()) <= smoke._HEX_DIGITS
+
+    def test_missing_file_is_a_download_failure_not_a_checksum_failure(self, tmp_path: Path) -> None:
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke.verify_model_checksum(tmp_path / "absent.gguf", "0" * 64)
+        assert exc_info.value.failure_class == smoke.FailureClass.DOWNLOAD
+
+    def test_download_skips_the_fetch_but_still_verifies_an_existing_file(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        payload = b"already here"
+        (tmp_path / "m.gguf").write_bytes(payload)
+
+        def _never(*args: object, **kwargs: object) -> None:
+            raise AssertionError("must not re-download an existing file")
+
+        monkeypatch.setattr(smoke, "_download_with_progress", _never)
+        # Correct digest: returns the path.
+        assert smoke.download_model(
+            "http://example.invalid/m.gguf", hashlib.sha256(payload).hexdigest(), "m", tmp_path
+        ) == tmp_path / "m.gguf"
+        # Wrong digest: a cache hit with drifted bytes still fails.
+        (tmp_path / "m.gguf").write_bytes(payload)
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke.download_model("http://example.invalid/m.gguf", "0" * 64, "m", tmp_path)
+        assert exc_info.value.failure_class == smoke.FailureClass.CHECKSUM
+
+
+class TestDownloadFailureClassification:
+    """Every way a fetch can break has to land in the `download` class."""
+
+    @staticmethod
+    def _urlopen_raising(exc: BaseException):
+        class _Resp:
+            headers = {"Content-Length": "1048576"}
+
+            def read(self, _n: int) -> bytes:
+                raise exc
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *a: object) -> bool:
+                return False
+
+        return lambda *args, **kwargs: _Resp()
+
+    @pytest.mark.parametrize("exc", [
+        # A cut-off response mid-body. NOT an OSError, so an OSError-only
+        # handler let it escape as a traceback — exit 1, which reads as `budget`.
+        http.client.IncompleteRead(b"partial"),
+        ConnectionResetError("peer hung up"),
+        TimeoutError("read timed out"),
+    ])
+    def test_a_broken_transfer_is_a_download_failure(
+        self, exc: BaseException, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setattr(
+            smoke.urllib.request, "urlopen", self._urlopen_raising(exc)
+        )
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._download_with_progress("https://example.invalid/m.gguf", tmp_path / "m.gguf")
+        assert exc_info.value.failure_class == smoke.FailureClass.DOWNLOAD
+        assert exc_info.value.exit_code == 2
+
+    def test_a_broken_transfer_leaves_no_partial_file_to_checksum_fail_on(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        dest = tmp_path / "m.gguf"
+        monkeypatch.setattr(
+            smoke.urllib.request, "urlopen",
+            self._urlopen_raising(http.client.IncompleteRead(b"partial")),
+        )
+        with pytest.raises(smoke.SmokeFailure):
+            smoke._download_with_progress("https://example.invalid/m.gguf", dest)
+        assert not dest.exists()
+
+    @pytest.mark.parametrize(
+        "url",
+        [
+            "<model_url from step 2>",  # the repro block's placeholder, pasted verbatim
+            "example.invalid/m.gguf",  # scheme omitted
+        ],
+    )
+    def test_an_unrequestable_url_is_not_reported_as_a_harness_bug(
+        self, url: str, tmp_path: Path
+    ) -> None:
+        # urllib raises ValueError out of Request's *constructor* for a string
+        # it cannot read as a URL, which used to sit above the handler — so it
+        # escaped to main's catch-all as "most likely a bug in the smoke
+        # harness", advising a re-run with the same arguments. The arguments are
+        # the problem, and the local repro in docs/real-model-smoke.md hands the
+        # reader a --model-url placeholder to substitute.
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._download_with_progress(url, tmp_path / "m.gguf")
+        assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE
+        assert "not a usable download url" in str(exc_info.value).lower()
+        # Its own remedy, not the class default: "inspect the per-turn
+        # used_fallback flags" is useless when no turn was ever played.
+        assert exc_info.value.remedy != smoke.REMEDIES[smoke.FailureClass.PIPELINE]
+        assert "--model-url" in exc_info.value.remedy
+        assert "registry" in exc_info.value.remedy
+
+    def test_a_crawling_transfer_is_bounded_and_classified(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # A transfer that keeps delivering bytes, just far too slowly, never
+        # trips urllib's per-read timeout. The download runs in its own step
+        # before the smoke, on a job clock the harness's Deadline cannot see, so
+        # without its own budget this is the one remaining way to reach the
+        # unattributable "The operation was canceled".
+        dest = tmp_path / "m.gguf"
+        clock = iter([0.0] + [1.0] * 8)  # started, then one read inside the budget
+
+        class _Resp:
+            headers = {"Content-Length": str(100 << 20)}
+
+            def read(self, _n: int) -> bytes:
+                return b"\0" * (1 << 20)  # never ends
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *a: object) -> bool:
+                return False
+
+        monkeypatch.setattr(smoke.urllib.request, "urlopen", lambda *a, **k: _Resp())
+        monkeypatch.setattr(
+            smoke.time, "monotonic", lambda: next(clock, 10.0)
+        )
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._download_with_progress(
+                "https://example.invalid/m.gguf", dest, budget_s=5.0
+            )
+        assert exc_info.value.failure_class == smoke.FailureClass.DOWNLOAD
+        assert exc_info.value.exit_code == 2
+        assert "budget" in str(exc_info.value)
+        # Its own remedy, naming why the bound exists at all — not the generic
+        # "download failed for <url>: SmokeFailure(...)" a re-wrap would give.
+        assert exc_info.value.remedy != smoke.REMEDIES[smoke.FailureClass.DOWNLOAD]
+        assert "job clock" in exc_info.value.remedy
+        # The partial file still goes, or the next run checksum-fails on it.
+        assert not dest.exists()
+
+    def test_one_quiet_stretch_cannot_outlast_the_whole_budget(self) -> None:
+        # urllib's timeout is per socket operation, so a read timeout at or
+        # above the transfer budget means a single silent connection decides how
+        # long the download runs, and the budget never gets to.
+        assert smoke.DOWNLOAD_READ_TIMEOUT_S < smoke.DOWNLOAD_BUDGET_S
+
+    def test_progress_is_logged_sparsely_not_once_per_megabyte(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
+    ) -> None:
+        # The "\r" updates collapse in a terminal but not in a CI log, where
+        # each flush is its own line: the one nightly that missed the model
+        # cache logged 2381 of them for a 2.3 GiB GGUF, directly above the
+        # verdict a triager is looking for.
+        megabytes = 200
+        chunks = [b"\0" * (1 << 20)] * megabytes
+
+        class _Resp:
+            headers = {"Content-Length": str(megabytes << 20)}
+
+            def read(self, _n: int) -> bytes:
+                return chunks.pop(0) if chunks else b""
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *a: object) -> bool:
+                return False
+
+        monkeypatch.setattr(smoke.urllib.request, "urlopen", lambda *a, **k: _Resp())
+        smoke._download_with_progress("https://example.invalid/m.gguf", tmp_path / "m.gguf")
+
+        progress_lines = [
+            line for line in capsys.readouterr().out.split("\r") if line.strip().endswith("MB)")
+        ]
+        # One per PROGRESS_LOG_STEP_PCT, plus the 0 % line: nowhere near one per MB.
+        assert len(progress_lines) <= 100 // smoke.PROGRESS_LOG_STEP_PCT + 1
+        # Still enough to show where a stalled transfer stopped.
+        assert len(progress_lines) >= 2
+        assert "100%" in progress_lines[-1]
+
+
+# ---------------------------------------------------------------------------
+# Registry resolution
+# ---------------------------------------------------------------------------
+
+
+class TestRegistryResolution:
+    """The workflow's cache key, URL and checksum come from one lookup."""
+
+    def test_resolves_the_real_starter_model(self) -> None:
+        model = smoke.resolve_registry_model("starter")
+        assert model["id"]
+        assert model["url"].startswith("https://")
+        assert len(model["sha256"]) == 64
+        assert model["sha256"] == model["sha256"].lower()
+
+    def test_starter_is_the_smallest_registry_model(self) -> None:
+        # #457 asks for the smallest registry model; guard against a future
+        # registry edit that makes "starter" no longer the smallest.
+        import yaml
+
+        registry = yaml.safe_load(
+            (REPO_ROOT / "model-registry" / "registry.yaml").read_text(encoding="utf-8")
+        )
+        # The user-supplied placeholder has no size and nothing to download.
+        sizes = {
+            m["id"]: m["size_gb"] for m in registry["models"]
+            if m.get("size_gb") is not None
+        }
+        starter = smoke.resolve_registry_model("starter")
+        assert sizes[starter["id"]] == min(sizes.values())
+
+    def test_unknown_role_is_not_a_download_failure(self, tmp_path: Path) -> None:
+        # `download`'s remedy ends "Nothing about the app changed — re-run the
+        # job", which would send a triager round a loop: no number of re-runs
+        # makes a missing `role: starter` entry appear. A registry that does not
+        # describe a usable model is a repository problem, so it takes the
+        # harness's catch-all class and its own remedy.
+        registry = tmp_path / "registry.yaml"
+        registry.write_text("models: []\n", encoding="utf-8")
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke.resolve_registry_model("starter", registry)
+        assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE
+        assert exc_info.value.exit_code == 5
+        assert "registry.yaml" in exc_info.value.remedy
+        assert "re-run the job" not in exc_info.value.remedy.lower()
+
+    def test_two_models_claiming_the_same_role_are_rejected(self, tmp_path: Path) -> None:
+        # The cache key, the download URL and the verified checksum all come
+        # from this one lookup, so an ambiguous role must stop the run rather
+        # than let it pick arbitrarily.
+        registry = tmp_path / "registry.yaml"
+        registry.write_text(
+            "models:\n"
+            "  - id: a\n"
+            "    role: starter\n"
+            "    download: {url: https://example.invalid/a.gguf, sha256: aa}\n"
+            "  - id: b\n"
+            "    role: starter\n"
+            "    download: {url: https://example.invalid/b.gguf, sha256: bb}\n",
+            encoding="utf-8",
+        )
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke.resolve_registry_model("starter", registry)
+        assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE
+        assert "expected exactly one" in str(exc_info.value)
+
+    def test_model_without_checksum_is_rejected(self, tmp_path: Path) -> None:
+        registry = tmp_path / "registry.yaml"
+        registry.write_text(
+            "models:\n"
+            "  - id: x\n"
+            "    role: starter\n"
+            "    download:\n"
+            "      url: https://example.invalid/x.gguf\n",
+            encoding="utf-8",
+        )
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke.resolve_registry_model("starter", registry)
+        assert "sha256" in str(exc_info.value)
+        assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE
+
+    @pytest.mark.parametrize(
+        ("url", "sha256", "expected_field"),
+        [
+            ("PENDING", "ab" * 32, "download.url"),
+            ("https://example.invalid/x.gguf", "PENDING", "download.sha256"),
+            ("PENDING", "PENDING", "download.url"),
+        ],
+    )
+    def test_an_unpinned_entry_is_a_registry_problem_not_a_download_one(
+        self, tmp_path: Path, url: str, sha256: str, expected_field: str
+    ) -> None:
+        # "PENDING" is the placeholder model-registry.schema.json allows for an
+        # entry that has been added but not pinned, and the no-PENDING check in
+        # scripts/validate-registry.py only runs in registry-nightly.yml — not
+        # per-PR — so an un-pinned `role: starter` can reach main.  Passed along
+        # as a value it produces two confidently wrong verdicts: a "PENDING"
+        # sha256 downloads 2.5 GB and then fails `checksum` with "the pinned
+        # upstream file was replaced … do NOT relax this check", and a "PENDING"
+        # url raises out of urllib's Request constructor into main's catch-all
+        # as a suspected harness bug.  It is neither: it is the registry failing
+        # to describe a usable model, which must stop the run at the lookup.
+        registry = tmp_path / "registry.yaml"
+        registry.write_text(
+            "models:\n"
+            "  - id: next-starter\n"
+            "    role: starter\n"
+            f"    download: {{url: {url!r}, sha256: {sha256!r}}}\n",
+            encoding="utf-8",
+        )
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke.resolve_registry_model("starter", registry)
+        assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE
+        assert exc_info.value.exit_code == 5
+        assert expected_field in str(exc_info.value)
+        assert "not pinned" in str(exc_info.value)
+        assert exc_info.value.remedy == smoke.REGISTRY_REMEDY
+
+    def test_the_real_starter_entry_is_pinned(self) -> None:
+        # The flip side: the guard above must not be able to red the nightly on
+        # the registry as it actually stands.
+        model = smoke.resolve_registry_model("starter")
+        assert model["url"] != smoke.REGISTRY_PENDING
+        assert model["sha256"] != smoke.REGISTRY_PENDING
+
+    def test_github_output_is_appended(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+        out = tmp_path / "gh-output"
+        monkeypatch.setenv("GITHUB_OUTPUT", str(out))
+        smoke._emit_github_output({"model_id": "abc", "model_sha256": "def"})
+        assert out.read_text(encoding="utf-8").splitlines() == [
+            "model_id=abc",
+            "model_sha256=def",
+        ]
+
+
+# ---------------------------------------------------------------------------
+# Conversation assertions
+# ---------------------------------------------------------------------------
+
+
+def _turn(**overrides: object) -> dict:
+    turn = {
+        "label": "player_turn_1",
+        "turn_number": 1,
+        "model_generated": True,
+        "npc_excerpt": "Tell me more about that platform.",
+        "used_fallback": False,
+    }
+    turn.update(overrides)
+    return turn
+
+
+class TestEvaluateTurns:
+    """Real NPC turns are the whole point of the nightly."""
+
+    def test_healthy_conversation_passes(self) -> None:
+        failures, warnings = smoke.evaluate_turns([_turn(), _turn(turn_number=2)])
+        assert failures == []
+        assert warnings == []
+
+    def test_no_turns_fails(self) -> None:
+        failures, _ = smoke.evaluate_turns([])
+        assert failures
+
+    def test_empty_npc_utterance_fails(self) -> None:
+        failures, _ = smoke.evaluate_turns([_turn(npc_excerpt="")])
+        assert any("no NPC utterance" in f for f in failures)
+
+    def test_all_generated_turns_falling_back_fails(self) -> None:
+        turns = [_turn(used_fallback=True), _turn(turn_number=2, used_fallback=True)]
+        failures, _ = smoke.evaluate_turns(turns)
+        assert any("fell back" in f for f in failures)
+
+    @pytest.mark.parametrize(
+        "turns, expected",
+        [
+            ([], False),
+            ([{"label": "npc_opening", "turn_number": 0, "model_generated": False,
+               "npc_excerpt": "Thanks for coming in."}], False),
+            ([_turn(used_fallback=True), _turn(turn_number=2, used_fallback=True)], True),
+            ([_turn(used_fallback=True), _turn(turn_number=2)], False),
+            # Flags unavailable: an absent used_fallback is not a fallback.
+            ([_turn(), _turn(turn_number=2)], False),
+        ],
+    )
+    def test_the_two_verdicts_agree_on_what_all_fell_back_means(
+        self, turns: list, expected: bool
+    ) -> None:
+        # evaluate_turns fails the run for it and evaluate_debrief uses it to
+        # say the unscored debrief is downstream of it. If the two disagreed,
+        # one verdict would carry both notes, each naming the other as the
+        # wrong place to start.
+        assert smoke._all_generated_turns_fell_back(turns) is expected
+        failures, _ = smoke.evaluate_turns(list(turns))
+        assert any("fell back to the canned" in f for f in failures) is expected
+
+    def test_one_fallback_among_several_only_warns(self) -> None:
+        turns = [_turn(used_fallback=True), _turn(turn_number=2), _turn(turn_number=3)]
+        failures, warnings = smoke.evaluate_turns(turns)
+        assert failures == []
+        assert any("fell back" in w for w in warnings)
+
+    def test_authored_opening_does_not_count_as_a_generated_turn(self) -> None:
+        # The NPC opening is scenario text, not inference. A run whose only
+        # non-fallback turn is the opening has produced no real NPC turns.
+        turns = [
+            {"label": "npc_opening", "turn_number": 0, "model_generated": False,
+             "npc_excerpt": "Thanks for coming in."},
+            _turn(used_fallback=True),
+        ]
+        failures, _ = smoke.evaluate_turns(turns)
+        assert any("fell back" in f for f in failures)
+
+    def test_missing_parse_flags_warn_that_the_fallback_check_did_not_run(self) -> None:
+        # The flags come from the best-effort debug endpoint. Without them a
+        # canned fallback looks exactly like real model output (a non-empty
+        # utterance), so the run must not imply a proof it does not have.
+        turns = [{"label": "player_turn_1", "turn_number": 1,
+                  "model_generated": True, "npc_excerpt": "Go on."}]
+        failures, warnings = smoke.evaluate_turns(turns)
+        assert failures == []
+        assert any("fallback check did not run" in w for w in warnings)
+
+    def test_parse_flags_covering_only_some_turns_warn_about_the_rest(self) -> None:
+        # _debug_flags_by_turn drops any /debug entry it cannot key by turn
+        # number, so the payload can cover some turns and not others. An absent
+        # used_fallback is falsy, which means the uncovered turns are credited
+        # as real replies with no evidence at all — here they are the only
+        # reason the run is not failed for a conversation of fallbacks. The
+        # report has to say which turns the check skipped.
+        turns = [
+            {"label": "player_turn_1", "turn_number": 1,
+             "model_generated": True, "npc_excerpt": "Go on."},
+            _turn(label="player_turn_2", turn_number=2, used_fallback=True),
+            _turn(label="player_turn_3", turn_number=3, used_fallback=True),
+        ]
+        failures, warnings = smoke.evaluate_turns(turns)
+        assert failures == []
+        warning = next(w for w in warnings if "fallback check did not run" in w)
+        assert "1 of 3" in warning
+        assert "player_turn_1" in warning
+
+    def test_an_unlabelled_turn_without_flags_is_named_by_its_number(self) -> None:
+        turns = [
+            {"turn_number": 1, "model_generated": True, "npc_excerpt": "Go on."},
+            _turn(turn_number=2),
+        ]
+        _, warnings = smoke.evaluate_turns(turns)
+        assert any("turn 1" in w for w in warnings)
+
+    def test_present_parse_flags_do_not_warn(self) -> None:
+        failures, warnings = smoke.evaluate_turns([_turn()])
+        assert failures == []
+        assert warnings == []
+
+    def test_fully_covered_parse_flags_do_not_warn_about_partial_coverage(self) -> None:
+        failures, warnings = smoke.evaluate_turns(
+            [_turn(), _turn(label="player_turn_2", turn_number=2)]
+        )
+        assert failures == []
+        assert not any("missing for" in w for w in warnings)
+
+    def test_opening_only_run_fails(self) -> None:
+        turns = [{"label": "npc_opening", "turn_number": 0, "model_generated": False,
+                  "npc_excerpt": "Thanks for coming in."}]
+        failures, _ = smoke.evaluate_turns(turns)
+        assert any("No model-generated" in f for f in failures)
+
+    def test_every_turn_reciting_the_opening_fails(self) -> None:
+        # Real model output, every parse flag healthy, and not one reply: the
+        # exact run the fallback check cannot see.
+        turns = [
+            _turn(replayed_opening=True),
+            _turn(turn_number=2, replayed_opening=True),
+        ]
+        failures, _ = smoke.evaluate_turns(turns)
+        assert any("reciting the authored opening" in f for f in failures)
+
+    def test_one_turn_reciting_the_opening_only_warns(self) -> None:
+        # Observed behaviour on CI: the starter model recites the opening back
+        # on the first player turn. One such turn degrades the run without
+        # making it worthless, so it warns — the same policy as one fallback.
+        turns = [
+            _turn(replayed_opening=True),
+            _turn(turn_number=2),
+            _turn(turn_number=3),
+        ]
+        failures, warnings = smoke.evaluate_turns(turns)
+        assert failures == []
+        assert any("reciting the authored opening" in w for w in warnings)
+
+    def test_a_conversation_split_between_the_two_anomalies_still_fails(self) -> None:
+        # Neither "all fell back" nor "all recited" fires, and yet not one turn
+        # was a reply. Each check warns on its own because recovering from a bad
+        # turn is the product working; together they leave nothing recovered.
+        turns = [
+            _turn(replayed_opening=True),
+            _turn(turn_number=2, used_fallback=True),
+            _turn(turn_number=3, used_fallback=True),
+        ]
+        failures, _ = smoke.evaluate_turns(turns)
+        assert any("Not one of the 3" in f and "was a reply" in f for f in failures)
+        # The two single-cause failures stay off: each is false here, and
+        # printing them would misdescribe the run.
+        assert not any("All 3" in f for f in failures)
+
+    def test_a_single_good_turn_keeps_a_mixed_conversation_green(self) -> None:
+        # One real reply among a fallback and a recital is the "the product
+        # recovered" case both warnings exist for, not a failure.
+        turns = [
+            _turn(replayed_opening=True),
+            _turn(turn_number=2, used_fallback=True),
+            _turn(turn_number=3),
+        ]
+        failures, warnings = smoke.evaluate_turns(turns)
+        assert failures == []
+        assert len(warnings) == 2
+
+    def test_the_composed_check_does_not_double_report_all_fallbacks(self) -> None:
+        turns = [_turn(used_fallback=True), _turn(turn_number=2, used_fallback=True)]
+        failures, _ = smoke.evaluate_turns(turns)
+        assert len(failures) == 1
+        assert "fell back to the canned" in failures[0]
+
+    def test_the_composed_check_does_not_double_report_all_recitals(self) -> None:
+        turns = [_turn(replayed_opening=True), _turn(turn_number=2, replayed_opening=True)]
+        failures, _ = smoke.evaluate_turns(turns)
+        assert len(failures) == 1
+        assert "reciting the authored opening" in failures[0]
+
+    def test_absent_parse_flags_cannot_produce_the_composed_failure(self) -> None:
+        # Without the debug endpoint no turn is known to have fallen back, so
+        # the only turns left are judged on replayed_opening alone — the
+        # composed check must not convict a run it has no evidence about.
+        turns = [
+            {"label": "player_turn_1", "turn_number": 1, "model_generated": True,
+             "npc_excerpt": "Go on.", "replayed_opening": True},
+            {"label": "player_turn_2", "turn_number": 2, "model_generated": True,
+             "npc_excerpt": "Go on.", "replayed_opening": False},
+        ]
+        failures, _ = smoke.evaluate_turns(turns)
+        assert not any("was a reply" in f for f in failures)
+
+    def test_a_recited_opening_is_not_mistaken_for_a_fallback(self) -> None:
+        # The two have different causes and different fixes, so the verdict
+        # must not describe one as the other.
+        failures, warnings = smoke.evaluate_turns([_turn(replayed_opening=True)])
+        assert not any("fell back" in m for m in failures + warnings)
+
+    def test_the_authored_opening_is_not_judged_for_reciting_itself(self) -> None:
+        turns = [
+            {"label": "npc_opening", "turn_number": 0, "model_generated": False,
+             "npc_excerpt": "Thanks for coming in.", "replayed_opening": True},
+            _turn(),
+        ]
+        failures, warnings = smoke.evaluate_turns(turns)
+        assert failures == []
+        assert warnings == []
+
+
+class TestReplaysOpening:
+    """Reciting the authored opening is real model output that is not a reply.
+
+    Every nightly run of the previous single-turn harness logged an NPC reply
+    whose leading characters were byte-identical to the scenario's
+    ``opening_npc_says``. ``used_fallback`` stays false for those turns — the
+    utterance is neither empty nor the canned safe one — so without this check
+    "the model produced real NPC turns" rests on the reply being non-empty.
+    """
+
+    OPENING = (
+        "Thanks for coming in today. I'm Alex Chen from HR. Tell me a little "
+        "about yourself and why you're interested in this role."
+    )
+
+    def test_a_verbatim_copy_is_caught(self) -> None:
+        assert smoke._replays_opening(self.OPENING, self.OPENING)
+
+    def test_a_copy_that_carries_on_into_fresh_prose_is_caught(self) -> None:
+        # What the nightly logs actually showed: the recital, then more text.
+        # Equality would have missed it.
+        assert smoke._replays_opening(
+            self.OPENING + " So, tell me about a time you shipped something.",
+            self.OPENING,
+        )
+
+    def test_a_recital_that_veers_off_partway_is_caught(self) -> None:
+        # The evidence this check was built from is the old harness's log line,
+        # which printed npc_text[:80] against a 122-character opening: the
+        # nightlies establish the first 80 characters matched and say nothing
+        # about the rest.  Demanding the entire opening would have made the
+        # guard silently unreachable for a recital that breaks off early, which
+        # is just as much "not a reply".
+        assert smoke._replays_opening(
+            "Thanks for coming in today. I'm Alex Chen from HR. Tell me a little "
+            "about your background in software development.",
+            self.OPENING,
+        )
+
+    def test_reformatted_whitespace_does_not_hide_a_copy(self) -> None:
+        assert smoke._replays_opening(
+            "Thanks for coming in today.\n  I'm Alex Chen from HR.\tTell me a "
+            "little\nabout yourself and why you're interested in this role.",
+            self.OPENING,
+        )
+
+    def test_a_real_reply_passes(self) -> None:
+        assert not smoke._replays_opening(
+            "Five years is a solid run. Walk me through the hardest trade-off "
+            "you made on that API platform.",
+            self.OPENING,
+        )
+
+    def test_a_reply_merely_echoing_a_few_words_passes(self) -> None:
+        # Only a leading recital long enough to be the scenario's own text
+        # counts (REPLAY_PREFIX_CHARS); sharing an opening phrase does not.
+        assert not smoke._replays_opening(
+            "Thanks for coming in today was my line, not yours — but go on.",
+            self.OPENING,
+        )
+
+    def test_a_short_opening_still_has_to_be_recited_in_full(self) -> None:
+        # The prefix threshold is capped at the opening's own length, so it
+        # only ever loosens the check for openings longer than it — a scenario
+        # with a terse opening keeps the strict whole-opening semantics and
+        # gains no new way to false-positive.
+        short = "Hello. Begin when ready."
+        assert len(short) < smoke.REPLAY_PREFIX_CHARS
+        assert smoke._replays_opening(short + " So, your background?", short)
+        assert not smoke._replays_opening("Hello. Five years in software.", short)
+
+    def test_an_empty_reply_is_not_convicted_of_reciting(self) -> None:
+        # An empty utterance is already a failure in its own right
+        # (evaluate_turns); it must not also be reported as a recital.
+        assert not smoke._replays_opening("", self.OPENING)
+
+    def test_an_unreadable_opening_cannot_convict_a_turn(self) -> None:
+        # _npc_turn_content returns '' when the opening event is missing or
+        # malformed. Every reply startswith('') is True, so an absent opening
+        # would otherwise fail the whole conversation.
+        assert not smoke._replays_opening("A perfectly good reply.", "")
+        assert not smoke._replays_opening("A perfectly good reply.", "   \n ")
+
+
+# ---------------------------------------------------------------------------
+# Debrief assertions
+# ---------------------------------------------------------------------------
+
+
+def _debrief(**overrides: object) -> dict:
+    doc = {
+        "scores": {"structure": 56.0, "evidence": 48.0},
+        "overall_score": 52.0,
+        "summary": "You gave concrete examples but hedged on the trade-off question.",
+        "turning_points": [{"turn_number": 2, "description": "d", "impact": "i"}],
+        "used_fallback": False,
+    }
+    doc.update(overrides)
+    return doc
+
+
+class TestEvaluateDebrief:
+    """"Assert a scored debrief is produced" — the #457 acceptance criterion."""
+
+    def test_scored_debrief_passes(self) -> None:
+        failures, warnings = smoke.evaluate_debrief(_debrief())
+        assert failures == []
+        assert warnings == []
+
+    def test_missing_debrief_fails(self) -> None:
+        assert smoke.evaluate_debrief(None)[0]
+
+    def test_unscored_debrief_fails(self) -> None:
+        failures, _ = smoke.evaluate_debrief(_debrief(scores={}, overall_score=None))
+        assert any("no rubric dimension scores" in f for f in failures)
+        assert any("not numeric" in f for f in failures)
+
+    def test_unscored_debrief_carries_its_own_note_not_the_class_remedy(self) -> None:
+        # The scores come only from rubric_observations the model volunteers,
+        # and nothing in the prompt asks for them (the built-in scenario defines
+        # no rubric and no prompt layer names one), so an empty array satisfies
+        # the turn schema. `pipeline`'s stock advice — inspect the per-turn
+        # used_fallback flags — would send triage hunting a parse failure that
+        # never happened, so the failure has to carry its own note.
+        failures, _ = smoke.evaluate_debrief(_debrief(scores={}))
+        no_scores = next(f for f in failures if "no rubric dimension scores" in f)
+        assert smoke.UNSCORED_DEBRIEF_NOTE in no_scores
+        assert "real-model-smoke.md" in no_scores
+
+    def test_no_observations_anywhere_is_named_and_still_worth_chasing(self) -> None:
+        # The run checked, and the model volunteered nothing to score: the
+        # failure can state that outright instead of listing both possibilities.
+        # It must not go on to excuse itself, though — the thin prompt coverage
+        # makes this reachable, but the real starter model answers the schema's
+        # bare hint on every turn, so a run that hits it changed something. A
+        # note that read "product gap, not a regression" would close the
+        # investigation the nightly exists to open.
+        failures, _ = smoke.evaluate_debrief(
+            _debrief(scores={}), rubric_observations_seen=0
+        )
+        no_scores = next(f for f in failures if "no rubric dimension scores" in f)
+        assert "no NPC turn carried a rubric_observation" in no_scores
+        assert smoke.UNSCORED_DEBRIEF_NOTE in no_scores
+        assert smoke.UNSCORED_WITH_OBSERVATIONS_NOTE not in no_scores
+        assert "not the normal outcome" in no_scores
+        assert "before writing this off" in no_scores
+
+    def test_a_zero_caused_by_fallbacks_is_not_blamed_on_rubric_prompting(self) -> None:
+        # A turn that fell back carries the canned safe utterance and an empty
+        # rubric_observations list whatever the prompt asked for, so this zero
+        # says nothing about rubric coverage. UNSCORED_DEBRIEF_NOTE would send
+        # triage after "the OUTPUT_SCHEMA layer, the starter model pin,
+        # sampling" while the fallback failure in the same verdict points
+        # somewhere else entirely.
+        failures, _ = smoke.evaluate_debrief(
+            _debrief(scores={}), rubric_observations_seen=0, turns_all_fell_back=True
+        )
+        no_scores = next(f for f in failures if "no rubric dimension scores" in f)
+        assert smoke.UNSCORED_AFTER_FALLBACK_NOTE in no_scores
+        assert smoke.UNSCORED_DEBRIEF_NOTE not in no_scores
+        assert smoke.UNSCORED_WITH_OBSERVATIONS_NOTE not in no_scores
+        assert "consequence of the fallback failure above" in no_scores
+
+    def test_observations_despite_fallbacks_still_blame_the_debrief_engine(self) -> None:
+        # Some turns fell back but the rest carried observations: the debrief
+        # still had something to accumulate, so the fallbacks are not the
+        # explanation and the regression note is the right one.
+        failures, _ = smoke.evaluate_debrief(
+            _debrief(scores={}), rubric_observations_seen=2, turns_all_fell_back=True
+        )
+        no_scores = next(f for f in failures if "no rubric dimension scores" in f)
+        assert smoke.UNSCORED_WITH_OBSERVATIONS_NOTE in no_scores
+        assert smoke.UNSCORED_AFTER_FALLBACK_NOTE not in no_scores
+
+    def test_an_unknown_observation_count_claims_neither_cause(self) -> None:
+        # The count is what tells the two causes apart, so without it the
+        # failure has to quote both. It must not state one as fact: a message
+        # that said "could not tell" and then "No NPC turn volunteered a single
+        # rubric observation" sends triage after the product weakness when the
+        # debrief engine dropping validated observations is just as open.
+        failures, _ = smoke.evaluate_debrief(_debrief(scores={}))
+        no_scores = next(f for f in failures if "no rubric dimension scores" in f)
+        assert "cannot be told apart" in no_scores
+        assert smoke.UNSCORED_DEBRIEF_NOTE in no_scores
+        assert smoke.UNSCORED_WITH_OBSERVATIONS_NOTE in no_scores
+        assert "No NPC turn volunteered" not in no_scores
+
+    def test_observations_that_never_reached_the_debrief_are_a_regression(self) -> None:
+        # The opposite case, and the one the harness exists to catch: the turns
+        # returned observations and the debrief scored nothing, so the model did
+        # its part and the debrief engine dropped the result. Printing the
+        # "nothing asked the model for them" note here would excuse a real bug.
+        failures, _ = smoke.evaluate_debrief(
+            _debrief(scores={}), rubric_observations_seen=4
+        )
+        no_scores = next(f for f in failures if "no rubric dimension scores" in f)
+        assert "returned 4 rubric observation(s)" in no_scores
+        assert smoke.UNSCORED_WITH_OBSERVATIONS_NOTE in no_scores
+        assert smoke.UNSCORED_DEBRIEF_NOTE not in no_scores
+        assert "_parse_rubric_observations" in no_scores
+
+    def test_a_scored_debrief_passes_whatever_the_turns_reported(self) -> None:
+        # The count only explains an *unscored* debrief; it must never fail a
+        # scored one (the debrief re-derives scores from stored raw output, so
+        # the two counts are not required to agree).
+        assert smoke.evaluate_debrief(_debrief(), rubric_observations_seen=0) == ([], [])
+
+    def test_non_numeric_overall_score_fails(self) -> None:
+        failures, _ = smoke.evaluate_debrief(_debrief(overall_score="52"))
+        assert any("not numeric" in f for f in failures)
+
+    def test_boolean_overall_score_is_not_a_number(self) -> None:
+        failures, _ = smoke.evaluate_debrief(_debrief(overall_score=True))
+        assert any("not numeric" in f for f in failures)
+
+    @pytest.mark.parametrize("score", [-1, 101])
+    def test_out_of_range_overall_score_fails(self, score: float) -> None:
+        failures, _ = smoke.evaluate_debrief(_debrief(overall_score=score))
+        assert any("outside [0, 100]" in f for f in failures)
+
+    def test_empty_summary_fails(self) -> None:
+        failures, _ = smoke.evaluate_debrief(_debrief(summary="   "))
+        assert any("summary is shorter" in f for f in failures)
+
+    def test_an_absent_summary_is_still_measured_as_too_short(self) -> None:
+        failures, _ = smoke.evaluate_debrief(_debrief(summary=None))
+        assert any("summary is shorter" in f for f in failures)
+
+    @pytest.mark.parametrize(
+        "summary",
+        [
+            # Long enough that a repr would clear MIN_SUMMARY_CHARS, so the type
+            # check cannot be mistaken for the length check passing.
+            {"text": "You gave concrete examples but hedged on the trade-off."},
+            ["You gave concrete examples but hedged on the trade-off."],
+            42,
+        ],
+    )
+    def test_a_summary_that_is_not_text_is_a_product_failure_not_a_harness_bug(
+        self, summary: object
+    ) -> None:
+        # len(summary.strip()) raises AttributeError on all of these, and an
+        # AttributeError out of the assertion layer reaches run_smoke's catch-all,
+        # which reports "most likely a bug in the smoke harness" -- the one
+        # verdict that is certainly wrong for a changed response contract.
+        failures, _ = smoke.evaluate_debrief(_debrief(summary=summary))
+        assert any("summary is not a string" in f for f in failures)
+        # Named as a type, and the value shown, so triage does not have to go and
+        # fetch the artifact to find out what arrived.
+        assert any(type(summary).__name__ in f for f in failures)
+        # And never reported as a length problem: nothing here is about brevity.
+        assert not any("summary is shorter" in f for f in failures)
+
+    def test_fallback_narrative_only_warns_because_scores_are_still_real(self) -> None:
+        failures, warnings = smoke.evaluate_debrief(_debrief(used_fallback=True))
+        assert failures == []
+        fallback = next(w for w in warnings if "fallback" in w)
+        # The debrief is generated once, so a fallback means no model-written
+        # debrief prose was produced at all and this is the only job that would
+        # notice. The run still passes -- the scores do not come from this
+        # generation -- so the warning has to say what the green verdict did not
+        # prove, or the limitation is invisible.
+        assert "proves nothing about real-model debrief generation" in fallback
+        assert "Recurring across nightlies" in fallback
+
+    def test_absent_turning_points_only_warn(self) -> None:
+        failures, warnings = smoke.evaluate_debrief(_debrief(turning_points=[]))
+        assert failures == []
+        assert any("turning points" in w for w in warnings)
+
+
+class TestUnscoredDebriefRemedyIsStillTrue:
+    """The unscored-debrief note explains *why* nothing asks for observations.
+
+    That explanation is the whole value of the note -- it is what keeps triage
+    off the two wrong tracks (a parse failure, or a different scenario) -- and it
+    is a claim about product source this harness does not import. If the product
+    grows a rubric route into the turn prompt, the note starts handing out advice
+    for a weakness that no longer exists, and nothing else in this file notices.
+    """
+
+    _PROMPT_SRC = REPO_ROOT / "packages" / "prompt-composer" / "src" / "convsim_prompt"
+
+    def test_no_prompt_layer_names_a_rubric_dimension(self) -> None:
+        layers = (self._PROMPT_SRC / "layers.py").read_text(encoding="utf-8")
+        assert "rubric" not in layers.lower(), (
+            "a prompt layer now mentions a rubric, so UNSCORED_DEBRIEF_NOTE's "
+            "'no prompt layer names the rubric dimensions' may no longer hold"
+        )
+
+    def test_a_scenario_cannot_carry_a_rubric_into_the_turn_prompt(self) -> None:
+        # Why the note must NOT say "or play a scenario that defines one": the
+        # official job-interview-basic pack's behavioral_interview does define a
+        # rubric, and composing a turn prompt from it would change nothing,
+        # because the dataclass the composer is handed has nowhere to put it.
+        types_src = (self._PROMPT_SRC / "types.py").read_text(encoding="utf-8")
+        start = types_src.index("class ScenarioData:")
+        body = types_src[start:types_src.index("\n@", start)]
+        assert "rubric" not in body.lower(), (
+            "ScenarioData now carries a rubric, so a rubric-defining scenario may "
+            "reach the turn prompt after all -- revisit UNSCORED_DEBRIEF_NOTE and "
+            "the 'Unscored debrief' section of docs/real-model-smoke.md"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Budget evaluation
+# ---------------------------------------------------------------------------
+
+
+class TestEvaluateBudgets:
+    def test_within_budget_passes(self) -> None:
+        failures, lines = smoke.evaluate_budgets({"full_response_ms": 100_000}, 20.0)
+        assert failures == []
+        assert any("PASS" in line for line in lines)
+
+    def test_tolerance_is_applied_before_failing(self) -> None:
+        # 10 000 ms × 2 × 1.20 = 24 000 ms ceiling.
+        assert smoke.evaluate_budgets({"full_response_ms": 23_999}, 2.0)[0] == []
+        assert smoke.evaluate_budgets({"full_response_ms": 24_001}, 2.0)[0]
+
+    def test_unbudgeted_metrics_are_not_checked(self) -> None:
+        # session_start_ms and debrief_ms are reported but have no budget: the
+        # opening is authored text and debrief latency has no documented SLO.
+        assert "session_start_ms" not in smoke.BUDGETS_MS
+        assert "debrief_ms" not in smoke.BUDGETS_MS
+        failures, lines = smoke.evaluate_budgets(
+            {"session_start_ms": 10 ** 9, "debrief_ms": 10 ** 9}, 1.0
+        )
+        assert failures == []
+        assert lines == []
+
+    def test_missing_measurement_is_skipped_not_failed(self) -> None:
+        assert smoke.evaluate_budgets({}, 20.0) == ([], [])
+
+    def test_failure_message_shows_the_arithmetic(self) -> None:
+        failures, _ = smoke.evaluate_budgets({"full_response_ms": 10 ** 6}, 20.0)
+        assert "documented budget 10000 ms × 20.0 × 1.2" in failures[0]
+
+
+class TestTimeoutLatencyEvidence:
+    """A `timeout` verdict must say when the real story is a latency regression.
+
+    Three turns at the 240 s CI ceiling plus a debrief allowed twice that very
+    nearly fill the 20 min wall-clock budget, so a uniform ~2x slowdown — the
+    regression `budget` exists to catch — exhausts the clock before the budget
+    phase ever runs.  TIMEOUT outranks it, so the measurement has to be attached
+    to the verdict or the remedy ("slower, or hung?") leaves triage guessing.
+    """
+
+    @staticmethod
+    def _turns(*latencies_ms: float) -> list[dict]:
+        return [
+            {"label": "npc_opening", "turn_number": 0, "model_generated": False,
+             "latency_ms": 10},
+            *[
+                {"label": f"player_turn_{i}", "turn_number": i,
+                 "model_generated": True, "latency_ms": ms}
+                for i, ms in enumerate(latencies_ms, start=1)
+            ],
+        ]
+
+    def test_turns_past_the_ceiling_are_named_as_a_likely_regression(self) -> None:
+        evidence = smoke.timeout_latency_evidence(
+            self._turns(250_000, 260_000, 270_000), 240_000
+        )
+        assert evidence is not None
+        assert "260000 ms" in evidence  # the median of the completed turns
+        assert "240000 ms CI ceiling" in evidence
+        assert "`budget`" in evidence
+
+    def test_turns_inside_the_ceiling_stay_a_plain_timeout(self) -> None:
+        # Something hung, or a phase other than the conversation ran long:
+        # claiming a latency regression here would be a guess.
+        assert smoke.timeout_latency_evidence(
+            self._turns(10_000, 11_000, 12_000), 240_000
+        ) is None
+
+    def test_a_run_that_timed_out_before_any_turn_says_nothing(self) -> None:
+        assert smoke.timeout_latency_evidence(self._turns(), 240_000) is None
+
+    def test_the_authored_opening_is_not_evidence_of_slow_inference(self) -> None:
+        # The opening is replayed scenario text, so its latency says nothing
+        # about the model -- counting it would invent evidence.
+        slow_opening = [{"label": "npc_opening", "turn_number": 0,
+                         "model_generated": False, "latency_ms": 10 ** 9}]
+        assert smoke.timeout_latency_evidence(slow_opening, 240_000) is None
+
+    def test_one_slow_turn_among_fast_ones_is_not_called_a_regression(self) -> None:
+        # Same reason full_response_ms is a median: one unlucky turn is noise.
+        assert smoke.timeout_latency_evidence(
+            self._turns(10_000, 10 ** 9, 12_000), 240_000
+        ) is None
+
+
+class TestDocumentedBudgetAgreement:
+    """BUDGETS_MS must agree with the budget the product publishes.
+
+    The harness deliberately does not import the TypeScript constant, so the
+    only thing keeping the number it enforces equal to the number the product
+    promises is a hand-maintained copy. These tests make that copy load-bearing:
+    a nightly that fails against a stale budget is worse than no nightly, and a
+    performance doc that advertises CI coverage of a budget it no longer lists
+    leaves the claim with no referent.
+    """
+
+    def test_the_shared_latency_constant_agrees_with_the_harness(self) -> None:
+        import re
+
+        metrics = (
+            REPO_ROOT / "packages" / "shared" / "src" / "types" / "metrics.ts"
+        ).read_text(encoding="utf-8")
+        match = re.search(r"FULL_RESPONSE_MS:\s*([\d_]+)", metrics)
+        assert match, "LATENCY_BUDGETS.FULL_RESPONSE_MS not found in metrics.ts"
+        assert int(match.group(1).replace("_", "")) == smoke.BUDGETS_MS["full_response_ms"]
+
+    @pytest.mark.parametrize(
+        "doc_path",
+        [
+            ("docs", "performance.md"),
+            ("docs-site", "src", "content", "docs", "play", "performance.md"),
+        ],
+    )
+    def test_the_enforced_budget_is_published_in_the_performance_doc(
+        self, doc_path: tuple
+    ) -> None:
+        doc = REPO_ROOT.joinpath(*doc_path).read_text(encoding="utf-8")
+        seconds = smoke.BUDGETS_MS["full_response_ms"] / 1000
+        assert "Full NPC response" in doc, (
+            "the only budget this job enforces is missing from the latency table, "
+            "so the doc's claim about CI coverage names nothing a reader can check"
+        )
+        assert f"< {seconds:.0f} s" in doc
+        assert "real-model-smoke" in doc
+
+
+# ---------------------------------------------------------------------------
+# Wall-clock budget
+# ---------------------------------------------------------------------------
+
+
+# Allowance for the steps that burn the job's clock before the harness starts and
+# so are invisible to its own deadline: checkout, three pip installs, the cache
+# restore and, on a cache miss, the download plus the cache save that follows it.
+#
+# Deliberately far above the measurement.  Those steps total ~1 min on the
+# nightlies run so far (see the measured breakdown in docs/real-model-smoke.md),
+# but they are the network-bound part of the job, so this asserts the two
+# ceilings stay compatible with a night on which they run several times slower.
+# Lowering it to the measured ~1 min would make the assertion vacuous.
+PRE_SMOKE_JOB_MINUTES = 9
+
+_WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "model-smoke-nightly.yml"
+
+
+def _workflow_text() -> str:
+    return _WORKFLOW_PATH.read_text(encoding="utf-8")
+
+
+def _workflow_budget_s() -> float:
+    import re
+
+    match = re.search(r"--wall-clock-budget-s (\d+)", _workflow_text())
+    assert match, "the smoke step must pass an explicit --wall-clock-budget-s"
+    return float(match.group(1))
+
+
+def _job_timeout_minutes() -> int:
+    import yaml
+
+    return yaml.safe_load(_workflow_text())["jobs"]["model-smoke"]["timeout-minutes"]
+
+
+class TestWorkflowBudgetAgreement:
+    """The harness deadline must beat the job timeout, or exit 6 is unreachable."""
+
+    def test_the_harness_deadline_trips_before_the_job_timeout(self) -> None:
+        # timeout-minutes covers the whole job; the harness clock starts only at
+        # its own step.  If the two are set as if they measured the same thing,
+        # GitHub cancels the job first and the attributed timeout never prints.
+        harness_min = _workflow_budget_s() / 60
+        job_min = _job_timeout_minutes()
+        assert harness_min + PRE_SMOKE_JOB_MINUTES <= job_min, (
+            f"a {harness_min:.0f} min harness budget plus ~{PRE_SMOKE_JOB_MINUTES} min of "
+            f"setup exceeds the {job_min} min job timeout, so GitHub cancels the job "
+            "before the harness can report an attributed timeout"
+        )
+
+    def test_the_documented_target_is_under_thirty_minutes(self) -> None:
+        # The acceptance criterion in #457: "< 30 min on standard GitHub runners".
+        assert _job_timeout_minutes() <= 30
+
+    def test_the_default_matches_what_the_workflow_passes(self) -> None:
+        assert _workflow_budget_s() == smoke.DEFAULT_WALL_CLOCK_BUDGET_S
+
+    def test_the_download_cannot_eat_the_margin_it_runs_in(self) -> None:
+        # The download is a separate step that runs before the smoke, so it
+        # spends the same margin the assertion above reserves for setup, and the
+        # harness's own Deadline never sees it. Its budget therefore has to fit
+        # inside that margin alongside the other pre-smoke steps, or a slow
+        # Hugging Face night hands the smoke a job clock with no room left and
+        # GitHub cancels the job mid-conversation — no class, no phase, no
+        # remedy, which is the one outcome this whole design rules out.
+        other_pre_smoke_steps_s = 120.0  # checkout, 3 pip installs, cache restore + save
+        assert (
+            smoke.DOWNLOAD_BUDGET_S + other_pre_smoke_steps_s
+            <= PRE_SMOKE_JOB_MINUTES * 60
+        ), (
+            f"a {smoke.DOWNLOAD_BUDGET_S:.0f} s download budget plus the other "
+            f"pre-smoke steps exceeds the ~{PRE_SMOKE_JOB_MINUTES} min margin between "
+            "the harness deadline and the job timeout"
+        )
+
+
+def _workflow_steps() -> list:
+    import yaml
+
+    return yaml.safe_load(_workflow_text())["jobs"]["model-smoke"]["steps"]
+
+
+def _step_running(fragment: str) -> dict:
+    """Return the single workflow step whose ``run`` contains ``fragment``."""
+    matches = [s for s in _workflow_steps() if fragment in (s.get("run") or "")]
+    assert len(matches) == 1, f"expected exactly one step running {fragment!r}"
+    return matches[0]
+
+
+def _steps_using(prefix: str) -> list:
+    return [s for s in _workflow_steps() if (s.get("uses") or "").startswith(prefix)]
+
+
+def _step_index(step: dict) -> int:
+    return _workflow_steps().index(step)
+
+
+class TestWorkflowChecksumInvariants:
+    """#457's "fail loudly on checksum drift" is enforced by the YAML, not the harness.
+
+    Two halves of that promise live in the workflow and nowhere else, and each
+    is one quiet edit away from being given up while every other test in this
+    file stays green:
+
+    * ``verify_model_checksum`` only runs on a full smoke when
+      ``--model-sha256`` is passed. Drop the flag and the harness downgrades to
+      a warning and loads whatever the cache restored — the exact cache-hit
+      drift this job exists to catch.
+    * The harness cannot keep a GGUF it rejected out of the Actions cache; only
+      the step ordering can. ``actions/cache``'s post step saves even when the
+      job failed, and the key is derived from the *expected* hash, so a single
+      poisoned entry would be restored by every later run.
+    """
+
+    def test_the_smoke_reverifies_the_file_it_is_about_to_load(self) -> None:
+        run = _step_running("--ci-hardware-factor")["run"]
+        assert "--model-sha256" in run, (
+            "without --model-sha256 the full smoke only warns that drift was "
+            "not checked, so a corrupt cache hit is fed to the model and the "
+            "run can still go green"
+        )
+
+    def test_the_cache_key_and_the_verified_hash_come_from_one_lookup(self) -> None:
+        import re
+
+        pattern = r"steps\.([A-Za-z0-9_-]+)\.outputs\.model_sha256"
+        restore = _steps_using("actions/cache/restore")
+        assert restore, "the model cache must be restored"
+        key_sources = {m for s in restore for m in re.findall(pattern, s["with"]["key"])}
+        smoke_sources = set(re.findall(pattern, _step_running("--ci-hardware-factor")["run"]))
+        assert key_sources and key_sources == smoke_sources, (
+            "the cache key and the hash the smoke verifies against must come "
+            f"from the same registry lookup (key: {key_sources}, smoke: {smoke_sources}); "
+            "two lookups can drift apart and cache a file under a hash nothing checked"
+        )
+
+    def test_an_unverified_model_is_never_written_to_the_cache(self) -> None:
+        assert not _steps_using("actions/cache@"), (
+            "actions/cache saves from a post step that runs even when the job "
+            "failed; use the split restore/save so the save can be gated"
+        )
+        save = _steps_using("actions/cache/save")
+        assert len(save) == 1, "exactly one step may write the model cache"
+        assert "success()" in str(save[0].get("if", "")), (
+            "the cache save must be gated on success(), or a GGUF that failed "
+            "checksum verification is cached under its expected hash and every "
+            "later run restores the same bad bytes"
+        )
+        download = _step_running("--download-only")
+        assert _step_index(download) < _step_index(save[0]), (
+            "the model must be downloaded and verified before it is cached"
+        )
+
+    def test_the_restored_and_saved_cache_entries_are_the_same(self) -> None:
+        restore = _steps_using("actions/cache/restore")[0]
+        save = _steps_using("actions/cache/save")[0]
+        assert restore["with"]["key"] == save["with"]["key"], (
+            "a save under a different key than the restore never produces a "
+            "cache hit, so every nightly re-downloads 2.5 GB"
+        )
+        assert restore["with"]["path"] == save["with"]["path"]
+
+
+_REAL_MODEL_SMOKE_DOCS = (
+    REPO_ROOT / "docs" / "real-model-smoke.md",
+    REPO_ROOT / "docs-site" / "src" / "content" / "docs" / "dev" / "real-model-smoke.md",
+)
+
+
+class TestLiveServerRemedyIsDocumented:
+    """The runtime row of the doc's failure table sends the reader somewhere real.
+
+    The remedy printed for a `runtime` failure with both children alive is three
+    sentences in a banner; the reasoning behind it — why a 504 is a latency
+    regression rather than a crash, and what the adapter budget has to do with it
+    — lives in the doc. The row links to that section, so the section has to
+    exist in both copies.
+    """
+
+    @pytest.mark.parametrize("doc", _REAL_MODEL_SMOKE_DOCS, ids=lambda p: p.parts[0])
+    def test_the_failure_table_links_a_section_that_exists(self, doc: Path) -> None:
+        text = doc.read_text(encoding="utf-8")
+        assert "(#a-5xx-from-a-live-server)" in text, (
+            "the runtime row no longer links the live-server section"
+        )
+        assert "### A 5xx from a live server" in text
+
+    @pytest.mark.parametrize("doc", _REAL_MODEL_SMOKE_DOCS, ids=lambda p: p.parts[0])
+    def test_the_documented_adapter_budget_is_the_one_the_harness_sets(
+        self, doc: Path
+    ) -> None:
+        # Both the doc and the remedy quote a number the harness computes, so
+        # either can drift from it silently. 540 s is the budget at the factor
+        # the workflow passes.
+        ceiling_s = (
+            smoke.BUDGETS_MS["full_response_ms"] * 20 * smoke.REGRESSION_TOLERANCE
+        ) / 1000
+        expected = max(300.0, ceiling_s * smoke.DEBRIEF_SLOWDOWN_FACTOR + 60.0)
+        assert f"{expected:.0f} s" in doc.read_text(encoding="utf-8")
+        assert f"{expected:.0f} s" in smoke.runtime_without_crash_remedy(expected)
+
+
+class TestWheelInstallInvariants:
+    """llama-cpp-python must arrive as a prebuilt wheel, never as a source build.
+
+    PyPI ships llama-cpp-python as an sdist only — the CPU wheels live on the
+    extra index — and ``--extra-index-url`` does not *prefer* them: pip resolves
+    the newest version across both indexes, so on any night the wheel index lags
+    PyPI, pip compiles llama.cpp on the runner.  That turns the ~16 s install
+    row of the runtime budget into 15+ min, which the 30 min job timeout has no
+    room for next to ~11 min of inference: the job dies with exactly the
+    unattributable "operation was canceled" that the harness's phase-attributed
+    deadline exists to replace, on a night when nothing about the product
+    changed.
+
+    Like the checksum promises above, this lives in the YAML and nowhere else,
+    and it reads as a redundant flag next to the extra index that is "obviously"
+    already doing the job.
+    """
+
+    def test_the_llama_wheel_is_never_built_from_source(self) -> None:
+        run = _step_running("llama-cpp-python[server]")["run"]
+        assert "--only-binary llama-cpp-python" in run, (
+            "--extra-index-url alone does not stop pip from picking a PyPI "
+            "sdist and compiling llama.cpp on the runner, which costs more "
+            "than the whole job budget allows"
+        )
+
+    @pytest.mark.parametrize("doc", _REAL_MODEL_SMOKE_DOCS, ids=lambda p: p.parts[0])
+    def test_the_documented_local_repro_installs_the_same_wheel(self, doc: Path) -> None:
+        # The repro block is a hand-made copy of the install step, so it drifts
+        # silently. A reader who follows it and gets a 15 min source build
+        # concludes the harness is the problem.
+        assert "--only-binary llama-cpp-python" in doc.read_text(encoding="utf-8")
+
+
+class TestDeadline:
+    def test_phase_durations_are_recorded(self) -> None:
+        clock = smoke.Deadline(60.0)
+        clock.enter("checksum")
+        clock.enter("runtime_start")
+        durations = clock.finish()
+        assert set(durations) >= {"startup", "checksum", "runtime_start"}
+
+    def test_exhausted_budget_raises_timeout_naming_the_phase(self) -> None:
+        clock = smoke.Deadline(0.0)
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            clock.check()
+        assert exc_info.value.failure_class == smoke.FailureClass.TIMEOUT
+        assert exc_info.value.exit_code == 6
+        # The phase recorded is the one that ran out of budget.
+        assert exc_info.value.phase == "startup"
+        assert "startup" in str(exc_info.value)
+
+    def test_an_overrun_found_at_a_boundary_blames_the_phase_that_ran_long(self) -> None:
+        # The remedy for a timeout is "look at phase_durations_s", so the phase
+        # it names has to be the one that consumed the budget. Blaming the phase
+        # about to start would point triage at code that never executed.
+        clock = smoke.Deadline(0.0)
+        clock._phase = "conversation"  # the phase that overran
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            clock.enter("debrief")
+        assert exc_info.value.phase == "conversation"
+        assert "conversation" in str(exc_info.value)
+        assert "debrief" not in str(exc_info.value)
+
+    def test_cap_clamps_a_request_timeout_to_the_remaining_budget(self) -> None:
+        clock = smoke.Deadline(30.0)
+        assert clock.cap(10.0) == 10.0
+        assert clock.cap(10_000.0) <= 30.0
+
+    def test_cap_never_returns_a_useless_zero_timeout(self) -> None:
+        clock = smoke.Deadline(1.0)
+        assert clock.cap(600.0) >= 5.0
+
+    def test_finish_does_not_raise_on_an_exhausted_budget(self) -> None:
+        # finish() runs from run_smoke's finally block, where the timeout has
+        # already been classified; raising there would discard the verdict.
+        clock = smoke.Deadline(0.0)
+        assert "startup" in clock.finish()
+
+
+# ---------------------------------------------------------------------------
+# Reporting
+# ---------------------------------------------------------------------------
+
+
+class TestStepSummary:
+    def test_failure_summary_names_class_and_remedy(self) -> None:
+        text = smoke.render_step_summary({
+            "verdict": "fail",
+            "model_id": "qwen3-4b",
+            "failure_class": smoke.FailureClass.RUNTIME,
+            "exit_code": 4,
+            "failures": ["convsim-core exited with code 1"],
+            "wall_clock_s": 123.0,
+            "wall_clock_budget_s": 1500.0,
+            "phase_durations_s": {"runtime_start": 42.0},
+        })
+        assert "FAIL" in text
+        assert "`runtime`" in text
+        assert "convsim-core exited with code 1" in text
+        assert smoke.REMEDIES[smoke.FailureClass.RUNTIME] in text
+
+    def test_pass_summary_tabulates_measurements_with_ceilings(self) -> None:
+        text = smoke.render_step_summary({
+            "verdict": "pass",
+            "model_id": "qwen3-4b",
+            "ci_hardware_factor": 20.0,
+            "measured_ms": {"full_response_ms": 110_000, "debrief_ms": 200_000},
+            "wall_clock_s": 900.0,
+            "wall_clock_budget_s": 1500.0,
+        })
+        assert "PASS" in text
+        assert "240000 ms" in text  # 10 000 × 20 × 1.2
+        assert "| `debrief_ms` | 200000 ms | — |" in text
+
+    def test_a_green_run_still_surfaces_what_it_did_not_prove(self) -> None:
+        # Every warning the harness raises says what an otherwise-green verdict
+        # does not establish -- a fallback narrative, a recited opening, a
+        # conversation the NPC cut short, parse flags it could not read. The step
+        # summary is where they are read, so a pass that drops them reports more
+        # confidence than the run earned.
+        text = smoke.render_step_summary({
+            "verdict": "pass",
+            "model_id": "qwen3-4b",
+            "warnings": ["Debrief narrative used the deterministic fallback"],
+        })
+        assert "### Warnings" in text
+        assert "- Debrief narrative used the deterministic fallback" in text
+
+    def test_a_pre_run_failure_does_not_claim_a_zero_second_run(self) -> None:
+        # --download-only / --verify-only and the registry lookup write a summary
+        # without ever starting a clock; reporting "Wall clock: 0 s (budget 0 s)"
+        # on a download verdict invents a measurement the run never took.
+        text = smoke.render_step_summary({
+            "verdict": "fail",
+            "model_id": "qwen3-4b",
+            "failure_class": smoke.FailureClass.DOWNLOAD,
+            "exit_code": 2,
+            "failures": ["Download failed for https://…: TimeoutError()"],
+            "remedy": smoke.REMEDIES[smoke.FailureClass.DOWNLOAD],
+        })
+        assert "Wall clock" not in text
+        assert "`download`" in text
+        assert smoke.REMEDIES[smoke.FailureClass.DOWNLOAD] in text
+        assert "**Model:** `qwen3-4b`" in text
+
+    def test_a_registry_failure_does_not_name_a_model_it_never_resolved(self) -> None:
+        # --print-registry-model runs before any --model-id exists, and it is
+        # the mode that fails on a registry with no usable (or not yet pinned)
+        # `starter` entry. Rendering `model_id` unconditionally put
+        # "**Model:** `None`" at the very top of that verdict.
+        text = smoke.render_step_summary({
+            "verdict": "fail",
+            "model_id": None,
+            "failure_class": smoke.FailureClass.PIPELINE,
+            "exit_code": 5,
+            "failures": ["No model with role 'starter' in model-registry/registry.yaml"],
+            "remedy": smoke.REGISTRY_REMEDY,
+        })
+        assert "Model:" not in text
+        assert "None" not in text
+        assert smoke.REGISTRY_REMEDY in text
+
+    def test_summary_is_not_written_without_the_github_env_var(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False)
+        smoke._write_step_summary({"verdict": "pass"})  # must not raise
+
+    def test_summary_is_appended_when_github_env_var_is_set(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        path = tmp_path / "summary.md"
+        monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(path))
+        smoke._write_step_summary({"verdict": "pass", "model_id": "m"})
+        assert "PASS" in path.read_text(encoding="utf-8")
+
+    def test_report_is_valid_json(self, tmp_path: Path) -> None:
+        path = tmp_path / "nested" / "report.json"
+        smoke._write_report(path, {"verdict": "pass", "failures": []})
+        assert json.loads(path.read_text(encoding="utf-8"))["verdict"] == "pass"
+
+
+# ---------------------------------------------------------------------------
+# Event-payload extraction
+# ---------------------------------------------------------------------------
+
+
+class TestEventExtraction:
+    def test_npc_turn_content_is_extracted(self) -> None:
+        events = [
+            {"event_type": "player_turn", "payload": {"content": "hi"}},
+            {"event_type": "npc_turn", "payload": {"content": "Go on."}},
+        ]
+        assert smoke._npc_turn_content(events) == "Go on."
+
+    def test_npc_opening_content_is_extracted(self) -> None:
+        events = [{"event_type": "npc_opening", "payload": {"content": "Welcome."}}]
+        assert smoke._npc_turn_content(events) == "Welcome."
+
+    def test_absent_npc_event_yields_empty_string(self) -> None:
+        assert smoke._npc_turn_content([{"event_type": "tts_chunk", "payload": {}}]) == ""
+
+    def test_a_non_dict_payload_does_not_raise(self) -> None:
+        # The excerpt is reported, not asserted on, so a surprising payload shape
+        # must not turn an otherwise-green run into a harness bug (exit 5).
+        assert smoke._npc_turn_content([{"event_type": "npc_turn", "payload": None}]) == ""
+        assert smoke._npc_turn_content([{"event_type": "npc_turn"}]) == ""
+
+    def test_rubric_observations_are_counted(self) -> None:
+        events = [{"event_type": "npc_turn", "payload": {
+            "content": "Go on.",
+            "rubric_observations": [
+                {"rubric_id": "structure", "observation": "o", "score_delta": 1},
+                {"rubric_id": "evidence", "observation": "o", "score_delta": -1},
+            ],
+        }}]
+        assert smoke._rubric_observation_count(events) == 2
+
+    def test_an_empty_observation_list_counts_zero(self) -> None:
+        # An empty list is a real observation of zero: the model was asked (by
+        # the schema's bare hint) and volunteered nothing.
+        assert smoke._rubric_observation_count(
+            [{"event_type": "npc_turn", "payload": {
+                "content": "x", "rubric_observations": []}}]
+        ) == 0
+
+    @pytest.mark.parametrize("payload", [
+        {"content": "x"},                                  # key absent entirely
+        {"content": "x", "rubric_observations": None},     # present but unusable
+        {"content": "x", "rubric_observations": "two"},
+    ])
+    def test_an_unreadable_observation_list_is_unknown_not_zero(
+        self, payload: dict
+    ) -> None:
+        # Counting an unreadable payload as zero would let the unscored-debrief
+        # verdict assert "no NPC turn carried a rubric_observation" — sending
+        # triage after the prompt layer and the model pin — when what actually
+        # happened is that the turn response no longer exposes the list.
+        assert smoke._rubric_observation_count(
+            [{"event_type": "npc_turn", "payload": payload}]
+        ) is None
+
+    @pytest.mark.parametrize(("counts", "expected"), [
+        ([0, 0, 0], 0),
+        ([1, 0, 2], 3),
+        ([None, None, None], None),      # nothing readable anywhere
+        ([None, 0, 0], None),            # a gap with no positive evidence
+        ([0, None, 0], None),            # ...wherever in the run the gap falls
+        ([None, 2, 0], 2),               # observations were seen regardless
+    ])
+    def test_a_zero_total_is_reported_only_when_every_turn_was_readable(
+        self, counts: list, expected: int | None
+    ) -> None:
+        # None does not mean "nothing was readable": two readable zeroes
+        # alongside one unreadable turn is still None, because reporting 0 would
+        # have the verdict assert the model volunteered nothing across the
+        # conversation when the unread turn may have carried plenty.
+        turns = [{"model_generated": False, "label": "npc_opening"}] + [
+            {"model_generated": True, "rubric_observation_count": c} for c in counts
+        ]
+        assert smoke._total_rubric_observations(turns) == expected
+
+    def test_a_run_with_no_generated_turns_reports_an_unknown_total(self) -> None:
+        # Nothing was observed, which is not the same as observing nothing.
+        assert smoke._total_rubric_observations(
+            [{"model_generated": False, "label": "npc_opening"}]
+        ) is None
+
+    def test_excerpt_is_bounded_and_collapsed(self) -> None:
+        excerpt = smoke._excerpt("a\n\n  b" + "x" * 500)
+        assert excerpt.startswith("a b")
+        assert len(excerpt) <= smoke.EXCERPT_CHARS + 1  # + the ellipsis
+        assert excerpt.endswith("…")
+
+    def test_excerpting_a_non_string_shows_its_shape_instead_of_raising(self) -> None:
+        # _excerpt alone raises AttributeError here (str.split), which is why the
+        # report assembly in run_smoke goes through _excerpt_any.
+        assert smoke._excerpt_any({"text": "hi"}) == "{'text': 'hi'}"
+        assert smoke._excerpt_any(42) == "42"
+
+    def test_excerpting_an_absent_value_is_empty_not_the_word_none(self) -> None:
+        assert smoke._excerpt_any(None) == ""
+
+    def test_excerpting_a_string_is_unchanged_by_the_tolerant_wrapper(self) -> None:
+        assert smoke._excerpt_any("a\n\n  b") == smoke._excerpt("a\n\n  b")
+
+
+# ---------------------------------------------------------------------------
+# Scripted conversation shape
+# ---------------------------------------------------------------------------
+
+
+class TestScriptedConversation:
+    def test_the_script_is_a_multi_turn_conversation(self) -> None:
+        assert len(smoke.SCRIPTED_PLAYER_TURNS) >= 2
+        assert all(t.strip() for t in smoke.SCRIPTED_PLAYER_TURNS)
+
+    def test_the_script_matches_the_fake_runtime_playthrough(self) -> None:
+        # Keeping the two harnesses on the same script means the nightly and
+        # the release-time fake-runtime smoke cover the same conversation shape,
+        # so a difference between them is the runtime and nothing else.
+        source = (REPO_ROOT / "tests" / "e2e" / "test_scripted_playthrough.py").read_text(
+            encoding="utf-8"
+        )
+        for turn in smoke.SCRIPTED_PLAYER_TURNS:
+            assert turn in source, f"scripted turn drifted from the e2e playthrough: {turn!r}"
+
+
+# ---------------------------------------------------------------------------
+# HTTP transport: status code -> failure class
+# ---------------------------------------------------------------------------
+
+
+class _Response:
+    """Minimum of an http.client.HTTPResponse that _request_json touches."""
+
+    def __init__(self, status: int = 200, body: bytes = b"{}") -> None:
+        self.status = status
+        self._body = body
+
+    def read(self) -> bytes:
+        return self._body
+
+    def __enter__(self) -> "_Response":
+        return self
+
+    def __exit__(self, *a: object) -> bool:
+        return False
+
+
+def _http_error(code: int, body: bytes = b"server traceback"):
+    def _raise(req, timeout=None):
+        raise urllib.error.HTTPError(
+            getattr(req, "full_url", "http://x"), code, "err", {}, io.BytesIO(body)
+        )
+
+    return _raise
+
+
+class TestRequestClassification:
+    """`runtime` means the server broke; `pipeline` means it rejected the ask.
+
+    Every orchestration test below replaces _request_json wholesale, so without
+    these the one function that decides "crashed server" vs "end-to-end
+    assertion" — the distinction #457 asks CI output to make — is never run.
+    """
+
+    @pytest.mark.parametrize("code", [500, 502, 503])
+    def test_a_5xx_is_a_runtime_failure(
+        self, code: int, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setattr(smoke.urllib.request, "urlopen", _http_error(code))
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._request_json("http://127.0.0.1:1/api/sessions", payload={}, timeout=1)
+        assert exc_info.value.failure_class == smoke.FailureClass.RUNTIME
+        assert exc_info.value.exit_code == 4
+
+    @pytest.mark.parametrize("code", [400, 404, 409, 422])
+    def test_a_4xx_is_a_pipeline_failure(
+        self, code: int, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # The server is up and routing; it refused what the smoke asked for —
+        # a 409 from POST /end on an already-ended session, say. That is an
+        # assertion about the API contract, not a crash.
+        monkeypatch.setattr(smoke.urllib.request, "urlopen", _http_error(code))
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._request_json("http://127.0.0.1:1/api/sessions", payload={}, timeout=1)
+        assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE
+        assert exc_info.value.exit_code == 5
+
+    def test_the_server_side_detail_reaches_the_failure_message(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # The 5xx body is the only place convsim-core's own error text appears
+        # on the client side, and it is what the runtime remedy sends a triager
+        # to read.
+        monkeypatch.setattr(
+            smoke.urllib.request, "urlopen",
+            _http_error(500, b"Debrief generation failed"),
+        )
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._request_json("http://127.0.0.1:1/api/x", payload={}, timeout=1)
+        assert "Debrief generation failed" in str(exc_info.value)
+
+    @pytest.mark.parametrize("exc", [
+        ConnectionResetError("peer hung up"),
+        TimeoutError("read timed out"),
+    ])
+    def test_a_transport_error_is_a_runtime_failure(
+        self, exc: BaseException, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        def _raise(req, timeout=None):
+            raise exc
+
+        monkeypatch.setattr(smoke.urllib.request, "urlopen", _raise)
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._request_json("http://127.0.0.1:1/api/health", timeout=1)
+        assert exc_info.value.failure_class == smoke.FailureClass.RUNTIME
+
+    def test_an_unparseable_body_is_a_runtime_failure(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setattr(
+            smoke.urllib.request, "urlopen",
+            lambda req, timeout=None: _Response(200, b"<html>502 Bad Gateway</html>"),
+        )
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._request_json("http://127.0.0.1:1/api/health", timeout=1)
+        assert exc_info.value.failure_class == smoke.FailureClass.RUNTIME
+
+    def test_an_unexpected_success_status_is_a_pipeline_failure(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # urllib raises for >= 400, so this branch only ever sees a 2xx/3xx the
+        # contract did not promise.
+        monkeypatch.setattr(
+            smoke.urllib.request, "urlopen", lambda req, timeout=None: _Response(204)
+        )
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._request_json("http://127.0.0.1:1/api/health", timeout=1)
+        assert exc_info.value.failure_class == smoke.FailureClass.PIPELINE
+        assert "HTTP 204" in str(exc_info.value)
+
+    def test_a_healthy_response_is_returned(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(
+            smoke.urllib.request, "urlopen",
+            lambda req, timeout=None: _Response(200, b'{"session_id": "sess-1"}'),
+        )
+        assert smoke._request_json(
+            "http://127.0.0.1:1/api/sessions", payload={"a": 1}, timeout=1
+        ) == {"session_id": "sess-1"}
+
+
+# ---------------------------------------------------------------------------
+# Readiness polling
+# ---------------------------------------------------------------------------
+
+
+class _PolledProc:
+    def __init__(self, returncode: int | None = None) -> None:
+        self.returncode = returncode
+
+    def poll(self) -> int | None:
+        return self.returncode
+
+
+class TestWaitForHttp:
+    """A server that died on startup is nameable immediately, not after a wait."""
+
+    def test_a_child_that_exited_short_circuits_the_wait(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # The realistic case is a runner OOM on model load. Polling a port
+        # nobody is listening on for the full timeout would bury the exit code
+        # and burn the wall-clock budget that exit 6 is measured against.
+        def _refused(*a: object, **k: object):
+            raise ConnectionRefusedError()
+
+        monkeypatch.setattr(smoke.urllib.request, "urlopen", _refused)
+        # The timeout is deliberately far below the 300 s the real model-load
+        # wait allows: a regression that drops the short-circuit must turn this
+        # test red quickly rather than stall the per-PR suite for five minutes.
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._wait_for_http(
+                "http://127.0.0.1:1/v1/models", timeout_s=2.0,
+                label="llama-server", proc=_PolledProc(137),
+            )
+        assert exc_info.value.failure_class == smoke.FailureClass.RUNTIME
+        assert "llama-server exited with code 137" in str(exc_info.value)
+
+    def test_a_server_that_never_answers_is_a_runtime_failure(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        def _refused(*a: object, **k: object):
+            raise ConnectionRefusedError()
+
+        monkeypatch.setattr(smoke.urllib.request, "urlopen", _refused)
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._wait_for_http(
+                "http://127.0.0.1:1/api/health", timeout_s=0.0, label="convsim-core"
+            )
+        assert exc_info.value.failure_class == smoke.FailureClass.RUNTIME
+        assert "convsim-core" in str(exc_info.value)
+
+    def test_the_not_ready_message_carries_the_last_connection_error(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # It is the only diagnostic a readiness `runtime` verdict leaves behind,
+        # and str() on an exception raised with no arguments is empty.
+        def _refused(*a: object, **k: object):
+            raise ConnectionRefusedError()
+
+        monkeypatch.setattr(smoke.urllib.request, "urlopen", _refused)
+        # The poll backs off a second between attempts; skip the wait so this
+        # test costs nothing. The timeout still has to leave room for one
+        # attempt, or last_err is never set.
+        monkeypatch.setattr(smoke.time, "sleep", lambda _s: None)
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._wait_for_http(
+                "http://127.0.0.1:1/api/health", timeout_s=0.01, label="convsim-core"
+            )
+        assert "ConnectionRefusedError" in str(exc_info.value)
+
+    def test_a_live_child_is_not_mistaken_for_a_crashed_one(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setattr(
+            smoke.urllib.request, "urlopen", lambda *a, **k: _Response(200)
+        )
+        assert smoke._wait_for_http(
+            "http://127.0.0.1:1/v1/models", timeout_s=30.0,
+            label="llama-server", proc=_PolledProc(None),
+        ) is None
+
+    def test_an_http_error_still_means_the_server_is_up(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # A 404 is a routed response: the process is listening, which is all
+        # readiness means.
+        monkeypatch.setattr(smoke.urllib.request, "urlopen", _http_error(404))
+        assert smoke._wait_for_http(
+            "http://127.0.0.1:1/api/health", timeout_s=30.0, label="convsim-core"
+        ) is None
+
+
+# ---------------------------------------------------------------------------
+# Port ownership
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture()
+def occupied_port():
+    """A loopback port with a live listener on it, released after the test."""
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.bind(("127.0.0.1", 0))
+    sock.listen(1)
+    try:
+        yield sock.getsockname()[1]
+    finally:
+        sock.close()
+
+
+@pytest.fixture()
+def free_port() -> int:
+    """A loopback port nothing is listening on."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+class TestPortOwnership:
+    """The smoke must own its ports, or its proof of a real model is worthless.
+
+    Readiness is a URL poll, so a server the harness did not start answers it
+    and the run proceeds against a model whose bytes were never verified. The
+    child that lost the bind dies, but ``_crashed_child`` is only consulted
+    once a failure has been raised, so a run that otherwise passes never
+    notices -- it just reports a green verdict for the wrong model.
+    """
+
+    def test_a_free_port_is_accepted(self, free_port: int) -> None:
+        assert smoke._assert_port_free(free_port, "llama-server") is None
+
+    def test_an_occupied_port_is_a_runtime_failure(self, occupied_port: int) -> None:
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._assert_port_free(occupied_port, "llama-server")
+        assert exc_info.value.failure_class == smoke.FailureClass.RUNTIME
+        assert exc_info.value.exit_code == 4
+        assert str(occupied_port) in str(exc_info.value)
+
+    def test_the_remedy_does_not_send_triage_to_an_unstarted_child(
+        self, occupied_port: int
+    ) -> None:
+        # `runtime`'s stock advice is "read the child stderr tail"; here no
+        # child has been started, so the class remedy would mislead.
+        with pytest.raises(smoke.SmokeFailure) as exc_info:
+            smoke._assert_port_free(occupied_port, "convsim-core")
+        remedy = exc_info.value.remedy
+        assert "Stop whatever is listening" in remedy
+        assert remedy != smoke.REMEDIES[smoke.FailureClass.RUNTIME]
+
+    def test_a_squatter_stops_the_run_before_any_server_is_started(
+        self,
+        staged_model,
+        occupied_port: int,
+        monkeypatch: pytest.MonkeyPatch,
+        tmp_path: Path,
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        started: list[str] = []
+
+        def _record(label: str):
+            def _start(*args: object, **kwargs: object) -> "_FakeProc":
+                started.append(label)
+                return _FakeProc()
+            return _start
+
+        monkeypatch.setattr(smoke, "LLAMA_SERVER_PORT", occupied_port)
+        monkeypatch.setattr(smoke, "_start_llama_server", _record("llama"))
+        monkeypatch.setattr(smoke, "_start_core", _record("core"))
+        monkeypatch.setattr(smoke, "_wait_for_http", lambda *a, **k: None)
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(_debrief()))
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == 4
+        assert started == [], "the squatter must be caught before anything is spawned"
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failure_class"] == smoke.FailureClass.RUNTIME
+        assert results["failed_phase"] == "runtime_start"
+        assert "Stop whatever is listening" in results["remedy"]
+
+
+# ---------------------------------------------------------------------------
+# Full-run orchestration, with llama-server and convsim-core faked out
+# ---------------------------------------------------------------------------
+
+
+class _FakeProc:
+    """Minimum of subprocess.Popen that run_smoke touches."""
+
+    def __init__(self, returncode: int | None = None) -> None:
+        self.stdout = iter(())
+        self.stderr = iter(())
+        self._returncode = returncode
+        self.terminated = False
+
+    @property
+    def returncode(self) -> int | None:
+        return self._returncode
+
+    def poll(self) -> int | None:
+        return self._returncode
+
+    def terminate(self) -> None:
+        self.terminated = True
+
+    def wait(self, timeout: float | None = None) -> int:
+        return self._returncode or 0
+
+    def kill(self) -> None:  # pragma: no cover - only on a hung child
+        self.terminated = True
+
+
+_OPENING = {"events": [{"event_type": "npc_opening",
+                        "payload": {"content": "Thanks for coming in today."}}]}
+# convsim-core always puts a rubric_observations list in the npc_turn payload
+# (see the npc_event payload in routers/sessions.py), empty or not, so the
+# baseline fake carries one too. A payload *without* the key is a different
+# situation the harness must not read as "the model volunteered none" — see
+# test_a_turn_payload_without_observations_is_not_read_as_zero.
+_NPC_TURN = {"events": [{"event_type": "npc_turn",
+                         "payload": {"content": "Walk me through that trade-off.",
+                                     "rubric_observations": []}}],
+             "ending_type": None}
+
+
+def _npc_turn_with_observations(count: int) -> dict:
+    """An NPC turn whose payload carries ``count`` validated rubric observations."""
+    return {"events": [{"event_type": "npc_turn", "payload": {
+        "content": "Walk me through that trade-off.",
+        "rubric_observations": [
+            {"rubric_id": f"dim{i}", "observation": "o", "score_delta": 1}
+            for i in range(count)
+        ],
+    }}], "ending_type": None}
+
+
+def _fake_core(debrief: dict, *, debug_turns: list | None = None, turn: dict | None = None):
+    """Build a _request_json stand-in that answers convsim-core's endpoints."""
+    calls: list[str] = []
+
+    def _request_json(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+        calls.append(url)
+        if url.endswith("/health"):
+            return {"llm_runtime": {"runtime_id": "llama_cpp"}}
+        if url.endswith("/api/sessions"):
+            return {"session_id": "sess-1"}
+        if url.endswith("/start"):
+            return _OPENING
+        if url.endswith("/turn"):
+            return turn if turn is not None else _NPC_TURN
+        if url.endswith("/debug"):
+            return {"turns": debug_turns if debug_turns is not None else [
+                {"turn_number": n, "used_fallback": False,
+                 "used_native_structured_output": True}
+                for n in range(1, len(smoke.SCRIPTED_PLAYER_TURNS) + 1)
+            ]}
+        if url.endswith("/end"):
+            return {"state": "Ended"}
+        if url.endswith("/debrief"):
+            return debrief
+        raise AssertionError(f"unexpected request to {url}")
+
+    _request_json.calls = calls  # type: ignore[attr-defined]
+    return _request_json
+
+
+@pytest.fixture()
+def staged_model(tmp_path: Path) -> tuple[Path, str, str]:
+    """A fake GGUF on disk plus its id and digest."""
+    models_dir = tmp_path / "models"
+    models_dir.mkdir()
+    payload = b"\x00gguf stand-in"
+    (models_dir / "test-model.gguf").write_bytes(payload)
+    return models_dir, "test-model", hashlib.sha256(payload).hexdigest()
+
+
+@pytest.fixture()
+def fake_servers(monkeypatch: pytest.MonkeyPatch) -> dict[str, _FakeProc]:
+    """Replace both child processes and the readiness polling with fakes."""
+    procs = {"llama": _FakeProc(), "core": _FakeProc()}
+    monkeypatch.setattr(smoke, "_start_llama_server", lambda *a, **k: procs["llama"])
+    monkeypatch.setattr(smoke, "_start_core", lambda *a, **k: procs["core"])
+    monkeypatch.setattr(smoke, "_wait_for_http", lambda *a, **k: None)
+    # No child is really spawned, so no port is really needed. Left live, the
+    # port check would make every test below depend on whether the developer
+    # happens to have a llama-server on 7356 -- which, on the machine this was
+    # written on, they did. TestPortOwnership covers the check itself.
+    monkeypatch.setattr(smoke, "_assert_port_free", lambda *a, **k: None)
+    return procs
+
+
+class TestRunSmokeOrchestration:
+    """The phases, the report and the exit code, without a real model."""
+
+    def test_healthy_run_passes_and_reports(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(_debrief()))
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["verdict"] == "pass"
+        assert results["failure_class"] is None
+        assert results["runtime_id"] == "llama_cpp"
+        # One authored opening + every scripted player turn.
+        assert len(results["turns"]) == len(smoke.SCRIPTED_PLAYER_TURNS) + 1
+        assert results["turns"][0]["model_generated"] is False
+        assert all(t["model_generated"] for t in results["turns"][1:])
+        assert all(t["used_native_structured_output"] for t in results["turns"][1:])
+        assert results["debrief"]["overall_score"] == 52.0
+        assert set(results["measured_ms"]) >= {
+            "session_start_ms", "full_response_ms", "debrief_ms"
+        }
+        assert set(results["phase_durations_s"]) >= {
+            "checksum", "runtime_start", "conversation", "debrief", "assertions", "budget"
+        }
+        assert results["warnings"] == []
+
+    def test_both_children_are_stopped_even_on_a_pass(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(_debrief()))
+        smoke.run_smoke(model_id, 20.0, None, model_sha256=digest, models_dir=models_dir)
+        assert fake_servers["llama"].terminated
+        assert fake_servers["core"].terminated
+
+    def test_the_data_directory_outlives_convsim_core_and_is_then_removed(
+        self, staged_model, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """The throwaway data dir must not be deleted while core is still running.
+
+        It holds convsim-core's SQLite database, WAL and logs. Removing it
+        before the child is stopped races the server's own writes, and an
+        rmtree that lost that race would be caught as an unexpected harness
+        error and reported as a `pipeline` failure on an otherwise green run.
+        """
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(_debrief()))
+        seen: dict = {}
+
+        core = _FakeProc()
+
+        def _terminate() -> None:
+            seen["existed_at_terminate"] = seen["data_dir"].exists()
+            core.terminated = True
+
+        core.terminate = _terminate  # type: ignore[method-assign]
+
+        def _start_core(data_dir, *args, **kwargs) -> _FakeProc:
+            seen["data_dir"] = Path(data_dir)
+            return core
+
+        monkeypatch.setattr(smoke, "_start_llama_server", lambda *a, **k: _FakeProc())
+        monkeypatch.setattr(smoke, "_start_core", _start_core)
+        monkeypatch.setattr(smoke, "_wait_for_http", lambda *a, **k: None)
+        # Builds its own children rather than using fake_servers, so it has to
+        # neutralise the port check the same way that fixture does.
+        monkeypatch.setattr(smoke, "_assert_port_free", lambda *a, **k: None)
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, None, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == 0
+        assert seen["existed_at_terminate"] is True
+        assert not seen["data_dir"].exists(), "the data directory leaked"
+
+    def test_a_conversation_of_recited_openings_does_not_pass(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        """Every flag healthy, every "reply" the authored opening read back.
+
+        This is the run the parse flags cannot fail: the utterances are real
+        model output, non-empty and not the canned fallback. It must not report
+        that the model produced real NPC turns.
+        """
+        models_dir, model_id, digest = staged_model
+        opening = _OPENING["events"][0]["payload"]["content"]
+        recital = {"events": [{"event_type": "npc_turn", "payload": {
+            "content": f"{opening} Now, tell me about a time you disagreed.",
+            "rubric_observations": [],
+        }}], "ending_type": None}
+        monkeypatch.setattr(
+            smoke, "_request_json", _fake_core(_debrief(), turn=recital)
+        )
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failed_phase"] == "assertions"
+        assert any("reciting the authored opening" in f for f in results["failures"])
+        # Recorded per turn, so triage can see which turns did it without
+        # re-reading the excerpts.
+        assert all(t["replayed_opening"] for t in results["turns"][1:])
+
+    def test_a_healthy_run_records_that_no_turn_recited_the_opening(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(_debrief()))
+        report = tmp_path / "report.json"
+
+        assert smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        ) == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert all(t["replayed_opening"] is False for t in results["turns"][1:])
+        assert results["warnings"] == []
+
+    def test_unscored_debrief_is_a_pipeline_failure(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(
+            smoke, "_request_json",
+            _fake_core(_debrief(scores={}, overall_score=None)),
+        )
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failure_class"] == smoke.FailureClass.PIPELINE
+        assert results["failed_phase"] == "assertions"
+        assert any("rubric dimension scores" in f for f in results["failures"])
+        # The artifact carries the advice, not just the class: the failure table
+        # in the docs sends triage here for a `pipeline` verdict.
+        assert results["remedy"] == smoke.REMEDIES[smoke.FailureClass.PIPELINE]
+        # The turns volunteered nothing to score, so the verdict says so rather
+        # than asking the reader to go and diff the previous nightly's artifact.
+        assert results["rubric_observations_seen"] == 0
+        assert any(smoke.UNSCORED_DEBRIEF_NOTE in f for f in results["failures"])
+
+    def test_a_debrief_summary_that_is_not_text_is_a_product_verdict(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        """A changed response contract must not be reported as a harness bug.
+
+        The summary is read twice on the way to a verdict, and the *first* read is
+        the report assembly in run_smoke, which runs before evaluate_debrief: a
+        guard in the assertion layer alone leaves the raise reachable. Both go
+        through the tolerant excerpt, so the run reaches its classified verdict
+        and the artifact still shows what arrived.
+        """
+        models_dir, model_id, digest = staged_model
+        summary = {"text": "You gave concrete examples but hedged on the trade-off."}
+        monkeypatch.setattr(
+            smoke, "_request_json", _fake_core(_debrief(summary=summary))
+        )
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failure_class"] == smoke.FailureClass.PIPELINE
+        assert results["failed_phase"] == "assertions"
+        assert any("summary is not a string" in f for f in results["failures"])
+        # The product's remedy, not the harness-bug one the catch-all prints.
+        assert results["remedy"] == smoke.REMEDIES[smoke.FailureClass.PIPELINE]
+        assert not any("bug in the smoke harness" in f for f in results["failures"])
+        # And the payload that caused it survives into the artifact.
+        assert "hedged on the trade-off" in results["debrief"]["summary_excerpt"]
+
+    def test_a_conversation_of_fallbacks_does_not_blame_the_rubric_prompt(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        """The failure mode a 4 B model on CPU actually produces.
+
+        Every turn's output fails validation, so every turn returns the canned
+        safe utterance with an empty rubric_observations list and the debrief
+        scores nothing. Both failures land in one verdict, and the unscored one
+        must not send triage after the OUTPUT_SCHEMA layer, the model pin and
+        sampling settings when the fallback failure beside it already names the
+        thing to read.
+        """
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(
+            smoke, "_request_json",
+            _fake_core(
+                _debrief(scores={}, overall_score=None),
+                debug_turns=[
+                    {"turn_number": n, "used_fallback": True,
+                     "used_native_structured_output": False}
+                    for n in range(1, len(smoke.SCRIPTED_PLAYER_TURNS) + 1)
+                ],
+            ),
+        )
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["rubric_observations_seen"] == 0
+        assert any("fell back to the canned" in f for f in results["failures"])
+        no_scores = next(f for f in results["failures"] if "rubric dimension scores" in f)
+        assert smoke.UNSCORED_AFTER_FALLBACK_NOTE in no_scores
+        assert smoke.UNSCORED_DEBRIEF_NOTE not in no_scores
+
+    def test_unscored_debrief_with_scorable_turns_is_named_a_regression(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        """The same exit 5, the opposite diagnosis.
+
+        When the NPC turns did return rubric observations and the debrief still
+        scored nothing, the model is not the problem and the stock note
+        ("nothing asks the model for them") would excuse a real bug in the
+        debrief engine's score accumulation.
+        """
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(
+            smoke, "_request_json",
+            _fake_core(
+                _debrief(scores={}, overall_score=None),
+                turn=_npc_turn_with_observations(2),
+            ),
+        )
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["rubric_observations_seen"] == 2 * len(smoke.SCRIPTED_PLAYER_TURNS)
+        assert all(
+            t["rubric_observation_count"] == 2 for t in results["turns"][1:]
+        ), "the per-turn count belongs in the artifact, not just the total"
+        assert any(smoke.UNSCORED_WITH_OBSERVATIONS_NOTE in f for f in results["failures"])
+        assert not any(smoke.UNSCORED_DEBRIEF_NOTE in f for f in results["failures"])
+
+    def test_a_turn_payload_without_observations_is_not_read_as_zero(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        """A third case: the harness could not read what the turns volunteered.
+
+        convsim-core always sends a rubric_observations list today, so a payload
+        without one means the turn response contract changed. Scoring that as
+        "the model volunteered nothing" would have the verdict confidently name
+        the one cause the run has no evidence for, and send triage to the prompt
+        layer and the model pin. Say both causes are open instead.
+        """
+        models_dir, model_id, digest = staged_model
+        no_observations_key = {
+            "events": [{"event_type": "npc_turn",
+                        "payload": {"content": "Walk me through that trade-off."}}],
+            "ending_type": None,
+        }
+        monkeypatch.setattr(
+            smoke, "_request_json",
+            _fake_core(
+                _debrief(scores={}, overall_score=None),
+                turn=no_observations_key,
+            ),
+        )
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["rubric_observations_seen"] is None
+        assert all(
+            t["rubric_observation_count"] is None for t in results["turns"][1:]
+        ), "a null per-turn count is what tells the reader the list was unreadable"
+        no_scores = next(f for f in results["failures"] if "rubric dimension" in f)
+        # Names what it actually observed (an unreadable list) and keeps both
+        # causes open, rather than asserting the one it has no evidence for.
+        assert "unreadable" in no_scores
+        assert "cannot be told apart" in no_scores
+        assert smoke.UNSCORED_DEBRIEF_NOTE in no_scores
+        assert smoke.UNSCORED_WITH_OBSERVATIONS_NOTE in no_scores
+
+    def test_latency_regression_is_a_budget_failure_not_a_pipeline_one(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        import time
+
+        models_dir, model_id, digest = staged_model
+        base = _fake_core(_debrief())
+
+        def _slow_turn(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/turn"):
+                time.sleep(0.02)  # measurable, so the 0 ms ceiling is exceeded
+            return base(url, payload=payload, timeout=timeout, expect=expect)
+
+        monkeypatch.setattr(smoke, "_request_json", _slow_turn)
+        report = tmp_path / "report.json"
+
+        # Factor 0 ⇒ a 0 ms ceiling, so any measurable latency regresses.
+        exit_code = smoke.run_smoke(
+            model_id, 0.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.BUDGET]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failure_class"] == smoke.FailureClass.BUDGET
+        assert results["failed_phase"] == "budget"
+
+    def test_the_headline_full_response_is_the_median_not_the_worst_turn(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # One unlucky turn -- runner steal, an unusually long NPC answer -- must
+        # not move the figure the budget judges, or the nightly flaps.  A real
+        # regression moves every turn and so moves the median.
+        import time
+
+        models_dir, model_id, digest = staged_model
+        base = _fake_core(_debrief())
+        total_turns = len(smoke.SCRIPTED_PLAYER_TURNS)
+        seen = {"turns": 0}
+
+        def _one_slow_turn(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/turn"):
+                seen["turns"] += 1
+                if seen["turns"] == total_turns:  # only the last turn is slow
+                    time.sleep(0.25)
+            return base(url, payload=payload, timeout=timeout, expect=expect)
+
+        monkeypatch.setattr(smoke, "_request_json", _one_slow_turn)
+        report = tmp_path / "report.json"
+
+        assert smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        ) == 0
+
+        results = json.loads(report.read_text(encoding="utf-8"))
+        per_turn = sorted(
+            t["latency_ms"] for t in results["turns"] if t["model_generated"]
+        )
+        assert len(per_turn) == total_turns
+        headline = results["measured_ms"]["full_response_ms"]
+        # The median of the turns, not their worst and not their mean.
+        assert abs(headline - statistics.median(per_turn)) <= 1
+        assert results["measured_ms"]["full_response_max_ms"] == per_turn[-1]
+        # The slow turn is still recorded -- it is just not the headline.
+        assert per_turn[-1] - headline > 100
+
+    def test_a_timeout_names_the_latency_regression_it_hides(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # Three turns at the CI ceiling plus a debrief allowed twice that very
+        # nearly fill the wall-clock budget, so the regression `budget` exists to
+        # catch exhausts the clock before the budget phase runs.  TIMEOUT is the
+        # honest class, but it has to carry the measurement or its remedy
+        # ("slower, or hung?") leaves triage to guess.
+        import time
+
+        models_dir, model_id, digest = staged_model
+        base = _fake_core(_debrief())
+
+        def _slow(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/turn"):
+                time.sleep(0.05)  # measurable, so a 0 ms ceiling is exceeded
+            if url.endswith("/debrief"):
+                time.sleep(0.6)  # outlast the budget below
+                raise smoke.SmokeFailure(
+                    smoke.FailureClass.RUNTIME, f"{url} did not answer with usable JSON"
+                )
+            return base(url, payload=payload, timeout=timeout, expect=expect)
+
+        monkeypatch.setattr(smoke, "_request_json", _slow)
+        report = tmp_path / "report.json"
+
+        # Factor 0 => a 0 ms ceiling, so the measured turns are over it.
+        exit_code = smoke.run_smoke(
+            model_id, 0.0, report, model_sha256=digest, models_dir=models_dir,
+            wall_clock_budget_s=0.5,
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.TIMEOUT]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failure_class"] == smoke.FailureClass.TIMEOUT
+        assert any(
+            "latency regression" in f and "CI ceiling" in f
+            for f in results["failures"]
+        )
+
+    def test_a_crashed_child_outranks_the_client_side_symptom(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # convsim-core dies mid-run: the client sees a pipeline-shaped error, but
+        # the real story is the crash, so the verdict must be `runtime`.
+        models_dir, model_id, digest = staged_model
+        fake_servers["core"]._returncode = 1
+
+        def _boom(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/health"):
+                return {"llm_runtime": {"runtime_id": "llama_cpp"}}
+            raise smoke.SmokeFailure(smoke.FailureClass.PIPELINE, "connection reset")
+
+        monkeypatch.setattr(smoke, "_request_json", _boom)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.RUNTIME]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failure_class"] == smoke.FailureClass.RUNTIME
+        assert "convsim-core exited with code 1" in results["failures"][0]
+        # A child really did die, so the class remedy's "read the stderr tail"
+        # is the right advice and must not be swapped for the live-server one.
+        assert results["remedy"] == smoke.REMEDIES[smoke.FailureClass.RUNTIME]
+
+    def test_a_live_server_answering_5xx_is_not_blamed_on_a_crash(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        """A `runtime` verdict with both children alive must not send triage after an OOM.
+
+        Every transport-level failure is a `runtime` failure, so a *live*
+        convsim-core that answered 504 TURN_TIMEOUT lands in the same class as a
+        crash — and the class remedy opens by telling the reader to read a child
+        stderr tail that shows a healthy server, then lists runner OOM and a bad
+        wheel as the likely causes.
+
+        The realistic cause is neither. `_start_core` raises the adapter's
+        generation budget above the latency ceiling so a slow turn is measured
+        rather than killed, but that only moves the cliff: a turn slower than
+        540 s (the budget at --ci-hardware-factor 20) is still cut off
+        adapter-side, and with 1200 s of wall clock the run still has time left,
+        so neither `timeout` nor `budget` takes the class over.
+        """
+        models_dir, model_id, digest = staged_model
+
+        def _timed_out(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/health"):
+                return {"llm_runtime": {"runtime_id": "llama_cpp"}}
+            if url.endswith("/api/sessions"):
+                return {"session_id": "sess-1"}
+            if url.endswith("/start"):
+                return _OPENING
+            raise smoke.SmokeFailure(
+                smoke.FailureClass.RUNTIME,
+                f"{url} returned HTTP 504: TURN_TIMEOUT",
+            )
+
+        monkeypatch.setattr(smoke, "_request_json", _timed_out)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.RUNTIME]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failure_class"] == smoke.FailureClass.RUNTIME
+        remedy = results["remedy"]
+        assert remedy != smoke.REMEDIES[smoke.FailureClass.RUNTIME]
+        assert "Neither child process exited" in remedy
+        assert "CONVSIM_LLAMA_CPP_CHAT_TIMEOUT" in remedy
+        # The number in the advice has to be the budget the harness actually set
+        # on the adapter, or the reader cannot tell a cut-off turn from a crash.
+        assert "540 s" in remedy
+        # And it has to send the reader at the latency measurements, which is
+        # where the cause actually shows up.
+        assert "measured_ms" in remedy and "phase_durations_s" in remedy
+
+    def test_a_port_conflict_keeps_its_own_remedy(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # The live-server remedy above must not displace this one: a port
+        # conflict starts no child at all, so "neither child process exited" is
+        # true but useless, and the advice the reader needs is the port to free.
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(
+            smoke, "_assert_port_free",
+            lambda port, label: (_ for _ in ()).throw(smoke.SmokeFailure(
+                smoke.FailureClass.RUNTIME,
+                f"Port {port} is already in use, so {label} cannot be started on it",
+                remedy="Stop whatever is listening on it, then re-run.",
+            )),
+        )
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.RUNTIME]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["remedy"] == "Stop whatever is listening on it, then re-run."
+
+    def test_a_fake_runtime_is_refused(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+
+        def _fake_runtime(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            assert url.endswith("/health"), "must not play a turn on the fake runtime"
+            return {"llm_runtime": {"runtime_id": "fake"}}
+
+        monkeypatch.setattr(smoke, "_request_json", _fake_runtime)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert "runtime_id" in results["failures"][0]
+
+    @pytest.mark.parametrize("health", [{}, {"llm_runtime": None}])
+    def test_an_unreadable_runtime_report_is_refused_not_called_a_harness_bug(
+        self, health: dict, staged_model, fake_servers,
+        monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
+    ) -> None:
+        """An absent *or null* llm_runtime must still reach the refusal verdict.
+
+        The smoke must never play a turn without having confirmed the real
+        runtime, so an unreadable /health is a refusal like any other -- and a
+        `{}` default would have let an explicit null raise AttributeError and
+        report a changed /health contract as a bug in the harness instead.
+        """
+        models_dir, model_id, digest = staged_model
+
+        def _unreadable(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            assert url.endswith("/health"), "must not play a turn on an unknown runtime"
+            return health
+
+        monkeypatch.setattr(smoke, "_request_json", _unreadable)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert "runtime_id" in results["failures"][0]
+        assert not any("bug in the smoke harness" in f for f in results["failures"])
+
+    def test_checksum_drift_stops_the_run_before_any_server_starts(
+        self, staged_model, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, _ = staged_model
+
+        def _never(*args: object, **kwargs: object):
+            raise AssertionError("must not load a model that failed verification")
+
+        monkeypatch.setattr(smoke, "_start_llama_server", _never)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256="0" * 64, models_dir=models_dir
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.CHECKSUM]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failed_phase"] == "checksum"
+
+    def test_run_without_a_checksum_warns_that_drift_was_not_checked(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, _ = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(_debrief()))
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(model_id, 20.0, report, models_dir=models_dir)
+
+        assert exit_code == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert any("checksum drift was not checked" in w for w in results["warnings"])
+
+    def test_an_absent_model_is_a_download_failure(
+        self, fake_servers, tmp_path: Path
+    ) -> None:
+        exit_code = smoke.run_smoke(
+            "absent", 20.0, None, models_dir=tmp_path / "empty"
+        )
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.DOWNLOAD]
+
+    def test_exhausted_wall_clock_budget_is_a_timeout(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(_debrief()))
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir,
+            wall_clock_budget_s=0.0,
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.TIMEOUT]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failure_class"] == smoke.FailureClass.TIMEOUT
+
+    def test_a_normal_run_still_ends_the_session_explicitly(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        fake = _fake_core(_debrief())
+        monkeypatch.setattr(smoke, "_request_json", fake)
+
+        assert smoke.run_smoke(
+            model_id, 20.0, None, model_sha256=digest, models_dir=models_dir
+        ) == 0
+        assert any(url.endswith("/end") for url in fake.calls)
+
+    def test_request_timeouts_outlast_the_adapter_so_slowness_is_not_a_crash(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # Whoever gives up first decides the verdict.  The adapter timeout
+        # handed to convsim-core has to clear the debrief (the slowest single
+        # generation), and our own POST timeout has to clear the adapter's, or
+        # a slow model is reported as a crashed server instead of a budget
+        # regression.
+        models_dir, model_id, digest = staged_model
+        adapter_timeout: dict[str, float] = {}
+        post_timeouts: list[float] = []
+
+        def _record_core(data_dir, port, llama_port, llama_timeout_s):
+            adapter_timeout["s"] = llama_timeout_s
+            return fake_servers["core"]
+
+        base = _fake_core(_debrief())
+
+        def _record_timeout(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith(("/turn", "/debrief")):
+                post_timeouts.append(timeout)
+            return base(url, payload=payload, timeout=timeout, expect=expect)
+
+        monkeypatch.setattr(smoke, "_start_core", _record_core)
+        monkeypatch.setattr(smoke, "_request_json", _record_timeout)
+
+        assert smoke.run_smoke(
+            model_id, 20.0, None, model_sha256=digest, models_dir=models_dir,
+            wall_clock_budget_s=24 * 3600,  # large, so cap() does not clamp
+        ) == 0
+
+        ci_ceiling_s = (
+            smoke.BUDGETS_MS["full_response_ms"] * 20.0 * smoke.REGRESSION_TOLERANCE
+        ) / 1000
+        assert adapter_timeout["s"] >= ci_ceiling_s * smoke.DEBRIEF_SLOWDOWN_FACTOR
+        assert post_timeouts and all(t > adapter_timeout["s"] for t in post_timeouts)
+
+    def test_a_deadline_induced_transport_error_is_a_timeout_not_a_crash(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # cap() shrinks every request timeout to the budget that is left, so the
+        # last request before the deadline dies client-side and _request_json
+        # reports it as RUNTIME ("the server never answered").  Nothing crashed
+        # — the budget ran out — and only the TIMEOUT remedy points at
+        # phase_durations_s, so the verdict has to be TIMEOUT.
+        import time
+
+        models_dir, model_id, digest = staged_model
+        base = _fake_core(_debrief())
+
+        def _overruns(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/api/sessions"):
+                time.sleep(0.3)  # outlast the budget below
+                raise smoke.SmokeFailure(
+                    smoke.FailureClass.RUNTIME,
+                    f"{url} did not answer with usable JSON after {timeout:.0f} s",
+                )
+            return base(url, payload=payload, timeout=timeout, expect=expect)
+
+        monkeypatch.setattr(smoke, "_request_json", _overruns)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir,
+            wall_clock_budget_s=0.2,
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.TIMEOUT]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["failure_class"] == smoke.FailureClass.TIMEOUT
+        assert any("Wall-clock budget" in f for f in results["failures"])
+
+    def test_a_crash_still_outranks_an_exhausted_budget(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # Both are true at once; "convsim-core exited with code 1" is the more
+        # actionable story, so RUNTIME keeps precedence over TIMEOUT.
+        import time
+
+        models_dir, model_id, digest = staged_model
+        fake_servers["core"]._returncode = 1
+        base = _fake_core(_debrief())
+
+        def _overruns(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/api/sessions"):
+                time.sleep(0.3)
+                raise smoke.SmokeFailure(smoke.FailureClass.PIPELINE, "connection reset")
+            return base(url, payload=payload, timeout=timeout, expect=expect)
+
+        monkeypatch.setattr(smoke, "_request_json", _overruns)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir,
+            wall_clock_budget_s=0.2,
+        )
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.RUNTIME]
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert "convsim-core exited with code 1" in results["failures"][0]
+
+    def test_an_unexpected_harness_bug_is_reported_not_raised(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+
+        def _bug(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/health"):
+                return {"llm_runtime": {"runtime_id": "llama_cpp"}}
+            raise TypeError("harness bug")
+
+        monkeypatch.setattr(smoke, "_request_json", _bug)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code != 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert "bug in the smoke harness" in results["failures"][0]
+        # It borrows `pipeline`'s exit code but not its cause, so the remedy
+        # must point at the traceback, not at the model's turn output.
+        assert results["remedy"] != smoke.REMEDIES[smoke.FailureClass.PIPELINE]
+        assert "harness bug" in results["remedy"]
+        assert results["remedy"] in smoke.render_step_summary(results)
+
+    def test_a_scenario_that_ends_early_stops_the_scripted_turns(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        base = _fake_core(_debrief(), debug_turns=[
+            {"turn_number": 1, "used_fallback": False, "used_native_structured_output": True},
+        ])
+
+        def _ends_after_one(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/turn"):
+                return {**_NPC_TURN, "ending_type": "success"}
+            return base(url, payload=payload, timeout=timeout, expect=expect)
+
+        monkeypatch.setattr(smoke, "_request_json", _ends_after_one)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert len(results["turns"]) == 2  # opening + the one turn that ended it
+        # The turn pipeline already moved the session to 'Ended', so POST /end
+        # would answer 409 and the run would die as a bogus `pipeline` failure.
+        assert not any(url.endswith("/end") for url in base.calls), (
+            "must not POST /end to a session the scenario already ended"
+        )
+        assert results["debrief"] is not None
+        # Green, but on one turn instead of three. The job exists to play a
+        # multi-turn conversation, so a short run must not read like a full one.
+        assert results["scripted_turns_played"] == 1
+        assert any(
+            f"Only 1 of {len(smoke.SCRIPTED_PLAYER_TURNS)} scripted player turns ran" in w
+            for w in results["warnings"]
+        ), results["warnings"]
+
+    def test_a_full_playthrough_does_not_warn_about_a_short_run(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(_debrief()))
+        report = tmp_path / "report.json"
+
+        assert smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        ) == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["scripted_turns_played"] == len(smoke.SCRIPTED_PLAYER_TURNS)
+        assert results["warnings"] == []
+
+    def test_unavailable_debug_flags_warn_instead_of_silently_passing(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # The debug endpoint is a diagnostic, so losing it must not turn a
+        # healthy run red — but it is also the only thing that tells a real NPC
+        # turn from the canned fallback, so the report has to say the check
+        # did not run.
+        models_dir, model_id, digest = staged_model
+        base = _fake_core(_debrief())
+
+        def _no_debug(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/debug"):
+                raise smoke.SmokeFailure(smoke.FailureClass.PIPELINE, "HTTP 404")
+            return base(url, payload=payload, timeout=timeout, expect=expect)
+
+        monkeypatch.setattr(smoke, "_request_json", _no_debug)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert any("fallback check did not run" in w for w in results["warnings"])
+
+    @pytest.mark.parametrize(
+        "debug_body",
+        [
+            {},                                      # no turns key at all
+            {"turns": None},                         # turns present but not a list
+            {"turns": [{"used_fallback": False}]},   # entry that cannot be keyed
+            {"turns": ["not-a-dict"]},
+        ],
+        ids=["no-turns-key", "turns-not-a-list", "entry-without-turn-number", "entry-not-a-dict"],
+    )
+    def test_a_malformed_debug_payload_does_not_fail_an_otherwise_green_run(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch,
+        tmp_path: Path, debug_body: dict,
+    ) -> None:
+        # Same contract as a 404 from the debug endpoint: it is a diagnostic,
+        # so a shape we cannot read must degrade to "the check did not run",
+        # not raise out of the helper and land in run_smoke's catch-all as a
+        # harness bug (exit 5) on a run where the product did nothing wrong.
+        models_dir, model_id, digest = staged_model
+        base = _fake_core(_debrief())
+
+        def _bad_debug(url: str, *, payload=None, timeout=None, expect=(200, 201)) -> dict:
+            if url.endswith("/debug"):
+                return debug_body
+            return base(url, payload=payload, timeout=timeout, expect=expect)
+
+        monkeypatch.setattr(smoke, "_request_json", _bad_debug)
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        assert results["verdict"] == "pass"
+        assert any("fallback check did not run" in w for w in results["warnings"])
+
+    def test_a_debug_payload_that_keys_only_some_turns_says_which_it_skipped(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # A partial payload is the same hole as a missing one, on the turns it
+        # missed — and here it is load-bearing: turn 1 having no flag is the
+        # only reason "every generated turn fell back" does not fire, so the
+        # run stays green on a conversation that may have had no real reply in
+        # it. The warning has to name the turn the check skipped rather than
+        # leave the reader with "2 of 3 fell back" and no hint about the third.
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(
+            _debrief(),
+            debug_turns=[
+                {"turn_number": 2, "used_fallback": True},
+                {"turn_number": 3, "used_fallback": True},
+                {"used_fallback": False},  # turn 1: unkeyable, so dropped
+            ],
+        ))
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        warning = next(
+            w for w in results["warnings"] if "fallback check did not run" in w
+        )
+        assert "1 of 3" in warning
+        assert "player_turn_1" in warning
+
+    def test_a_debug_entry_without_the_flags_is_not_read_as_a_passed_check(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # The other way a /debug payload can leave the fallback check unrun: an
+        # entry that *is* keyed by turn number but carries no used_fallback.
+        # Reading that as False would credit every turn as a real reply on no
+        # evidence — and because evaluate_turns decides whether the check ran
+        # from the key's presence, writing it would also suppress the warning
+        # that says so, which is strictly worse than the unkeyable-entry case
+        # above. The turn must come back unflagged, and the run must say it is
+        # unproven.
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(
+            _debrief(),
+            debug_turns=[{"turn_number": n} for n in range(
+                1, len(smoke.SCRIPTED_PLAYER_TURNS) + 1
+            )],
+        ))
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        generated = [t for t in results["turns"] if t["model_generated"]]
+        assert generated
+        for turn in generated:
+            assert "used_fallback" not in turn, turn
+            assert "used_native_structured_output" not in turn, turn
+        assert any("fallback check did not run" in w for w in results["warnings"])
+
+    def test_a_debug_entry_with_only_some_flags_keeps_the_one_it_carries(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # Per flag, not per entry: used_fallback is the one the verdict turns
+        # on, so an entry carrying it must still prove the turn even when the
+        # structured-output flag beside it is gone.
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(
+            _debrief(),
+            debug_turns=[
+                {"turn_number": n, "used_fallback": False}
+                for n in range(1, len(smoke.SCRIPTED_PLAYER_TURNS) + 1)
+            ],
+        ))
+        report = tmp_path / "report.json"
+
+        exit_code = smoke.run_smoke(
+            model_id, 20.0, report, model_sha256=digest, models_dir=models_dir
+        )
+
+        assert exit_code == 0
+        results = json.loads(report.read_text(encoding="utf-8"))
+        generated = [t for t in results["turns"] if t["model_generated"]]
+        for turn in generated:
+            assert turn["used_fallback"] is False
+            assert "used_native_structured_output" not in turn, turn
+        assert not any("fallback check did not run" in w for w in results["warnings"])
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+class TestMainEntryPoint:
+    """exit 1 means `budget` and nothing else, so nothing may escape unclassified."""
+
+    def test_registry_lookup_writes_github_output_and_exits_zero(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        out = tmp_path / "gh-output"
+        monkeypatch.setenv("GITHUB_OUTPUT", str(out))
+        assert smoke.main(["--print-registry-model", "starter"]) == 0
+        written = dict(
+            line.split("=", 1) for line in out.read_text(encoding="utf-8").splitlines()
+        )
+        assert len(written["model_sha256"]) == 64
+        assert written["model_url"].startswith("https://")
+
+    def test_a_malformed_registry_is_classified_not_a_bare_traceback(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # yaml raises ParserError, not SmokeFailure.  Escaping exits 1, which
+        # EXIT_CODES reads as `budget` — "the product is fine, just slow" — and
+        # writes no banner or step summary at all.
+        registry = tmp_path / "registry.yaml"
+        registry.write_text("models: [ unterminated\n", encoding="utf-8")
+        # Point the real resolver at the malformed file (its registry_path
+        # default is bound at def time, so the module constant cannot be
+        # patched), so the failure under test is yaml's, not the patch's.
+        real_resolve = smoke.resolve_registry_model
+        monkeypatch.setattr(
+            smoke, "resolve_registry_model",
+            lambda role, registry_path=None: real_resolve(role, registry),
+        )
+
+        summary = tmp_path / "summary.md"
+        monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+
+        exit_code = smoke.main(["--print-registry-model", "starter"])
+
+        assert exit_code != smoke.EXIT_CODES[smoke.FailureClass.BUDGET]
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        assert "FAIL" in summary.read_text(encoding="utf-8")
+
+    def test_a_registry_missing_the_role_prints_registry_advice(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # The step summary is the whole triage surface for a red nightly, so
+        # the advice in it has to match the failure: `pipeline`'s own remedy
+        # ("inspect the per-turn used_fallback flags") describes a run that
+        # never happened here.
+        registry = tmp_path / "registry.yaml"
+        registry.write_text("models: []\n", encoding="utf-8")
+        real_resolve = smoke.resolve_registry_model
+        monkeypatch.setattr(
+            smoke, "resolve_registry_model",
+            lambda role, registry_path=None: real_resolve(role, registry),
+        )
+        summary = tmp_path / "summary.md"
+        monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+
+        exit_code = smoke.main(["--print-registry-model", "starter"])
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        text = summary.read_text(encoding="utf-8")
+        assert "registry.yaml" in text
+        assert "used_fallback" not in text
+
+    def test_a_download_failure_keeps_its_own_exit_code_and_summary(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        summary = tmp_path / "summary.md"
+        monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+        # --verify-only against an empty directory: no file to hash.
+        exit_code = smoke.main([
+            "--verify-only", "--model-id", "absent",
+            "--model-sha256", "0" * 64, "--models-dir", str(tmp_path / "empty"),
+        ])
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.DOWNLOAD]
+        assert "`download`" in summary.read_text(encoding="utf-8")
+
+    def test_a_failure_in_run_smokes_prologue_is_classified_not_exit_one(
+        self, staged_model, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # run_smoke classifies everything inside its own try/finally, but the
+        # throwaway data directory is created before it and the report is
+        # written after it.  A full disk there — realistic right after a 2.5 GB
+        # download and a 2.5 GB cache save — must not escape as a bare
+        # traceback: Python would exit 1, which EXIT_CODES reads as `budget`.
+        models_dir, model_id, digest = staged_model
+
+        def _no_space(*args: object, **kwargs: object) -> str:
+            raise OSError(28, "No space left on device")
+
+        monkeypatch.setattr(smoke.tempfile, "mkdtemp", _no_space)
+        summary = tmp_path / "summary.md"
+        monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+
+        exit_code = smoke.main([
+            "--model-id", model_id, "--model-sha256", digest,
+            "--models-dir", str(models_dir),
+        ])
+
+        assert exit_code != smoke.EXIT_CODES[smoke.FailureClass.BUDGET]
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        text = summary.read_text(encoding="utf-8")
+        assert "FAIL" in text
+        assert "No space left on device" in text
+
+    def test_a_failure_writing_the_report_does_not_become_a_budget_verdict(
+        self, staged_model, fake_servers, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+    ) -> None:
+        # The report write follows the verdict, outside run_smoke's handlers.
+        models_dir, model_id, digest = staged_model
+        monkeypatch.setattr(smoke, "_request_json", _fake_core(_debrief()))
+
+        def _no_space(*args: object, **kwargs: object) -> None:
+            raise OSError(28, "No space left on device")
+
+        monkeypatch.setattr(smoke, "_write_report", _no_space)
+
+        exit_code = smoke.main([
+            "--model-id", model_id, "--model-sha256", digest,
+            "--models-dir", str(models_dir), "--report-path", str(tmp_path / "r.json"),
+        ])
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+
+    def test_a_usage_error_is_not_mistaken_for_a_download_failure(self) -> None:
+        # argparse exits 2 on a bad command line, and 2 is the `download` code,
+        # whose remedy is "nothing about the app changed, re-run the job" —
+        # advice that loops forever on a job that cannot succeed. A malformed
+        # invocation is a harness problem: `pipeline`, like every other surprise.
+        with pytest.raises(SystemExit) as exc_info:
+            smoke.main(["--download-only", "--model-id", "x"])  # no url / sha256
+        assert exc_info.value.code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+        assert exc_info.value.code != smoke.EXIT_CODES[smoke.FailureClass.DOWNLOAD]
+
+    def test_a_missing_model_id_is_a_classified_usage_error(self) -> None:
+        with pytest.raises(SystemExit) as exc_info:
+            smoke.main([])
+        assert exc_info.value.code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+
+    def test_verify_only_without_a_digest_is_a_classified_usage_error(self) -> None:
+        with pytest.raises(SystemExit) as exc_info:
+            smoke.main(["--verify-only", "--model-id", "m"])
+        assert exc_info.value.code == smoke.EXIT_CODES[smoke.FailureClass.PIPELINE]
+
+    def test_download_only_verifies_what_it_fetched_and_exits_zero(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # The workflow's own download step, and step 3 of the documented local
+        # repro. Only its failure modes were covered above, so a typo in the
+        # success path would break the nightly before the smoke ever ran.
+        models_dir = tmp_path / "models"
+        payload = b"\x00gguf stand-in"
+
+        def _fake_fetch(url: str, dest: Path, *, budget_s: float = 0.0) -> None:
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            dest.write_bytes(payload)
+
+        monkeypatch.setattr(smoke, "_download_with_progress", _fake_fetch)
+
+        exit_code = smoke.main([
+            "--download-only", "--model-id", "m",
+            "--model-url", "https://example.invalid/m.gguf",
+            "--model-sha256", hashlib.sha256(payload).hexdigest(),
+            "--models-dir", str(models_dir),
+        ])
+
+        assert exit_code == 0
+        assert (models_dir / "m.gguf").read_bytes() == payload
+
+    def test_verify_only_accepts_a_matching_file_and_exits_zero(
+        self, staged_model
+    ) -> None:
+        models_dir, model_id, digest = staged_model
+        exit_code = smoke.main([
+            "--verify-only", "--model-id", model_id,
+            "--model-sha256", digest, "--models-dir", str(models_dir),
+        ])
+        assert exit_code == 0
+        assert (models_dir / f"{model_id}.gguf").exists()
+
+    def test_verify_only_on_drift_exits_three_and_deletes_the_file(
+        self, staged_model, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # The documented warning about --verify-only: a mismatch deletes the
+        # file so a re-run re-downloads instead of re-verifying the same bytes.
+        models_dir, model_id, _ = staged_model
+        summary = tmp_path / "summary.md"
+        monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+
+        exit_code = smoke.main([
+            "--verify-only", "--model-id", model_id,
+            "--model-sha256", "a" * 64, "--models-dir", str(models_dir),
+        ])
+
+        assert exit_code == smoke.EXIT_CODES[smoke.FailureClass.CHECKSUM]
+        assert not (models_dir / f"{model_id}.gguf").exists()
+        assert "`checksum`" in summary.read_text(encoding="utf-8")
+
+
+# ---------------------------------------------------------------------------
+# Child process wiring
+# ---------------------------------------------------------------------------
+
+
+def _annotated_settings_fields(path: Path, class_name: str) -> set:
+    """Field names declared on a pydantic-settings class in ``path``."""
+    import re
+
+    src = path.read_text(encoding="utf-8")
+    body = src[src.index(f"class {class_name}("):]
+    end = re.search(r"\n(?:@|class |def )", body[1:])
+    if end:
+        body = body[: end.start() + 1]
+    return set(re.findall(r"^    ([a-z_][a-z0-9_]*)\s*:", body, re.M))
+
+
+class TestChildProcessWiring:
+    """The argv and the env that make the smoke a *real-model* smoke.
+
+    Every orchestration test above fakes both children out, so the code that
+    builds this argv and this env is the one part of the harness no test runs —
+    and two of the values in it fail silently:
+
+    * ``CONVSIM_RUNTIME_ID`` is the only thing selecting the real runtime over
+      the deterministic fake default. The run does check ``/health`` afterwards,
+      so this one at least fails loudly.
+    * ``CONVSIM_LLAMA_CPP_CHAT_TIMEOUT`` does not. Misspell it and the adapter
+      silently keeps its 180 s default, so the first turn slower than that is
+      killed adapter-side and convsim-core answers 500 — class ``runtime``, "the
+      server crashed" — which is precisely the misclassification the override
+      exists to prevent, on a night when the only thing wrong was slow
+      inference.
+
+    Pydantic reads these from a prefix plus a field name, so a name that is not
+    a declared setting is silently ignored rather than rejected. The last test
+    therefore checks every override the harness sets against the product's own
+    settings classes.
+    """
+
+    _CORE_SRC = REPO_ROOT / "services" / "convsim-core" / "convsim_core"
+
+    @pytest.fixture()
+    def spawned(self, monkeypatch: pytest.MonkeyPatch) -> dict:
+        captured: dict = {}
+
+        def _popen(argv, **kwargs):
+            captured["argv"] = argv
+            captured["env"] = kwargs.get("env")
+            return _FakeProc()
+
+        monkeypatch.setattr(smoke.subprocess, "Popen", _popen)
+        return captured
+
+    def test_llama_server_is_started_on_the_verified_model_and_port(
+        self, spawned: dict, tmp_path: Path
+    ) -> None:
+        model = tmp_path / "m.gguf"
+        smoke._start_llama_server(model, 7356)
+        argv = spawned["argv"]
+        assert argv[1:3] == ["-m", "llama_cpp.server"]
+        assert argv[argv.index("--model") + 1] == str(model)
+        assert argv[argv.index("--port") + 1] == "7356"
+        assert argv[argv.index("--host") + 1] == "127.0.0.1"
+
+    def test_core_is_wired_to_the_real_runtime_not_the_fake_default(
+        self, spawned: dict, tmp_path: Path
+    ) -> None:
+        smoke._start_core(tmp_path / "data", 7399, 7356, 540.0)
+        env = spawned["env"]
+        assert env["CONVSIM_RUNTIME_ID"] == "llama_cpp"
+        assert env["CONVSIM_LLAMA_CPP_BASE_URL"] == "http://127.0.0.1:7356"
+
+    def test_the_adapter_outlasts_the_latency_the_run_means_to_measure(
+        self, spawned: dict, tmp_path: Path
+    ) -> None:
+        smoke._start_core(tmp_path / "data", 7399, 7356, 540.0)
+        env = spawned["env"]
+        # Both, not just the chat budget: TIMEOUT covers the model listing a
+        # cold runner does first, and a listing cut short is also a 500.
+        assert env["CONVSIM_LLAMA_CPP_CHAT_TIMEOUT"] == "540"
+        assert env["CONVSIM_LLAMA_CPP_TIMEOUT"] == "540"
+
+    def test_core_is_confined_to_the_throwaway_data_directory(
+        self, spawned: dict, tmp_path: Path
+    ) -> None:
+        # docs/real-model-smoke.md's privacy claim: the run leaves no database,
+        # log or pack behind outside a directory the finally block deletes.
+        data_dir = tmp_path / "data"
+        smoke._start_core(data_dir, 7399, 7356, 540.0)
+        env = spawned["env"]
+        for key in ("CONVSIM_DATA_DIR", "CONVSIM_LOG_DIR",
+                    "CONVSIM_DB_DIR", "CONVSIM_PACKS_DIR"):
+            assert env[key].startswith(str(data_dir)), key
+
+    def test_every_override_names_a_setting_the_product_actually_reads(
+        self, spawned: dict, tmp_path: Path
+    ) -> None:
+        import os
+
+        smoke._start_core(tmp_path / "data", 7399, 7356, 540.0)
+        env = spawned["env"]
+        overrides = sorted(
+            k for k, v in env.items()
+            if k.startswith("CONVSIM_") and os.environ.get(k) != v
+        )
+        assert overrides, "the env the harness builds sets no CONVSIM_* override"
+
+        pools = {
+            "CONVSIM_LLAMA_CPP_": (
+                "LlamaCppConfig",
+                _annotated_settings_fields(
+                    self._CORE_SRC / "runtime" / "llama_cpp.py", "LlamaCppConfig"
+                ),
+            ),
+            "CONVSIM_": (
+                "ServiceConfig",
+                _annotated_settings_fields(self._CORE_SRC / "config.py", "ServiceConfig"),
+            ),
+        }
+        for name in overrides:
+            prefix = "CONVSIM_LLAMA_CPP_" if name.startswith("CONVSIM_LLAMA_CPP_") else "CONVSIM_"
+            class_name, fields = pools[prefix]
+            field = name[len(prefix):].lower()
+            assert field in fields, (
+                f"{name} is not a {class_name} setting, so pydantic ignores it "
+                "and the harness silently runs on the product's default"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Child stderr tails
+# ---------------------------------------------------------------------------
+
+
+class _LiveTail:
+    """A tail that raises when iterated, the way a deque does mid-append.
+
+    ``_dump_stderr_tails`` runs from ``run_smoke``'s except handler, before the
+    finally block stops either child, so the draining thread is still appending.
+    Iterating the live buffer therefore has to be impossible, not merely
+    unlikely -- a ``RuntimeError`` from there escapes the handler and discards
+    the classified verdict.
+    """
+
+    def __init__(self, *lines: str) -> None:
+        self._lines = list(lines)
+
+    def __iter__(self):
+        raise RuntimeError("deque mutated during iteration")
+
+    def __len__(self) -> int:
+        raise RuntimeError("deque mutated during iteration")
+
+    def snapshot(self) -> list:
+        return list(self._lines)
+
+
+class TestStderrTails:
+    """The dump must not iterate a buffer another thread is writing to."""
+
+    def test_the_dump_reads_a_snapshot_not_the_live_buffer(
+        self, capsys: pytest.CaptureFixture
+    ) -> None:
+        smoke._dump_stderr_tails((("convsim-core", _LiveTail("boom", "trace")),))
+        err = capsys.readouterr().err
+        assert "last 2 lines" in err
+        assert "boom" in err and "trace" in err
+
+    def test_an_empty_tail_prints_no_header(self, capsys: pytest.CaptureFixture) -> None:
+        smoke._dump_stderr_tails((("llama-server", smoke.StderrTail()),))
+        assert capsys.readouterr().err == ""
+
+    def test_the_tail_keeps_only_the_most_recent_lines(self) -> None:
+        tail = smoke.StderrTail(maxlen=3)
+        for i in range(10):
+            tail.append(f"line {i}")
+        assert tail.snapshot() == ["line 7", "line 8", "line 9"]
+
+    def test_a_snapshot_does_not_alias_the_buffer(self) -> None:
+        tail = smoke.StderrTail()
+        tail.append("first")
+        snapshot = tail.snapshot()
+        tail.append("second")
+        assert snapshot == ["first"]
+
+    def test_draining_a_byte_stream_decodes_and_strips(self) -> None:
+        tail = smoke.StderrTail()
+        smoke._drain(iter([b"warning: slow\n", b"\xff bad utf-8\n"]), tail)
+        assert tail.snapshot() == ["warning: slow", "� bad utf-8"]
+
+    def test_a_stream_that_breaks_mid_read_does_not_kill_the_drain_thread(self) -> None:
+        # The drain runs in a daemon thread with nothing to catch for it, and a
+        # pipe closed under it while the child is being terminated is routine.
+        def _broken():
+            yield b"last words\n"
+            raise ValueError("pipe closed")
+
+        tail = smoke.StderrTail()
+        smoke._drain(_broken(), tail)
+        assert tail.snapshot() == ["last words"]
+
+
+# ---------------------------------------------------------------------------
+# Log ordering
+# ---------------------------------------------------------------------------
+
+
+class TestLogOrdering:
+    """The CI log has to read in the order things happened.
+
+    CI captures the harness through a pipe, so Python block-buffers stdout
+    while stderr stays unbuffered. Unfixed, that means the phase progress and
+    per-turn latencies (stdout) arrive after the failure banner and child
+    stderr tails (stderr) — and during the ~11 min the conversation and
+    debrief phases take, nothing streams at all.
+    """
+
+    def test_progress_is_logged_before_the_banner_it_explains(
+        self, tmp_path: Path
+    ) -> None:
+        import subprocess
+        import sys
+
+        model = tmp_path / "drifted.gguf"
+        model.write_bytes(b"not the pinned bytes")
+
+        # Piped, merged, and run with the inherited environment so that only
+        # the harness's own buffering decides the order.
+        proc = subprocess.run(
+            [
+                # The interpreter running the tests, not whatever "python3"
+                # resolves to on PATH: the harness imports PyYAML, which is
+                # only guaranteed in this environment.
+                sys.executable, str(SCRIPT_PATH), "--verify-only",
+                "--model-id", "drifted", "--model-sha256", "0" * 64,
+                "--models-dir", str(tmp_path),
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            timeout=120,
+        )
+
+        assert proc.returncode == smoke.EXIT_CODES[smoke.FailureClass.CHECKSUM]
+        log = proc.stdout
+        progress_at = log.find("Verifying cached model")
+        banner_at = log.find("FAILED — class: CHECKSUM")
+        assert progress_at != -1, f"progress line missing from:\n{log}"
+        assert banner_at != -1, f"banner missing from:\n{log}"
+        assert progress_at < banner_at, (
+            "the stdout progress line arrived after the stderr banner, so the "
+            f"CI log reads out of order:\n{log}"
+        )
+
+    def test_reconfiguring_a_stream_that_cannot_be_reconfigured_is_survivable(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # Under pytest's capture, and under contextlib.redirect_stdout, stdout
+        # is not a TextIOWrapper. Buffering is then the caller's business — but
+        # it must not take the harness down on the way past.
+        monkeypatch.setattr("sys.stdout", io.StringIO())
+        monkeypatch.setattr("sys.stderr", io.StringIO())
+        smoke._use_line_buffered_output()