From 6ed0e9156e8d5ccb340f30f48f90632176653a4f Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 16:50:41 -0800
Subject: [PATCH 01/76] feat(evals): bind held-out qualification to reviewed
 evidence

---
 evals/intent_engine/BUILD.bazel           |  16 ++
 evals/intent_engine/HTTP_RUNNER.md        |  53 +++++
 evals/intent_engine/qualification.py      | 182 ++++++++++++++++
 evals/intent_engine/run_http.py           |  12 ++
 evals/intent_engine/score.py              |  14 ++
 evals/intent_engine/test_qualification.py | 242 ++++++++++++++++++++++
 6 files changed, 519 insertions(+)
 create mode 100644 evals/intent_engine/qualification.py
 create mode 100644 evals/intent_engine/test_qualification.py

diff --git a/evals/intent_engine/BUILD.bazel b/evals/intent_engine/BUILD.bazel
index c1b8a4683ca..0fb8fe0cc4e 100644
--- a/evals/intent_engine/BUILD.bazel
+++ b/evals/intent_engine/BUILD.bazel
@@ -5,6 +5,7 @@ py_test(
     size = "small",
     srcs = [
         "common.py",
+        "qualification.py",
         "run_http.py",
         "score.py",
         "test_run_http.py",
@@ -13,11 +14,26 @@ py_test(
     main = "test_run_http.py",
 )
 
+py_test(
+    name = "qualification_test",
+    size = "small",
+    srcs = [
+        "common.py",
+        "qualification.py",
+        "run_http.py",
+        "score.py",
+        "test_qualification.py",
+    ],
+    imports = ["."],
+    main = "test_qualification.py",
+)
+
 py_test(
     name = "scoring_test",
     size = "small",
     srcs = [
         "common.py",
+        "qualification.py",
         "compare.py",
         "score.py",
         "test_score.py",
diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index e8f355dab66..600cd944a1b 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -106,3 +106,56 @@ bazel test //evals/intent_engine:http_runner_test //evals/intent_engine:scoring_
 The tests exercise the actual runner and filesystem with substituted HTTP
 responses. They do not download models, spend inference tokens, or qualify a
 model's quality or performance.
+
+## Frozen held-out qualification (additional, not runtime parity)
+
+The historical nine-category evaluation is **not** a qualification of the ten
+current Scout intent questions. The optional qualification mode makes the
+existing HTTP runner/scorer fail closed on case-group leakage, unreviewed labels,
+changed question instructions, post-hoc thresholds and incomplete holdout
+metrics; it cannot reconstruct a canonical Scout transcript or attest a model.
+
+Create a separate JSON annotation manifest **before running inference** with
+`version: 1`, `application`, the SHA-256 of the original dataset file
+(`dataset_sha256`), and `questions_sha256` equal to the canonical
+`qualification.sha256_json(common.noul_questions())` of the runner's exact
+question dictionary. This last digest must change with question text. Include
+`thresholds` with one frozen probability threshold per requested question and
+`error_tolerances` with all three numbers:
+`max_none_false_hint_rate` (0–1), `min_positive_case_recall` (0–1) and
+`max_p95_ms` (nonnegative milliseconds).
+
+The `annotations` array must cover every case in the original dataset, in
+that dataset's order. Each annotation has `id`, `split` (`tune` or
+`holdout`), `project_id`, `session_id`, `author`, a distinct independent
+`reviewer`, `review_ref`, `verified_labels` and Boolean
+`verified_context_needed`. The independently reviewed labels must exactly
+match the inference dataset. Tune and held-out projects cannot overlap; session
+groups cannot overlap. The held-out group must contain both a positive and a
+negative example. Reviewer identifiers and references are **declarations**,
+not cryptographic verification of independent label quality. Do not forge them
+from model predictions or adoption events.
+
+Run with the pinned remote serving declaration as before, adding
+`--qualification-manifest /path/to/frozen.json`. The runner refuses an
+additional `--subset`, validates every case and its group before network
+traffic, infers **only the frozen held-out cases**, and binds the raw manifest's
+SHA-256 to the experiment identity and resumable partial log. A changed
+manifest cannot resume the old run. The scorer accepts exactly one result and
+the same manifest:
+
+```sh
+python evals/intent_engine/score.py --qualified /path/to/frozen.json \
+  evals/intent_engine/results/pinned-run_B_noul_text.json
+bazel test //evals/intent_engine:qualification_test --test_output=errors
+```
+
+This writes `evals/intent_engine/qualification_summary.json` separately from
+the historical `summary.json`, and reports only **frozen-threshold** held-out
+false hints, positive-case recall, p50/p95 measured end-to-end HTTP latency and
+the predeclared tolerances. It deliberately does not publish optimized
+in-sample thresholds as qualification. Neither correct answers nor those metrics
+alone demonstrate task success, ignored/acted outcomes, cold/warm cost, native
+runtime renderer/settings parity or a real Kev checkpoint. All those require
+canonical transcript export, independent review, real host inference and
+application-specific trials before any replacement or merge decision.
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
new file mode 100644
index 00000000000..73be56d1a0d
--- /dev/null
+++ b/evals/intent_engine/qualification.py
@@ -0,0 +1,182 @@
+"""Frozen, independently reviewed held-out contracts for the existing HTTP evaluator.
+
+This validates the *declared* reviewer and split provenance, not the honesty of
+reviewers or the identity of a remote model. Historical runs remain unqualified.
+"""
+import hashlib
+import json
+import math
+import re
+
+_DIGEST = re.compile(r"[0-9a-f]{64}\Z")
+
+
+def sha256_bytes(data):
+    return hashlib.sha256(data).hexdigest()
+
+
+def sha256_json(value):
+    return sha256_bytes(json.dumps(value, sort_keys=True, allow_nan=False).encode("utf-8"))
+
+
+def load_manifest(path):
+    with open(path, "rb") as source:
+        raw = source.read()
+    return json.loads(raw), sha256_bytes(raw)
+
+
+def validate_manifest(manifest, dataset_sha, questions, cases=None):
+    """Fail closed before an HTTP request or a qualified result is published.
+
+    `cases` is the complete original dataset, not a filtered subset. Scoring
+    passes None because the experiment already binds the dataset's byte digest.
+    """
+    if not isinstance(manifest, dict) or manifest.get("version") != 1:
+        raise ValueError("unsupported qualification manifest version")
+    if not isinstance(dataset_sha, str) or not _DIGEST.fullmatch(dataset_sha):
+        raise ValueError("invalid qualification dataset digest")
+    if manifest.get("dataset_sha256") != dataset_sha:
+        raise ValueError("qualification dataset digest mismatch")
+    if manifest.get("questions_sha256") != sha256_json(questions):
+        raise ValueError("qualification question vocabulary or instruction mismatch")
+
+    thresholds = manifest.get("thresholds")
+    if not isinstance(thresholds, dict) or set(thresholds) != set(questions):
+        raise ValueError("qualification thresholds must cover every requested question")
+    for value in thresholds.values():
+        if (isinstance(value, bool) or not isinstance(value, (int, float))
+                or not math.isfinite(value) or not 0 <= value <= 1):
+            raise ValueError("qualification thresholds must be finite probabilities")
+
+    tolerances = manifest.get("error_tolerances")
+    required = ("max_none_false_hint_rate", "min_positive_case_recall", "max_p95_ms")
+    if not isinstance(tolerances, dict) or set(tolerances) != set(required):
+        raise ValueError("qualification tolerances must be frozen before inference")
+    for key in required:
+        value = tolerances[key]
+        if (isinstance(value, bool) or not isinstance(value, (int, float))
+                or not math.isfinite(value) or value < 0
+                or (key != "max_p95_ms" and value > 1)):
+            raise ValueError(f"invalid qualification tolerance {key}")
+
+    application = manifest.get("application")
+    if not isinstance(application, str) or not application.strip():
+        raise ValueError("qualification requires an application")
+    annotations = manifest.get("annotations")
+    if not isinstance(annotations, list) or not annotations:
+        raise ValueError("qualification requires reviewer-verified annotations")
+    seen = set()
+    projects = {"tune": set(), "holdout": set()}
+    sessions = {"tune": set(), "holdout": set()}
+    heldout = []
+    tune = []
+    negative = positive = 0
+    case_map = None
+    if cases is not None:
+        if not isinstance(cases, list) or not all(isinstance(c, dict) for c in cases):
+            raise ValueError("qualification dataset must contain cases")
+        case_map = {c.get("id"): c for c in cases}
+        if len(case_map) != len(cases):
+            raise ValueError("qualification dataset has duplicate case IDs")
+    for annotation in annotations:
+        if not isinstance(annotation, dict):
+            raise ValueError("qualification annotation must be an object")
+        case_id = annotation.get("id")
+        if not isinstance(case_id, str) or not case_id or case_id in seen:
+            raise ValueError("duplicate or missing qualification case ID")
+        seen.add(case_id)
+        split = annotation.get("split")
+        if split not in projects:
+            raise ValueError("qualification split must be tune or holdout")
+        for key in ("project_id", "session_id", "author", "reviewer", "review_ref"):
+            value = annotation.get(key)
+            if not isinstance(value, str) or not value.strip():
+                raise ValueError(f"qualification {case_id} lacks {key}")
+        if annotation["author"] == annotation["reviewer"]:
+            raise ValueError(f"qualification {case_id} is self-labelled")
+        labels = annotation.get("verified_labels")
+        if (not isinstance(labels, list)
+                or any(not isinstance(label, str) or label not in questions for label in labels)
+                or len(set(labels)) != len(labels)):
+            raise ValueError(f"qualification {case_id} has invalid verified labels")
+        if not isinstance(annotation.get("verified_context_needed"), bool):
+            raise ValueError(f"qualification {case_id} lacks reviewed context-needed label")
+        if case_map is not None:
+            recorded = case_map.get(case_id)
+            if (recorded is None or recorded.get("labels") != labels
+                    or recorded.get("context_needed") != annotation["verified_context_needed"]):
+                raise ValueError(f"qualification {case_id} disagrees with the dataset")
+        projects[split].add(annotation["project_id"])
+        sessions[split].add((annotation["project_id"], annotation["session_id"]))
+        if split == "holdout":
+            heldout.append(case_id)
+            positive += bool(labels)
+            negative += not labels
+        else:
+            tune.append(case_id)
+    if case_map is not None:
+        if seen != set(case_map):
+            raise ValueError("qualification must account for every dataset case")
+        if [a["id"] for a in annotations] != [c["id"] for c in cases]:
+            raise ValueError("qualification case ordering differs from the frozen dataset")
+    if not tune or not heldout or not positive or not negative:
+        raise ValueError("qualification needs tune and positive/negative holdout cases")
+    if projects["tune"] & projects["holdout"] or sessions["tune"] & sessions["holdout"]:
+        raise ValueError("qualification leaks project/session groups between splits")
+    return {
+        "heldout_ids": heldout,
+        "thresholds": thresholds,
+        "error_tolerances": tolerances,
+        "annotations": {a["id"]: a for a in annotations},
+        "application": application,
+    }
+
+
+def qualified_report(result, manifest, manifest_sha, questions):
+    """Use the existing scorer, but publish only pre-frozen held-out metrics."""
+    from score import score_config
+
+    experiment = result.get("experiment")
+    if not isinstance(experiment, dict):
+        raise ValueError("qualification requires an identity-bound HTTP experiment")
+    if experiment.get("qualification_manifest_sha256") != manifest_sha:
+        raise ValueError("qualification experiment is not bound to this frozen manifest")
+    validated = validate_manifest(manifest, result.get("dataset_sha256"), questions)
+    rows = result.get("rows")
+    if not isinstance(rows, list) or [r.get("id") for r in rows if isinstance(r, dict)] != validated["heldout_ids"]:
+        raise ValueError("qualification result differs from frozen held-out case order")
+    for row in rows:
+        annotation = validated["annotations"][row["id"]]
+        if (row.get("labels") != annotation["verified_labels"]
+                or row.get("context_needed") != annotation["verified_context_needed"]):
+            raise ValueError(f"qualification result {row['id']} does not match reviewed labels")
+    scores = score_config(result, fixed_t=validated["thresholds"])
+    fixed = scores["fixed"]["all"]
+    latency = scores["latency"]
+    tolerances = validated["error_tolerances"]
+    rates = {
+        "none_false_hint_rate": fixed["none_false_hint_rate"],
+        "positive_case_recall": fixed["pos_cases_with_a_correct_hint"],
+        "p95_ms": latency["p95_ms"],
+    }
+    if any(value is None for value in rates.values()):
+        raise ValueError("qualification cannot pass with unmeasured error or latency evidence")
+    checks = {
+        "none_false_hint_rate": rates["none_false_hint_rate"] <= tolerances["max_none_false_hint_rate"],
+        "positive_case_recall": rates["positive_case_recall"] >= tolerances["min_positive_case_recall"],
+        "p95_ms": rates["p95_ms"] <= tolerances["max_p95_ms"],
+    }
+    return {
+        "application": validated["application"],
+        "kind": "declared_independent_labels_heldout_not_runtime_parity",
+        "experiment_sha256": result["experiment_sha256"],
+        "qualification_manifest_sha256": manifest_sha,
+        "cases": len(rows),
+        "thresholds": validated["thresholds"],
+        "tolerances": tolerances,
+        "fixed_threshold_metrics": fixed,
+        "latency": latency,
+        "checks": checks,
+        "qualified_against_frozen_tolerances": all(checks.values()),
+        "limits": "Reviewer independence is declared, not authenticated; task effectiveness and canonical Scout transcript parity are not proven.",
+    }
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 07f9340f7f8..7c0867ee599 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -23,6 +23,7 @@
 HERE = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, HERE)
 from common import calibrated, noul_questions, prompt_only, text_window
+from qualification import load_manifest, validate_manifest
 
 STATES = {
     "A_noul_prompt": prompt_only,
@@ -365,6 +366,7 @@ def main(argv=None):
     ap.add_argument("--tag", required=True)
     ap.add_argument("--server-config", required=True, help="JSON file with immutable serving identity; omit credentials")
     ap.add_argument("--subset")
+    ap.add_argument("--qualification-manifest", help="freeze reviewed held-out labels, groups, thresholds and tolerances")
     ap.add_argument("--send-model")
     ap.add_argument("--bearer-token-env", metavar="NAME",
                     help="name of an environment variable holding a bearer token (never recorded)")
@@ -389,6 +391,14 @@ def main(argv=None):
     dataset_sha = hashlib.sha256(dataset_bytes).hexdigest()
     questions = noul_questions()
     case_ids = validate_dataset(dataset, questions)
+    qualification_sha = None
+    if args.qualification_manifest:
+        if args.subset:
+            raise ValueError("qualification selects its own frozen holdout; --subset is not allowed")
+        manifest, qualification_sha = load_manifest(args.qualification_manifest)
+        qualified = validate_manifest(manifest, dataset_sha, questions, dataset)
+        keep = set(qualified["heldout_ids"])
+        dataset = [case for case in dataset if case["id"] in keep]
     if args.subset:
         with open(args.subset, encoding="utf-8") as source:
             ids = json.load(source)["ids"]
@@ -433,6 +443,8 @@ def main(argv=None):
             "noul_temperature": args.noul_temperature,
             "server_config": server_config,
         }
+        if qualification_sha is not None:
+            identity["qualification_manifest_sha256"] = qualification_sha
         header = {"experiment": identity, "experiment_sha256": digest(identity)}
         partial_path = path + ".partial"
         try:
diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py
index da9375d9624..f7c4df27c0a 100644
--- a/evals/intent_engine/score.py
+++ b/evals/intent_engine/score.py
@@ -210,6 +210,8 @@ def lat(rows):
     observed_truncated = sum(value > 0 for value in dropped)
     result = {
         "median_ms": round(statistics.median(ms)),
+        "p50_ms": statistics.median(ms),
+        "p95_ms": ms[math.ceil(0.95 * len(ms)) - 1],
         "p90_ms": round(ms[int(0.9 * (len(ms) - 1))]),
         "total_cases": len(rows),
         "cases_truncated": observed_truncated if len(dropped) == len(rows) else None,
@@ -304,6 +306,18 @@ def ch(rs):
 
 if __name__ == "__main__":
     here = os.path.dirname(os.path.abspath(__file__))
+    if len(sys.argv) > 1:
+        if len(sys.argv) != 4 or sys.argv[1] != "--qualified":
+            raise ValueError("usage: score.py [--qualified MANIFEST RESULT]")
+        from qualification import load_manifest, qualified_report
+        from common import noul_questions
+        manifest, manifest_sha = load_manifest(sys.argv[2])
+        with open(sys.argv[3], encoding="utf-8") as source:
+            result = json.load(source)
+        report = qualified_report(result, manifest, manifest_sha, noul_questions())
+        publish_summary(os.path.join(here, "qualification_summary.json"), report)
+        print(json.dumps(report, indent=2, allow_nan=False))
+        sys.exit(0)
     fixed = json.load(open(os.path.join(here, "fixed_thresholds.json"))) if os.path.exists(os.path.join(here, "fixed_thresholds.json")) else None
     summary = {}; allres = {}
     for f in sorted(glob.glob(os.path.join(here, "results", "*.json"))):
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
new file mode 100644
index 00000000000..8bc6a723ea0
--- /dev/null
+++ b/evals/intent_engine/test_qualification.py
@@ -0,0 +1,242 @@
+"""Falsifiable held-out qualification tests; no synthetic model is scored as Kev."""
+import copy
+import hashlib
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import qualification
+import run_http
+import score
+from common import CATS, noul_questions
+
+
+class QualificationTests(unittest.TestCase):
+    def setUp(self):
+        self.questions = noul_questions()
+        self.dataset = [
+            {"id": "tune-1", "labels": ["impact"], "context_needed": True,
+             "current_prompt": "What breaks?", "turns": []},
+            {"id": "tune-2", "labels": [], "context_needed": False,
+             "current_prompt": "Hello", "turns": []},
+            {"id": "holdout-positive", "labels": ["impact"], "context_needed": True,
+             "current_prompt": "What changes?", "turns": []},
+            {"id": "holdout-negative", "labels": [], "context_needed": False,
+             "current_prompt": "Show a greeting", "turns": []},
+        ]
+        self.dataset_bytes = json.dumps(self.dataset).encode("utf-8")
+        self.dataset_sha = qualification.sha256_bytes(self.dataset_bytes)
+        annotations = []
+        for case in self.dataset:
+            holdout = case["id"].startswith("holdout")
+            annotations.append({
+                "id": case["id"], "split": "holdout" if holdout else "tune",
+                "project_id": "project-b" if holdout else "project-a",
+                "session_id": "session-b" if holdout else "session-a",
+                "author": "case-creator", "reviewer": "separate-reviewer",
+                "review_ref": "https://review.example/evidence/" + case["id"],
+                "verified_labels": case["labels"],
+                "verified_context_needed": case["context_needed"],
+            })
+        self.manifest = {
+            "version": 1, "application": "historical-intent-evaluation",
+            "dataset_sha256": self.dataset_sha,
+            "questions_sha256": qualification.sha256_json(self.questions),
+            "thresholds": {key: 0.6 for key in self.questions},
+            "error_tolerances": {
+                "max_none_false_hint_rate": 0.0,
+                "min_positive_case_recall": 1.0,
+                "max_p95_ms": 20.0,
+            },
+            "annotations": annotations,
+        }
+        self.manifest_bytes = json.dumps(self.manifest).encode("utf-8")
+        self.manifest_sha = qualification.sha256_bytes(self.manifest_bytes)
+
+    def valid(self, manifest=None, cases=None):
+        return qualification.validate_manifest(
+            self.manifest if manifest is None else manifest,
+            self.dataset_sha, self.questions,
+            self.dataset if cases is None else cases,
+        )
+
+    def test_positive_and_negative_holdout_is_separate_from_tuning(self):
+        verified = self.valid()
+        self.assertEqual(verified["heldout_ids"], ["holdout-positive", "holdout-negative"])
+        self.assertEqual(verified["thresholds"]["impact"], 0.6)
+        self.assertEqual(len(verified["annotations"]), 4)
+
+    def test_near_neighbor_session_or_project_leak_refuses_entire_split(self):
+        for key, value in (("project_id", "project-a"), ("session_id", "session-a")):
+            with self.subTest(key=key):
+                changed = copy.deepcopy(self.manifest)
+                changed["annotations"][2][key] = value
+                if key == "session_id":
+                    changed["annotations"][2]["project_id"] = "project-a"
+                with self.assertRaisesRegex(ValueError, "leaks project/session"):
+                    self.valid(changed)
+
+    def test_reviewed_labels_are_not_inferred_from_prediction_or_actions(self):
+        for name, mutate, refusal in (
+            ("self_label", lambda m: m["annotations"][2].update(reviewer="case-creator"), "self-labelled"),
+            ("missing_review", lambda m: m["annotations"][2].update(review_ref=""), "review_ref"),
+            ("ambiguous_label", lambda m: m["annotations"][2].update(verified_labels=["impact", "impact"]), "invalid verified labels"),
+            ("missing_negative", lambda m: m["annotations"][3].update(split="tune"), "positive/negative"),
+            ("unreviewed_question", lambda m: m["annotations"][3].update(verified_labels=["unknown"]), "invalid verified labels"),
+        ):
+            with self.subTest(name=name):
+                manifest = copy.deepcopy(self.manifest)
+                mutate(manifest)
+                with self.assertRaisesRegex(ValueError, refusal):
+                    self.valid(manifest)
+        changed = copy.deepcopy(self.dataset)
+        changed[2]["labels"] = []
+        with self.assertRaisesRegex(ValueError, "disagrees with the dataset"):
+            self.valid(cases=changed)
+
+    def test_question_or_threshold_drift_fails_before_inference(self):
+        for name, mutate, refusal in (
+            ("question", lambda m: m.update(questions_sha256="f" * 64), "question vocabulary"),
+            ("threshold", lambda m: m["thresholds"].pop("impact"), "thresholds"),
+            ("nonfinite", lambda m: m["thresholds"].update(impact=float("nan")), "finite"),
+            ("unfrozen_tolerance", lambda m: m["error_tolerances"].pop("max_p95_ms"), "tolerances"),
+            ("dataset", lambda m: m.update(dataset_sha256="f" * 64), "dataset digest mismatch"),
+        ):
+            with self.subTest(name=name):
+                changed = copy.deepcopy(self.manifest)
+                mutate(changed)
+                with self.assertRaisesRegex(ValueError, refusal):
+                    self.valid(changed)
+
+    def signed_result(self):
+        rows = []
+        for case in self.dataset[2:]:
+            probs = {cat: (0.9 if cat in case["labels"] else 0.1) for cat in CATS}
+            rows.append({
+                "id": case["id"], "labels": case["labels"],
+                "context_needed": case["context_needed"],
+                "probs": probs,
+                "ms": 10 if case["labels"] else 20,
+                "request_sha256": hashlib.sha256(case["id"].encode()).hexdigest(),
+            })
+        experiment = {
+            "version": 2,
+            "config": "B_noul_text",
+            "case_ids": [row["id"] for row in rows],
+            "dataset_sha256": self.dataset_sha,
+            "case_annotations_sha256": score.evidence_digest([
+                [r["id"], r["labels"], r["context_needed"]] for r in rows
+            ]),
+            "requests_sha256": score.evidence_digest([
+                [r["id"], r["request_sha256"]] for r in rows
+            ]),
+            "noul_temperature": None,
+            "qualification_manifest_sha256": self.manifest_sha,
+        }
+        return {
+            "config": "trial_B_noul_text",
+            "dataset_sha256": self.dataset_sha,
+            "experiment": experiment,
+            "experiment_sha256": score.evidence_digest(experiment),
+            "rows": rows,
+        }
+
+    def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
+        result = self.signed_result()
+        report = qualification.qualified_report(
+            result, self.manifest, self.manifest_sha, self.questions
+        )
+        self.assertTrue(report["qualified_against_frozen_tolerances"])
+        self.assertEqual(report["cases"], 2)
+        self.assertEqual(report["latency"]["p50_ms"], 15)
+        self.assertEqual(report["latency"]["p95_ms"], 20)
+        self.assertEqual(report["fixed_threshold_metrics"]["none_false_hint_rate"], 0)
+        self.assertEqual(report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 1)
+
+        stricter = copy.deepcopy(self.manifest)
+        stricter["error_tolerances"]["max_p95_ms"] = 19
+        # An altered post-run manifest is not a valid requalification.
+        with self.assertRaisesRegex(ValueError, "not bound"):
+            qualification.qualified_report(result, stricter, qualification.sha256_json(stricter), self.questions)
+
+    def test_qualified_reporting_refuses_tampered_labels_and_unbound_results(self):
+        base = self.signed_result()
+        for field, mutation, pattern in (
+            ("label", lambda r: r["rows"][0].update(labels=[]), "reviewed labels"),
+            ("order", lambda r: r["rows"].reverse(), "held-out case order"),
+            ("identity", lambda r: r["experiment"].pop("qualification_manifest_sha256"), "not bound"),
+            ("forged_review", lambda r: r["rows"][1].update(context_needed=True), "reviewed labels"),
+        ):
+            with self.subTest(field=field):
+                result = copy.deepcopy(base)
+                mutation(result)
+                with self.assertRaisesRegex(ValueError, pattern):
+                    qualification.qualified_report(result, self.manifest, self.manifest_sha, self.questions)
+
+    def test_http_runner_asks_only_pre_frozen_holdout_and_resumes_with_exact_identity(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps({
+                "checkpoint": "example-pinned-checkpoint",
+                "server_revision": "example-pinned-server",
+                "backend": "example-backend", "dtype": "float32", "temperature": 2.41,
+            }))
+            calls = []
+
+            def answer(url, body):
+                calls.append(body)
+                return {"answers": {
+                    key: {"type": "noul", "noul": 0.5}
+                    for key in body["questions"]
+                }}, 12.0
+
+            args = [
+                "--url", "http://127.0.0.1:8009/v1/systemone",
+                "--tag", "frozen",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "B_noul_text",
+            ]
+            with patch.object(run_http, "HERE", str(here)), patch.object(run_http, "post", side_effect=answer):
+                run_http.main(args)
+            self.assertEqual(len(calls), 2, "must exclude training/tune cases from inference")
+            result = json.loads((here / "results" / "frozen_B_noul_text.json").read_text())
+            self.assertEqual([r["id"] for r in result["rows"]], [
+                "holdout-positive", "holdout-negative"
+            ])
+            self.assertEqual(result["experiment"]["qualification_manifest_sha256"], self.manifest_sha)
+            self.assertEqual(result["experiment"]["case_ids"], [
+                "holdout-positive", "holdout-negative"
+            ])
+
+    def test_runner_refuses_group_leakage_without_network_or_partial_output(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            changed = copy.deepcopy(self.manifest)
+            changed["annotations"][2]["project_id"] = "project-a"
+            (here / "manifest.json").write_text(json.dumps(changed))
+            (here / "server.json").write_text(json.dumps({
+                "checkpoint": "pinned", "server_revision": "r1", "backend": "cpu",
+                "dtype": "float32", "temperature": 2.41,
+            }))
+            args = [
+                "--url", "http://127.0.0.1:8009/v1/systemone",
+                "--tag", "refused",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "B_noul_text",
+            ]
+            with patch.object(run_http, "HERE", str(here)), patch.object(run_http, "post") as post:
+                with self.assertRaisesRegex(ValueError, "leaks project/session"):
+                    run_http.main(args)
+                post.assert_not_called()
+            self.assertFalse((here / "results").exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

From 9dfcb8ff509560dc25b4b83f3495dc2cb5b9ee9b Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 16:52:09 -0800
Subject: [PATCH 02/76] test(evals): reject altered resumed qualification

---
 evals/intent_engine/test_qualification.py | 55 +++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 8bc6a723ea0..ff0bc894599 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -238,5 +238,60 @@ def test_runner_refuses_group_leakage_without_network_or_partial_output(self):
             self.assertFalse((here / "results").exists())
 
 
+    def test_frozen_manifest_cannot_reorder_a_dataset(self):
+        changed = copy.deepcopy(self.manifest)
+        changed["annotations"][0], changed["annotations"][1] = (
+            changed["annotations"][1], changed["annotations"][0]
+        )
+        with self.assertRaisesRegex(ValueError, "case ordering"):
+            self.valid(changed)
+
+    def test_changed_manifest_cannot_resume_or_spend_inference_tokens(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            manifest_path = here / "manifest.json"
+            manifest_path.write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps({
+                "checkpoint": "pinned", "server_revision": "r1", "backend": "cpu",
+                "dtype": "float32", "temperature": 2.41,
+            }))
+            args = [
+                "--url", "http://127.0.0.1:8009/v1/systemone",
+                "--tag", "interrupted",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(manifest_path),
+                "B_noul_text",
+            ]
+            answer = {"answers": {
+                key: {"type": "noul", "noul": 0.5} for key in self.questions
+            }}
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post", side_effect=[
+                    (answer, 11.0), RuntimeError("interrupted"),
+                ]) as initial:
+                    with self.assertRaisesRegex(RuntimeError, "interrupted"):
+                        run_http.main(args)
+                    self.assertEqual(initial.call_count, 2)
+                partial = here / "results" / "interrupted_B_noul_text.json.partial"
+                frozen_bytes = partial.read_bytes()
+                self.assertEqual(len(frozen_bytes.splitlines()), 2,
+                                 "first accepted prediction must be durably resumable")
+                changed = copy.deepcopy(self.manifest)
+                changed["error_tolerances"]["max_p95_ms"] = 21.0
+                manifest_path.write_text(json.dumps(changed))
+                with patch.object(run_http, "post") as no_inference:
+                    with self.assertRaisesRegex(ValueError, "another or unrecorded experiment"):
+                        run_http.main(args)
+                    no_inference.assert_not_called()
+                self.assertEqual(partial.read_bytes(), frozen_bytes)
+                manifest_path.write_bytes(self.manifest_bytes)
+                with patch.object(run_http, "post", return_value=(answer, 12.0)) as remaining:
+                    run_http.main(args)
+                    self.assertEqual(remaining.call_count, 1)
+            completed = here / "results" / "interrupted_B_noul_text.json"
+            self.assertEqual(len(json.loads(completed.read_text())["rows"]), 2)
+
+
 if __name__ == "__main__":
     unittest.main()

From bce313b4c16225a1ca2e19ddc2afce4ad3abecf5 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 16:53:41 -0800
Subject: [PATCH 03/76] fix(evals): keep held-out results out of exploratory
 scoring

---
 evals/intent_engine/score.py              |  3 ++
 evals/intent_engine/test_qualification.py | 50 +++++++++++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py
index f7c4df27c0a..bc9bfa3ffd3 100644
--- a/evals/intent_engine/score.py
+++ b/evals/intent_engine/score.py
@@ -322,6 +322,9 @@ def ch(rs):
     summary = {}; allres = {}
     for f in sorted(glob.glob(os.path.join(here, "results", "*.json"))):
         res = json.load(open(f))
+        if (isinstance(res.get("experiment"), dict)
+                and "qualification_manifest_sha256" in res["experiment"]):
+            raise ValueError("held-out results require --qualified MANIFEST RESULT; refuse in-sample scoring")
         if res["config"] in allres:
             raise ValueError("duplicate configuration %r in result files" % res["config"])
         allres[res["config"]] = res
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index ff0bc894599..865d6f0cc48 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -2,6 +2,9 @@
 import copy
 import hashlib
 import json
+import shutil
+import subprocess
+import sys
 import tempfile
 import unittest
 from pathlib import Path
@@ -293,5 +296,52 @@ def test_changed_manifest_cannot_resume_or_spend_inference_tokens(self):
             self.assertEqual(len(json.loads(completed.read_text())["rows"]), 2)
 
 
+    def test_qualified_cli_is_separate_and_historical_cli_refuses_heldout(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            for filename in ("score.py", "qualification.py", "common.py"):
+                shutil.copyfile(Path(score.__file__).with_name(filename), here / filename)
+            (here / "results").mkdir()
+            result_path = here / "results" / "qualified.json"
+            result_path.write_text(json.dumps(self.signed_result()))
+            manifest_path = here / "qualification.json"
+            manifest_path.write_bytes(self.manifest_bytes)
+            summary = here / "summary.json"
+            summary.write_bytes(b"existing historical evidence\\n")
+
+            default = subprocess.run(
+                [sys.executable, str(here / "score.py")],
+                capture_output=True, text=True, check=False,
+            )
+            self.assertNotEqual(default.returncode, 0)
+            self.assertIn("refuse in-sample scoring", default.stderr)
+            self.assertEqual(summary.read_bytes(), b"existing historical evidence\\n")
+
+            qualified = subprocess.run(
+                [sys.executable, str(here / "score.py"), "--qualified",
+                 str(manifest_path), str(result_path)],
+                capture_output=True, text=True, check=False,
+            )
+            self.assertEqual(qualified.returncode, 0, qualified.stderr)
+            report_path = here / "qualification_summary.json"
+            report = json.loads(report_path.read_text())
+            self.assertTrue(report["qualified_against_frozen_tolerances"])
+            self.assertEqual(report["latency"]["p95_ms"], 20)
+            self.assertEqual(summary.read_bytes(), b"existing historical evidence\\n")
+
+            corrupted = self.signed_result()
+            corrupted["rows"][0]["labels"] = []
+            result_path.write_text(json.dumps(corrupted))
+            previous_report = report_path.read_bytes()
+            rejected = subprocess.run(
+                [sys.executable, str(here / "score.py"), "--qualified",
+                 str(manifest_path), str(result_path)],
+                capture_output=True, text=True, check=False,
+            )
+            self.assertNotEqual(rejected.returncode, 0)
+            self.assertIn("reviewed labels", rejected.stderr)
+            self.assertEqual(report_path.read_bytes(), previous_report)
+
+
 if __name__ == "__main__":
     unittest.main()

From 470cbb044d5e921820978ad1e4e9eded04e3d47b Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 16:54:47 -0800
Subject: [PATCH 04/76] fix(evals): require pinned server identity for
 qualification

---
 evals/intent_engine/qualification.py      | 10 ++++++++++
 evals/intent_engine/test_qualification.py | 21 +++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 73be56d1a0d..70b0befe883 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -139,6 +139,16 @@ def qualified_report(result, manifest, manifest_sha, questions):
     experiment = result.get("experiment")
     if not isinstance(experiment, dict):
         raise ValueError("qualification requires an identity-bound HTTP experiment")
+    server = experiment.get("server_config")
+    if not isinstance(server, dict):
+        raise ValueError("qualification requires a pinned serving declaration")
+    for field in ("checkpoint", "server_revision", "backend", "dtype"):
+        if not isinstance(server.get(field), str) or not server[field].strip():
+            raise ValueError(f"qualification serving declaration lacks {field}")
+    temperature = server.get("temperature")
+    if (isinstance(temperature, bool) or not isinstance(temperature, (int, float))
+            or not math.isfinite(temperature) or temperature < 0):
+        raise ValueError("qualification requires a finite serving temperature")
     if experiment.get("qualification_manifest_sha256") != manifest_sha:
         raise ValueError("qualification experiment is not bound to this frozen manifest")
     validated = validate_manifest(manifest, result.get("dataset_sha256"), questions)
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 865d6f0cc48..15d6db0f97b 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -136,6 +136,11 @@ def signed_result(self):
                 [r["id"], r["request_sha256"]] for r in rows
             ]),
             "noul_temperature": None,
+            "server_config": {
+                "checkpoint": "checkpoint-pinned-by-operator",
+                "server_revision": "immutable-server-revision",
+                "backend": "cpu", "dtype": "float32", "temperature": 2.41,
+            },
             "qualification_manifest_sha256": self.manifest_sha,
         }
         return {
@@ -164,6 +169,22 @@ def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
         with self.assertRaisesRegex(ValueError, "not bound"):
             qualification.qualified_report(result, stricter, qualification.sha256_json(stricter), self.questions)
 
+
+    def test_qualification_requires_complete_serving_identity(self):
+        for field, change in (
+            ("missing_server", lambda r: r["experiment"].pop("server_config")),
+            ("missing_checkpoint", lambda r: r["experiment"]["server_config"].pop("checkpoint")),
+            ("unknown_revision", lambda r: r["experiment"]["server_config"].update(server_revision="")),
+            ("nan_temperature", lambda r: r["experiment"]["server_config"].update(temperature=float("nan"))),
+        ):
+            with self.subTest(field=field):
+                result = self.signed_result()
+                change(result)
+                with self.assertRaisesRegex(ValueError, "serving"):
+                    qualification.qualified_report(
+                        result, self.manifest, self.manifest_sha, self.questions
+                    )
+
     def test_qualified_reporting_refuses_tampered_labels_and_unbound_results(self):
         base = self.signed_result()
         for field, mutation, pattern in (

From 09915be916a0b20d5d46f093177953dfd179105b Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 17:14:38 -0800
Subject: [PATCH 05/76] fix(evals): bind qualification to original rendered
 requests

---
 evals/intent_engine/BUILD.bazel           |  1 +
 evals/intent_engine/HTTP_RUNNER.md        | 11 +++-
 evals/intent_engine/qualification.py      | 31 +++++++++-
 evals/intent_engine/run_http.py           |  3 +-
 evals/intent_engine/score.py              | 10 +--
 evals/intent_engine/test_qualification.py | 75 ++++++++++++++++++++---
 6 files changed, 112 insertions(+), 19 deletions(-)

diff --git a/evals/intent_engine/BUILD.bazel b/evals/intent_engine/BUILD.bazel
index 0fb8fe0cc4e..e2e81e0c16d 100644
--- a/evals/intent_engine/BUILD.bazel
+++ b/evals/intent_engine/BUILD.bazel
@@ -34,6 +34,7 @@ py_test(
     srcs = [
         "common.py",
         "qualification.py",
+        "run_http.py",
         "compare.py",
         "score.py",
         "test_score.py",
diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 600cd944a1b..89fa0333b8e 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -137,7 +137,9 @@ not cryptographic verification of independent label quality. Do not forge them
 from model predictions or adoption events.
 
 Run with the pinned remote serving declaration as before, adding
-`--qualification-manifest /path/to/frozen.json`. The runner refuses an
+`--dataset /path/to/original-cases.json` and
+`--qualification-manifest /path/to/frozen.json`. The dataset file's bytes
+are bound to the run and must be supplied again when scoring. The runner refuses an
 additional `--subset`, validates every case and its group before network
 traffic, infers **only the frozen held-out cases**, and binds the raw manifest's
 SHA-256 to the experiment identity and resumable partial log. A changed
@@ -146,14 +148,17 @@ the same manifest:
 
 ```sh
 python evals/intent_engine/score.py --qualified /path/to/frozen.json \
-  evals/intent_engine/results/pinned-run_B_noul_text.json
+  /path/to/original-cases.json evals/intent_engine/results/pinned-run_B_noul_text.json
 bazel test //evals/intent_engine:qualification_test --test_output=errors
 ```
 
 This writes `evals/intent_engine/qualification_summary.json` separately from
 the historical `summary.json`, and reports only **frozen-threshold** held-out
 false hints, positive-case recall, p50/p95 measured end-to-end HTTP latency and
-the predeclared tolerances. It deliberately does not publish optimized
+the predeclared tolerances. Scoring reads the original frozen case bytes,
+checks all reviewed labels/splits against them and recreates each HTTP
+request fingerprint with the runner's own renderer. Self-consistent result
+hashes alone do not prove the submitted inputs. It deliberately does not publish optimized
 in-sample thresholds as qualification. Neither correct answers nor those metrics
 alone demonstrate task success, ignored/acted outcomes, cold/warm cost, native
 runtime renderer/settings parity or a real Kev checkpoint. All those require
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 70b0befe883..c249f010e64 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -132,9 +132,18 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     }
 
 
-def qualified_report(result, manifest, manifest_sha, questions):
-    """Use the existing scorer, but publish only pre-frozen held-out metrics."""
+def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
+    """Reproduce qualified predictions from the frozen original dataset."""
     from score import score_config
+    from run_http import STATES, request_bytes, validate_dataset
+
+    if not isinstance(dataset_bytes, bytes):
+        raise ValueError("qualification requires the original dataset bytes")
+    dataset_sha = sha256_bytes(dataset_bytes)
+    if result.get("dataset_sha256") != dataset_sha:
+        raise ValueError("qualification result dataset digest differs from supplied evidence")
+    cases = json.loads(dataset_bytes)
+    validate_dataset(cases, questions)
 
     experiment = result.get("experiment")
     if not isinstance(experiment, dict):
@@ -151,7 +160,7 @@ def qualified_report(result, manifest, manifest_sha, questions):
         raise ValueError("qualification requires a finite serving temperature")
     if experiment.get("qualification_manifest_sha256") != manifest_sha:
         raise ValueError("qualification experiment is not bound to this frozen manifest")
-    validated = validate_manifest(manifest, result.get("dataset_sha256"), questions)
+    validated = validate_manifest(manifest, dataset_sha, questions, cases)
     rows = result.get("rows")
     if not isinstance(rows, list) or [r.get("id") for r in rows if isinstance(r, dict)] != validated["heldout_ids"]:
         raise ValueError("qualification result differs from frozen held-out case order")
@@ -160,6 +169,22 @@ def qualified_report(result, manifest, manifest_sha, questions):
         if (row.get("labels") != annotation["verified_labels"]
                 or row.get("context_needed") != annotation["verified_context_needed"]):
             raise ValueError(f"qualification result {row['id']} does not match reviewed labels")
+    # Reconstruct real HTTP request bytes: self-consistent metadata hashes
+    # alone cannot prove the frozen prompts were used.
+    renderer = STATES.get(experiment.get("config"))
+    if renderer is None:
+        raise ValueError("qualification has an unsupported request renderer")
+    case_map = {case["id"]: case for case in cases}
+    for row in rows:
+        body = {"state": renderer(case_map[row["id"]]), "questions": questions}
+        if experiment.get("requested_model"):
+            body["model"] = experiment["requested_model"]
+        expected = sha256_bytes(request_bytes(body))
+        if row.get("request_sha256") != expected:
+            raise ValueError(
+                f"qualification request fingerprint disagrees with frozen renderer for {row['id']}"
+            )
+
     scores = score_config(result, fixed_t=validated["thresholds"])
     fixed = scores["fixed"]["all"]
     latency = scores["latency"]
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 7c0867ee599..4dad799d8a8 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -366,6 +366,7 @@ def main(argv=None):
     ap.add_argument("--tag", required=True)
     ap.add_argument("--server-config", required=True, help="JSON file with immutable serving identity; omit credentials")
     ap.add_argument("--subset")
+    ap.add_argument("--dataset", help="original JSON cases; frozen digest recorded in each run")
     ap.add_argument("--qualification-manifest", help="freeze reviewed held-out labels, groups, thresholds and tolerances")
     ap.add_argument("--send-model")
     ap.add_argument("--bearer-token-env", metavar="NAME",
@@ -385,7 +386,7 @@ def main(argv=None):
                     if args.bearer_token_env is not None else None)
     if args.sample_out and os.path.lexists(args.sample_out) and not os.path.isfile(args.sample_out):
         raise ValueError(f"sample path exists but is not a regular file: {args.sample_out}")
-    with open(os.path.join(HERE, "dataset.json"), "rb") as source:
+    with open(args.dataset or os.path.join(HERE, "dataset.json"), "rb") as source:
         dataset_bytes = source.read()
     dataset = json.loads(dataset_bytes)
     dataset_sha = hashlib.sha256(dataset_bytes).hexdigest()
diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py
index bc9bfa3ffd3..844f8408845 100644
--- a/evals/intent_engine/score.py
+++ b/evals/intent_engine/score.py
@@ -307,14 +307,16 @@ def ch(rs):
 if __name__ == "__main__":
     here = os.path.dirname(os.path.abspath(__file__))
     if len(sys.argv) > 1:
-        if len(sys.argv) != 4 or sys.argv[1] != "--qualified":
-            raise ValueError("usage: score.py [--qualified MANIFEST RESULT]")
+        if len(sys.argv) != 5 or sys.argv[1] != "--qualified":
+            raise ValueError("usage: score.py [--qualified MANIFEST DATASET RESULT]")
         from qualification import load_manifest, qualified_report
         from common import noul_questions
         manifest, manifest_sha = load_manifest(sys.argv[2])
-        with open(sys.argv[3], encoding="utf-8") as source:
+        with open(sys.argv[3], "rb") as source:
+            dataset_bytes = source.read()
+        with open(sys.argv[4], encoding="utf-8") as source:
             result = json.load(source)
-        report = qualified_report(result, manifest, manifest_sha, noul_questions())
+        report = qualified_report(result, manifest, manifest_sha, noul_questions(), dataset_bytes)
         publish_summary(os.path.join(here, "qualification_summary.json"), report)
         print(json.dumps(report, indent=2, allow_nan=False))
         sys.exit(0)
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 15d6db0f97b..660df17ada9 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -122,7 +122,10 @@ def signed_result(self):
                 "context_needed": case["context_needed"],
                 "probs": probs,
                 "ms": 10 if case["labels"] else 20,
-                "request_sha256": hashlib.sha256(case["id"].encode()).hexdigest(),
+                "request_sha256": hashlib.sha256(run_http.request_bytes({
+                    "state": run_http.STATES["B_noul_text"](case),
+                    "questions": self.questions,
+                })).hexdigest(),
             })
         experiment = {
             "version": 2,
@@ -154,7 +157,7 @@ def signed_result(self):
     def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
         result = self.signed_result()
         report = qualification.qualified_report(
-            result, self.manifest, self.manifest_sha, self.questions
+            result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes
         )
         self.assertTrue(report["qualified_against_frozen_tolerances"])
         self.assertEqual(report["cases"], 2)
@@ -167,7 +170,7 @@ def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
         stricter["error_tolerances"]["max_p95_ms"] = 19
         # An altered post-run manifest is not a valid requalification.
         with self.assertRaisesRegex(ValueError, "not bound"):
-            qualification.qualified_report(result, stricter, qualification.sha256_json(stricter), self.questions)
+            qualification.qualified_report(result, stricter, qualification.sha256_json(stricter), self.questions, self.dataset_bytes)
 
 
     def test_qualification_requires_complete_serving_identity(self):
@@ -182,7 +185,7 @@ def test_qualification_requires_complete_serving_identity(self):
                 change(result)
                 with self.assertRaisesRegex(ValueError, "serving"):
                     qualification.qualified_report(
-                        result, self.manifest, self.manifest_sha, self.questions
+                        result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes
                     )
 
     def test_qualified_reporting_refuses_tampered_labels_and_unbound_results(self):
@@ -197,7 +200,7 @@ def test_qualified_reporting_refuses_tampered_labels_and_unbound_results(self):
                 result = copy.deepcopy(base)
                 mutation(result)
                 with self.assertRaisesRegex(ValueError, pattern):
-                    qualification.qualified_report(result, self.manifest, self.manifest_sha, self.questions)
+                    qualification.qualified_report(result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes)
 
     def test_http_runner_asks_only_pre_frozen_holdout_and_resumes_with_exact_identity(self):
         with tempfile.TemporaryDirectory() as temp:
@@ -320,13 +323,14 @@ def test_changed_manifest_cannot_resume_or_spend_inference_tokens(self):
     def test_qualified_cli_is_separate_and_historical_cli_refuses_heldout(self):
         with tempfile.TemporaryDirectory() as temp:
             here = Path(temp)
-            for filename in ("score.py", "qualification.py", "common.py"):
+            for filename in ("score.py", "qualification.py", "common.py", "run_http.py"):
                 shutil.copyfile(Path(score.__file__).with_name(filename), here / filename)
             (here / "results").mkdir()
             result_path = here / "results" / "qualified.json"
             result_path.write_text(json.dumps(self.signed_result()))
             manifest_path = here / "qualification.json"
             manifest_path.write_bytes(self.manifest_bytes)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
             summary = here / "summary.json"
             summary.write_bytes(b"existing historical evidence\\n")
 
@@ -340,7 +344,7 @@ def test_qualified_cli_is_separate_and_historical_cli_refuses_heldout(self):
 
             qualified = subprocess.run(
                 [sys.executable, str(here / "score.py"), "--qualified",
-                 str(manifest_path), str(result_path)],
+                 str(manifest_path), str(here / "dataset.json"), str(result_path)],
                 capture_output=True, text=True, check=False,
             )
             self.assertEqual(qualified.returncode, 0, qualified.stderr)
@@ -356,13 +360,68 @@ def test_qualified_cli_is_separate_and_historical_cli_refuses_heldout(self):
             previous_report = report_path.read_bytes()
             rejected = subprocess.run(
                 [sys.executable, str(here / "score.py"), "--qualified",
-                 str(manifest_path), str(result_path)],
+                 str(manifest_path), str(here / "dataset.json"), str(result_path)],
                 capture_output=True, text=True, check=False,
             )
             self.assertNotEqual(rejected.returncode, 0)
             self.assertIn("reviewed labels", rejected.stderr)
             self.assertEqual(report_path.read_bytes(), previous_report)
 
+    def test_qualified_requires_original_cases_and_reproducible_requests(self):
+        result = self.signed_result()
+        altered_cases = copy.deepcopy(self.dataset)
+        altered_cases[2]["current_prompt"] = "A different prompt"
+        with self.assertRaisesRegex(ValueError, "dataset digest differs"):
+            qualification.qualified_report(
+                result, self.manifest, self.manifest_sha, self.questions,
+                json.dumps(altered_cases).encode("utf-8"),
+            )
+        # The stored hashes can be re-computed consistently around a forged
+        # request fingerprint; reconstruction from real cases must catch it.
+        altered = copy.deepcopy(result)
+        altered["rows"][0]["request_sha256"] = "f" * 64
+        altered["experiment"]["requests_sha256"] = score.evidence_digest([
+            [row["id"], row["request_sha256"]] for row in altered["rows"]
+        ])
+        altered["experiment_sha256"] = score.evidence_digest(altered["experiment"])
+        with self.assertRaisesRegex(ValueError, "request fingerprint disagrees"):
+            qualification.qualified_report(
+                altered, self.manifest, self.manifest_sha,
+                self.questions, self.dataset_bytes,
+            )
+
+    def test_runner_can_consume_explicit_frozen_input_file(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            dataset_path = here / "external-cases.json"
+            dataset_path.write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps({
+                "checkpoint": "pinned", "server_revision": "r1",
+                "backend": "cpu", "dtype": "float32", "temperature": 2.41,
+            }))
+            answer = {"answers": {
+                key: {"type": "noul", "noul": 0.5} for key in self.questions
+            }}
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post", return_value=(answer, 12.0)) as post:
+                    run_http.main([
+                        "--url", "http://127.0.0.1:8009/v1/systemone",
+                        "--tag", "external",
+                        "--server-config", str(here / "server.json"),
+                        "--dataset", str(dataset_path),
+                        "--qualification-manifest", str(here / "manifest.json"),
+                        "B_noul_text",
+                    ])
+                    self.assertEqual(post.call_count, 2)
+            result = json.loads(
+                (here / "results" / "external_B_noul_text.json").read_text()
+            )
+            qualification.qualified_report(
+                result, self.manifest, self.manifest_sha,
+                self.questions, self.dataset_bytes,
+            )
+
 
 if __name__ == "__main__":
     unittest.main()

From c7b6e0eed5b96cd6e019f7213b0b80940ec03a98 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 18:19:24 -0800
Subject: [PATCH 06/76] fix(evals): gate qualification on faithful measured
 inputs

Refs #4535

Count wrong extra suggestions on positive cases with frozen unwanted-hint
budgets; disclose coverage, abstention and missed positives. Refuse a
passing report for missing or nonzero server truncation telemetry. Also
prevent cross-project reuse of a session between train and holdout.
---
 evals/intent_engine/HTTP_RUNNER.md        | 17 +++++---
 evals/intent_engine/qualification.py      | 42 ++++++++++++++++++-
 evals/intent_engine/test_qualification.py | 50 +++++++++++++++++++++++
 3 files changed, 102 insertions(+), 7 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 89fa0333b8e..07179b1aa7c 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -121,9 +121,12 @@ Create a separate JSON annotation manifest **before running inference** with
 `qualification.sha256_json(common.noul_questions())` of the runner's exact
 question dictionary. This last digest must change with question text. Include
 `thresholds` with one frozen probability threshold per requested question and
-`error_tolerances` with all three numbers:
-`max_none_false_hint_rate` (0–1), `min_positive_case_recall` (0–1) and
-`max_p95_ms` (nonnegative milliseconds).
+`error_tolerances` with all four numbers:
+`max_none_false_hint_rate` (0–1), `max_unwanted_hint_rate` (0–1),
+`min_positive_case_recall` (0–1), and `max_p95_ms` (nonnegative milliseconds).
+The unwanted-hint budget counts an incorrect extra suggestion on a positive
+case, even if another correct suggestion made its recall appear successful.
+These tolerances must be frozen before examining the held-out results.
 
 The `annotations` array must cover every case in the original dataset, in
 that dataset's order. Each annotation has `id`, `split` (`tune` or
@@ -154,8 +157,12 @@ bazel test //evals/intent_engine:qualification_test --test_output=errors
 
 This writes `evals/intent_engine/qualification_summary.json` separately from
 the historical `summary.json`, and reports only **frozen-threshold** held-out
-false hints, positive-case recall, p50/p95 measured end-to-end HTTP latency and
-the predeclared tolerances. Scoring reads the original frozen case bytes,
+false hints, all-case unwanted suggestions, missed positive cases, coverage,
+abstention, positive-case recall, p50/p95 measured end-to-end HTTP latency and
+the predeclared tolerances. A passing report requires every held-out case to
+report measured zero dropped state tokens. Missing truncation telemetry is
+unknown, and a response measured after truncated input cannot prove the full
+frozen request was evaluated; the measurements remain in the failing report. Scoring reads the original frozen case bytes,
 checks all reviewed labels/splits against them and recreates each HTTP
 request fingerprint with the runner's own renderer. Self-consistent result
 hashes alone do not prove the submitted inputs. It deliberately does not publish optimized
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index c249f010e64..50d892ef41b 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -49,7 +49,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
             raise ValueError("qualification thresholds must be finite probabilities")
 
     tolerances = manifest.get("error_tolerances")
-    required = ("max_none_false_hint_rate", "min_positive_case_recall", "max_p95_ms")
+    required = ("max_none_false_hint_rate", "max_unwanted_hint_rate", "min_positive_case_recall", "max_p95_ms")
     if not isinstance(tolerances, dict) or set(tolerances) != set(required):
         raise ValueError("qualification tolerances must be frozen before inference")
     for key in required:
@@ -107,7 +107,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
                     or recorded.get("context_needed") != annotation["verified_context_needed"]):
                 raise ValueError(f"qualification {case_id} disagrees with the dataset")
         projects[split].add(annotation["project_id"])
-        sessions[split].add((annotation["project_id"], annotation["session_id"]))
+        sessions[split].add(annotation["session_id"])
         if split == "holdout":
             heldout.append(case_id)
             positive += bool(labels)
@@ -132,6 +132,32 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     }
 
 
+def holdout_decision_metrics(rows, thresholds):
+    """Count unwanted suggestions even when another hint on the same case is correct.
+
+    False hints on negative-only cases are not the full false-positive budget:
+    a positive case can still receive an incorrect additional suggestion.
+    """
+    hinted = unwanted = missed_positive = positive = 0
+    for row in rows:
+        labels = set(row["labels"])
+        fired = {key for key in thresholds if row["probs"][key] >= thresholds[key]}
+        hinted += bool(fired)
+        unwanted += bool(fired - labels)
+        positive += bool(labels)
+        missed_positive += bool(labels and not (fired & labels))
+    return {
+        "hinted_cases": hinted,
+        "abstained_cases": len(rows) - hinted,
+        "coverage_rate": hinted / len(rows),
+        "abstention_rate": (len(rows) - hinted) / len(rows),
+        "unwanted_hint_cases": unwanted,
+        "unwanted_hint_rate": unwanted / len(rows),
+        "missed_positive_cases": missed_positive,
+        "missed_positive_rate": missed_positive / positive if positive else None,
+    }
+
+
 def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     """Reproduce qualified predictions from the frozen original dataset."""
     from score import score_config
@@ -188,18 +214,29 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     scores = score_config(result, fixed_t=validated["thresholds"])
     fixed = scores["fixed"]["all"]
     latency = scores["latency"]
+    decisions = holdout_decision_metrics(rows, validated["thresholds"])
     tolerances = validated["error_tolerances"]
     rates = {
         "none_false_hint_rate": fixed["none_false_hint_rate"],
+        "unwanted_hint_rate": decisions["unwanted_hint_rate"],
         "positive_case_recall": fixed["pos_cases_with_a_correct_hint"],
         "p95_ms": latency["p95_ms"],
     }
     if any(value is None for value in rates.values()):
         raise ValueError("qualification cannot pass with unmeasured error or latency evidence")
+    # A successful HTTP call can conceal server-side truncation. Such a score
+    # cannot prove that the model saw the frozen full input. Unknown telemetry
+    # is not a measured zero, even when every prediction happens to be correct.
+    complete_input_evidence = (
+        latency["truncation_measured_cases"] == len(rows)
+        and latency["cases_truncated"] == 0
+    )
     checks = {
         "none_false_hint_rate": rates["none_false_hint_rate"] <= tolerances["max_none_false_hint_rate"],
+        "unwanted_hint_rate": rates["unwanted_hint_rate"] <= tolerances["max_unwanted_hint_rate"],
         "positive_case_recall": rates["positive_case_recall"] >= tolerances["min_positive_case_recall"],
         "p95_ms": rates["p95_ms"] <= tolerances["max_p95_ms"],
+        "untruncated_input_evidence": complete_input_evidence,
     }
     return {
         "application": validated["application"],
@@ -210,6 +247,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
         "thresholds": validated["thresholds"],
         "tolerances": tolerances,
         "fixed_threshold_metrics": fixed,
+        "holdout_decisions": decisions,
         "latency": latency,
         "checks": checks,
         "qualified_against_frozen_tolerances": all(checks.values()),
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 660df17ada9..e34d5cb4654 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -50,6 +50,7 @@ def setUp(self):
             "thresholds": {key: 0.6 for key in self.questions},
             "error_tolerances": {
                 "max_none_false_hint_rate": 0.0,
+                "max_unwanted_hint_rate": 0.0,
                 "min_positive_case_recall": 1.0,
                 "max_p95_ms": 20.0,
             },
@@ -81,6 +82,49 @@ def test_near_neighbor_session_or_project_leak_refuses_entire_split(self):
                 with self.assertRaisesRegex(ValueError, "leaks project/session"):
                     self.valid(changed)
 
+    def test_reused_session_id_under_different_project_is_not_independent(self):
+        changed = copy.deepcopy(self.manifest)
+        changed["annotations"][2]["session_id"] = "session-a"
+        # Changing the project label must not conceal a shared session.
+        self.assertNotEqual(changed["annotations"][2]["project_id"], "project-a")
+        with self.assertRaisesRegex(ValueError, "leaks project/session"):
+            self.valid(changed)
+
+    def test_unwanted_hints_on_positive_cases_are_not_hidden_by_recall(self):
+        result = self.signed_result()
+        result["rows"][0]["probs"]["call_graph"] = 0.99
+        report = qualification.qualified_report(
+            result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes
+        )
+        self.assertEqual(report["fixed_threshold_metrics"]["none_false_hint_rate"], 0)
+        self.assertEqual(report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 1)
+        self.assertEqual(report["holdout_decisions"]["unwanted_hint_cases"], 1)
+        self.assertEqual(report["holdout_decisions"]["unwanted_hint_rate"], 0.5)
+        self.assertEqual(report["holdout_decisions"]["abstention_rate"], 0.5)
+        self.assertFalse(report["checks"]["unwanted_hint_rate"])
+        self.assertFalse(report["qualified_against_frozen_tolerances"])
+
+    def test_missing_or_dropped_server_tokens_cannot_qualify(self):
+        for value in (None, 1):
+            with self.subTest(state_tokens_dropped=value):
+                result = self.signed_result()
+                result["rows"][0]["dropped"] = value
+                report = qualification.qualified_report(
+                    result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes
+                )
+                self.assertFalse(report["checks"]["untruncated_input_evidence"])
+                self.assertFalse(report["qualified_against_frozen_tolerances"])
+                self.assertEqual(
+                    report["latency"]["truncation_measured_cases"],
+                    1 if value is None else 2,
+                )
+        complete = qualification.qualified_report(
+            self.signed_result(), self.manifest, self.manifest_sha,
+            self.questions, self.dataset_bytes,
+        )
+        self.assertTrue(complete["checks"]["untruncated_input_evidence"])
+        self.assertTrue(complete["qualified_against_frozen_tolerances"])
+
     def test_reviewed_labels_are_not_inferred_from_prediction_or_actions(self):
         for name, mutate, refusal in (
             ("self_label", lambda m: m["annotations"][2].update(reviewer="case-creator"), "self-labelled"),
@@ -105,6 +149,7 @@ def test_question_or_threshold_drift_fails_before_inference(self):
             ("threshold", lambda m: m["thresholds"].pop("impact"), "thresholds"),
             ("nonfinite", lambda m: m["thresholds"].update(impact=float("nan")), "finite"),
             ("unfrozen_tolerance", lambda m: m["error_tolerances"].pop("max_p95_ms"), "tolerances"),
+            ("unfrozen_unwanted", lambda m: m["error_tolerances"].pop("max_unwanted_hint_rate"), "tolerances"),
             ("dataset", lambda m: m.update(dataset_sha256="f" * 64), "dataset digest mismatch"),
         ):
             with self.subTest(name=name):
@@ -122,6 +167,7 @@ def signed_result(self):
                 "context_needed": case["context_needed"],
                 "probs": probs,
                 "ms": 10 if case["labels"] else 20,
+                "dropped": 0,
                 "request_sha256": hashlib.sha256(run_http.request_bytes({
                     "state": run_http.STATES["B_noul_text"](case),
                     "questions": self.questions,
@@ -163,6 +209,10 @@ def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
         self.assertEqual(report["cases"], 2)
         self.assertEqual(report["latency"]["p50_ms"], 15)
         self.assertEqual(report["latency"]["p95_ms"], 20)
+        self.assertEqual(report["holdout_decisions"]["coverage_rate"], 0.5)
+        self.assertEqual(report["holdout_decisions"]["abstention_rate"], 0.5)
+        self.assertEqual(report["holdout_decisions"]["unwanted_hint_rate"], 0)
+        self.assertTrue(report["checks"]["untruncated_input_evidence"])
         self.assertEqual(report["fixed_threshold_metrics"]["none_false_hint_rate"], 0)
         self.assertEqual(report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 1)
 

From 711a18f8a53b62380af3113cd83108e0ef947623 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 19:16:42 -0800
Subject: [PATCH 07/76] fix(evals): bind qualification split groups to frozen
 cases

Refs #4535
---
 evals/intent_engine/HTTP_RUNNER.md        |  7 ++++--
 evals/intent_engine/qualification.py      | 12 ++++++++--
 evals/intent_engine/test_qualification.py | 28 +++++++++++++++++++++++
 3 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 07179b1aa7c..6081d2aee42 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -134,8 +134,11 @@ that dataset's order. Each annotation has `id`, `split` (`tune` or
 `reviewer`, `review_ref`, `verified_labels` and Boolean
 `verified_context_needed`. The independently reviewed labels must exactly
 match the inference dataset. Tune and held-out projects cannot overlap; session
-groups cannot overlap. The held-out group must contain both a positive and a
-negative example. Reviewer identifiers and references are **declarations**,
+groups cannot overlap. The original frozen dataset must also carry each case's
+`project_id` and `session_id`, identical to its reviewed annotation. Inventing
+independent groups solely inside the manifest is refused even if the manifest
+and result hashes are recomputed. The held-out group must contain both a
+positive and a negative example. Reviewer identifiers and references are **declarations**,
 not cryptographic verification of independent label quality. Do not forge them
 from model predictions or adoption events.
 
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 50d892ef41b..d9b9a4f6ec7 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -28,8 +28,8 @@ def load_manifest(path):
 def validate_manifest(manifest, dataset_sha, questions, cases=None):
     """Fail closed before an HTTP request or a qualified result is published.
 
-    `cases` is the complete original dataset, not a filtered subset. Scoring
-    passes None because the experiment already binds the dataset's byte digest.
+    `cases` is the complete original dataset, not a filtered subset. Its
+    source-owned grouping identities must match the declared split metadata.
     """
     if not isinstance(manifest, dict) or manifest.get("version") != 1:
         raise ValueError("unsupported qualification manifest version")
@@ -123,6 +123,14 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
         raise ValueError("qualification needs tune and positive/negative holdout cases")
     if projects["tune"] & projects["holdout"] or sessions["tune"] & sessions["holdout"]:
         raise ValueError("qualification leaks project/session groups between splits")
+    if case_map is not None:
+        for annotation in annotations:
+            recorded = case_map[annotation["id"]]
+            if (recorded.get("project_id") != annotation["project_id"]
+                    or recorded.get("session_id") != annotation["session_id"]):
+                raise ValueError(
+                    f"qualification {annotation['id']} group identity disagrees with frozen dataset"
+                )
     return {
         "heldout_ids": heldout,
         "thresholds": thresholds,
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index e34d5cb4654..538a214ac27 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -29,6 +29,10 @@ def setUp(self):
             {"id": "holdout-negative", "labels": [], "context_needed": False,
              "current_prompt": "Show a greeting", "turns": []},
         ]
+        for case in self.dataset:
+            holdout = case["id"].startswith("holdout")
+            case["project_id"] = "project-b" if holdout else "project-a"
+            case["session_id"] = "session-b" if holdout else "session-a"
         self.dataset_bytes = json.dumps(self.dataset).encode("utf-8")
         self.dataset_sha = qualification.sha256_bytes(self.dataset_bytes)
         annotations = []
@@ -72,6 +76,30 @@ def test_positive_and_negative_holdout_is_separate_from_tuning(self):
         self.assertEqual(verified["thresholds"]["impact"], 0.6)
         self.assertEqual(len(verified["annotations"]), 4)
 
+    def test_group_identity_cannot_be_invented_by_the_annotation_manifest(self):
+        for field in ("project_id", "session_id"):
+            with self.subTest(field=field):
+                forged = copy.deepcopy(self.manifest)
+                forged["annotations"][2][field] = "invented-" + field
+                with self.assertRaisesRegex(ValueError, "group identity disagrees with frozen dataset"):
+                    self.valid(forged)
+                missing_source = copy.deepcopy(self.dataset)
+                del missing_source[2][field]
+                with self.assertRaisesRegex(ValueError, "group identity disagrees with frozen dataset"):
+                    self.valid(cases=missing_source)
+
+    def test_self_consistent_rehashed_metadata_cannot_hide_group_forgery(self):
+        forged = copy.deepcopy(self.manifest)
+        forged["annotations"][2]["project_id"] = "invented-independent-project"
+        forged_sha = qualification.sha256_json(forged)
+        result = self.signed_result()
+        result["experiment"]["qualification_manifest_sha256"] = forged_sha
+        result["experiment_sha256"] = score.evidence_digest(result["experiment"])
+        with self.assertRaisesRegex(ValueError, "group identity disagrees with frozen dataset"):
+            qualification.qualified_report(
+                result, forged, forged_sha, self.questions, self.dataset_bytes
+            )
+
     def test_near_neighbor_session_or_project_leak_refuses_entire_split(self):
         for key, value in (("project_id", "project-a"), ("session_id", "session-a")):
             with self.subTest(key=key):

From d656712ccff897cbab79ace4a6e26c38a927f928 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 19:18:48 -0800
Subject: [PATCH 08/76] fix(evals): fail qualification commands on rejected
 evidence

Refs #4535

Return nonzero after publishing a failed frozen qualification report and
name failed checks on stderr. Preserve machine-readable measurements and
correct the held-out refusal command to include the original dataset.

Add Bazel-wired CLI regressions for every qualification check, replacement
of previous passing reports, invalid evidence, and the passing control.
Preserve the concurrent frozen split-group fix in parent 711a18f8.
Tests require Bazel execution; no local test pass or Kev inference claimed.
Keep this decision-model work DRAFT and NO MERGE.
---
 evals/intent_engine/BUILD.bazel               |  15 +++
 evals/intent_engine/score.py                  |   8 +-
 evals/intent_engine/test_qualification_cli.py | 114 ++++++++++++++++++
 3 files changed, 136 insertions(+), 1 deletion(-)
 create mode 100644 evals/intent_engine/test_qualification_cli.py

diff --git a/evals/intent_engine/BUILD.bazel b/evals/intent_engine/BUILD.bazel
index e2e81e0c16d..73211f61518 100644
--- a/evals/intent_engine/BUILD.bazel
+++ b/evals/intent_engine/BUILD.bazel
@@ -28,6 +28,21 @@ py_test(
     main = "test_qualification.py",
 )
 
+py_test(
+    name = "qualification_cli_test",
+    size = "small",
+    srcs = [
+        "common.py",
+        "qualification.py",
+        "run_http.py",
+        "score.py",
+        "test_qualification.py",
+        "test_qualification_cli.py",
+    ],
+    imports = ["."],
+    main = "test_qualification_cli.py",
+)
+
 py_test(
     name = "scoring_test",
     size = "small",
diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py
index 844f8408845..ded55d94f92 100644
--- a/evals/intent_engine/score.py
+++ b/evals/intent_engine/score.py
@@ -319,6 +319,12 @@ def ch(rs):
         report = qualified_report(result, manifest, manifest_sha, noul_questions(), dataset_bytes)
         publish_summary(os.path.join(here, "qualification_summary.json"), report)
         print(json.dumps(report, indent=2, allow_nan=False))
+        # A completed evaluation is not necessarily a qualified model. Keep
+        # the measurements available while failing the calling automation.
+        if not report["qualified_against_frozen_tolerances"]:
+            failed = ", ".join(name for name, passed in report["checks"].items() if not passed)
+            print(f"qualification failed frozen checks: {failed}", file=sys.stderr)
+            sys.exit(1)
         sys.exit(0)
     fixed = json.load(open(os.path.join(here, "fixed_thresholds.json"))) if os.path.exists(os.path.join(here, "fixed_thresholds.json")) else None
     summary = {}; allres = {}
@@ -326,7 +332,7 @@ def ch(rs):
         res = json.load(open(f))
         if (isinstance(res.get("experiment"), dict)
                 and "qualification_manifest_sha256" in res["experiment"]):
-            raise ValueError("held-out results require --qualified MANIFEST RESULT; refuse in-sample scoring")
+            raise ValueError("held-out results require --qualified MANIFEST DATASET RESULT; refuse in-sample scoring")
         if res["config"] in allres:
             raise ValueError("duplicate configuration %r in result files" % res["config"])
         allres[res["config"]] = res
diff --git a/evals/intent_engine/test_qualification_cli.py b/evals/intent_engine/test_qualification_cli.py
new file mode 100644
index 00000000000..d7ceb454db6
--- /dev/null
+++ b/evals/intent_engine/test_qualification_cli.py
@@ -0,0 +1,114 @@
+"""Exercise qualification exit status through the real scoring entry point.
+
+The shared cases are evaluator fixtures, not model-qualification evidence.
+"""
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import test_qualification as fixtures
+
+
+class QualificationCliTests(unittest.TestCase):
+    def setUp(self):
+        self.fixture = fixtures.QualificationTests()
+        self.fixture.setUp()
+        staging = tempfile.TemporaryDirectory()
+        self.addCleanup(staging.cleanup)
+        self.root = Path(staging.name)
+        source = Path(__file__).resolve().parent
+        for name in ("common.py", "qualification.py", "run_http.py", "score.py"):
+            shutil.copyfile(source / name, self.root / name)
+        (self.root / "manifest.json").write_bytes(self.fixture.manifest_bytes)
+        (self.root / "dataset.json").write_bytes(self.fixture.dataset_bytes)
+
+    def run_score(self, result):
+        (self.root / "result.json").write_text(
+            json.dumps(result, allow_nan=False), encoding="utf-8"
+        )
+        return subprocess.run(
+            [sys.executable, str(self.root / "score.py"), "--qualified",
+             str(self.root / "manifest.json"), str(self.root / "dataset.json"),
+             str(self.root / "result.json")],
+            cwd=self.root, capture_output=True, text=True, timeout=30,
+        )
+
+    def published_report(self, process):
+        report = json.loads(process.stdout)
+        published = json.loads(
+            (self.root / "qualification_summary.json").read_text(encoding="utf-8")
+        )
+        self.assertEqual(published, report)
+        self.assertEqual(report["cases"], 2)
+        self.assertFalse((self.root / "summary.json").exists())
+        return report
+
+    def test_passing_qualification_exits_zero_with_matching_report(self):
+        process = self.run_score(self.fixture.signed_result())
+        self.assertEqual(process.returncode, 0, process.stderr)
+        self.assertEqual(process.stderr, "")
+        report = self.published_report(process)
+        self.assertTrue(report["qualified_against_frozen_tolerances"])
+        self.assertTrue(all(report["checks"].values()))
+
+    def test_failed_checks_exit_nonzero_and_replace_a_previous_passing_report(self):
+        failures = (
+            ("negative_false_hint", "none_false_hint_rate",
+             lambda r: r["rows"][1]["probs"].update(impact=0.9)),
+            ("extra_positive_hint", "unwanted_hint_rate",
+             lambda r: r["rows"][0]["probs"].update(call_graph=0.9)),
+            ("missed_positive", "positive_case_recall",
+             lambda r: r["rows"][0]["probs"].update(impact=0.1)),
+            ("slow_response", "p95_ms", lambda r: r["rows"][1].update(ms=21)),
+            ("unknown_truncation", "untruncated_input_evidence",
+             lambda r: r["rows"][0].update(dropped=None)),
+            ("truncated_input", "untruncated_input_evidence",
+             lambda r: r["rows"][0].update(dropped=1)),
+        )
+        for name, failed_check, mutate in failures:
+            with self.subTest(name=name):
+                passing = self.run_score(self.fixture.signed_result())
+                self.assertEqual(passing.returncode, 0, passing.stderr)
+                self.assertTrue(
+                    self.published_report(passing)["qualified_against_frozen_tolerances"]
+                )
+                result = self.fixture.signed_result()
+                mutate(result)
+                process = self.run_score(result)
+                self.assertEqual(process.returncode, 1, process.stderr)
+                report = self.published_report(process)
+                self.assertFalse(report["qualified_against_frozen_tolerances"])
+                self.assertFalse(report["checks"][failed_check])
+                self.assertIn("qualification failed frozen checks:", process.stderr)
+                self.assertIn(failed_check, process.stderr)
+
+    def test_invalid_dataset_fails_without_publishing_a_qualification(self):
+        (self.root / "dataset.json").write_bytes(self.fixture.dataset_bytes + b"\n")
+        process = self.run_score(self.fixture.signed_result())
+        self.assertNotEqual(process.returncode, 0)
+        self.assertIn("dataset digest differs from supplied evidence", process.stderr)
+        self.assertEqual(process.stdout, "")
+        self.assertFalse((self.root / "qualification_summary.json").exists())
+
+    def test_unqualified_mode_reports_the_complete_required_command(self):
+        results = self.root / "results"
+        results.mkdir()
+        (results / "heldout.json").write_text(
+            json.dumps(self.fixture.signed_result(), allow_nan=False), encoding="utf-8"
+        )
+        process = subprocess.run(
+            [sys.executable, str(self.root / "score.py")],
+            cwd=self.root, capture_output=True, text=True, timeout=30,
+        )
+        self.assertNotEqual(process.returncode, 0)
+        self.assertIn("--qualified MANIFEST DATASET RESULT", process.stderr)
+        self.assertFalse((self.root / "summary.json").exists())
+        self.assertFalse((self.root / "qualification_summary.json").exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

From 1259eef20feb7e750ad587895c5038bbd0b0ae1f Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 20:13:03 -0800
Subject: [PATCH 09/76] fix(evals): freeze client calibration before held-out
 inference

Refs #4535

Require an explicit null-or-positive client calibration in the frozen
qualification manifest. Refuse a changed --noul-temperature before any
network work, and reject an otherwise rehashed result with a different
experiment calibration before scoring.

Cover malformed declarations, post-hoc rehashing, no-network refusal and
the matching calibrated HTTP path with behavior-based regressions.
Historical nine-question evaluation remains unqualified for runtime parity.
DRAFT ONLY; NO MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md        |  9 +++-
 evals/intent_engine/qualification.py      | 19 +++++++
 evals/intent_engine/run_http.py           |  2 +
 evals/intent_engine/test_qualification.py | 66 +++++++++++++++++++++++
 4 files changed, 95 insertions(+), 1 deletion(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 6081d2aee42..250edc455c9 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -120,6 +120,11 @@ Create a separate JSON annotation manifest **before running inference** with
 (`dataset_sha256`), and `questions_sha256` equal to the canonical
 `qualification.sha256_json(common.noul_questions())` of the runner's exact
 question dictionary. This last digest must change with question text. Include
+`noul_temperature` as JSON `null` for no client calibration or a finite,
+positive number matching the runner's `--noul-temperature` argument. This
+client-side calibration must be frozen alongside the thresholds: changing it
+after seeing holdout predictions would select a new effective decision boundary
+without changing the threshold manifest. Include
 `thresholds` with one frozen probability threshold per requested question and
 `error_tolerances` with all four numbers:
 `max_none_false_hint_rate` (0–1), `max_unwanted_hint_rate` (0–1),
@@ -145,7 +150,9 @@ from model predictions or adoption events.
 Run with the pinned remote serving declaration as before, adding
 `--dataset /path/to/original-cases.json` and
 `--qualification-manifest /path/to/frozen.json`. The dataset file's bytes
-are bound to the run and must be supplied again when scoring. The runner refuses an
+are bound to the run and must be supplied again when scoring. The runner also
+refuses a missing or mismatched frozen client-calibration setting before any
+HTTP request and the scorer rechecks it against the experiment identity. The runner refuses an
 additional `--subset`, validates every case and its group before network
 traffic, infers **only the frozen held-out cases**, and binds the raw manifest's
 SHA-256 to the experiment identity and resumable partial log. A changed
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index d9b9a4f6ec7..4c9c252b475 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -48,6 +48,20 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
                 or not math.isfinite(value) or not 0 <= value <= 1):
             raise ValueError("qualification thresholds must be finite probabilities")
 
+    # Freeze client-side probability calibration before any held-out inference.
+    # Otherwise an operator can pick a post-hoc temperature while preserving
+    # the same question, threshold, and holdout identities.
+    if "noul_temperature" not in manifest:
+        raise ValueError("qualification requires frozen client calibration")
+    client_temperature = manifest["noul_temperature"]
+    if client_temperature is not None and (
+        isinstance(client_temperature, bool)
+        or not isinstance(client_temperature, (int, float))
+        or not math.isfinite(client_temperature)
+        or client_temperature <= 0
+    ):
+        raise ValueError("qualification client calibration must be null or positive finite")
+
     tolerances = manifest.get("error_tolerances")
     required = ("max_none_false_hint_rate", "max_unwanted_hint_rate", "min_positive_case_recall", "max_p95_ms")
     if not isinstance(tolerances, dict) or set(tolerances) != set(required):
@@ -134,6 +148,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     return {
         "heldout_ids": heldout,
         "thresholds": thresholds,
+        "noul_temperature": client_temperature,
         "error_tolerances": tolerances,
         "annotations": {a["id"]: a for a in annotations},
         "application": application,
@@ -219,6 +234,10 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
                 f"qualification request fingerprint disagrees with frozen renderer for {row['id']}"
             )
 
+    if ("noul_temperature" not in experiment
+            or experiment["noul_temperature"] != validated["noul_temperature"]):
+        raise ValueError("qualification client calibration differs from frozen manifest")
+
     scores = score_config(result, fixed_t=validated["thresholds"])
     fixed = scores["fixed"]["all"]
     latency = scores["latency"]
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 4dad799d8a8..8e6ec05ce54 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -398,6 +398,8 @@ def main(argv=None):
             raise ValueError("qualification selects its own frozen holdout; --subset is not allowed")
         manifest, qualification_sha = load_manifest(args.qualification_manifest)
         qualified = validate_manifest(manifest, dataset_sha, questions, dataset)
+        if args.noul_temperature != qualified["noul_temperature"]:
+            raise ValueError("qualification client calibration differs from frozen manifest")
         keep = set(qualified["heldout_ids"])
         dataset = [case for case in dataset if case["id"] in keep]
     if args.subset:
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 538a214ac27..b135c2c07fd 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -51,6 +51,7 @@ def setUp(self):
             "version": 1, "application": "historical-intent-evaluation",
             "dataset_sha256": self.dataset_sha,
             "questions_sha256": qualification.sha256_json(self.questions),
+            "noul_temperature": None,
             "thresholds": {key: 0.6 for key in self.questions},
             "error_tolerances": {
                 "max_none_false_hint_rate": 0.0,
@@ -179,6 +180,10 @@ def test_question_or_threshold_drift_fails_before_inference(self):
             ("unfrozen_tolerance", lambda m: m["error_tolerances"].pop("max_p95_ms"), "tolerances"),
             ("unfrozen_unwanted", lambda m: m["error_tolerances"].pop("max_unwanted_hint_rate"), "tolerances"),
             ("dataset", lambda m: m.update(dataset_sha256="f" * 64), "dataset digest mismatch"),
+            ("missing_calibration", lambda m: m.pop("noul_temperature"), "frozen client calibration"),
+            ("nonfinite_calibration", lambda m: m.update(noul_temperature=float("nan")), "client calibration"),
+            ("zero_calibration", lambda m: m.update(noul_temperature=0), "client calibration"),
+            ("boolean_calibration", lambda m: m.update(noul_temperature=True), "client calibration"),
         ):
             with self.subTest(name=name):
                 changed = copy.deepcopy(self.manifest)
@@ -280,6 +285,67 @@ def test_qualified_reporting_refuses_tampered_labels_and_unbound_results(self):
                 with self.assertRaisesRegex(ValueError, pattern):
                     qualification.qualified_report(result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes)
 
+
+    def test_frozen_client_calibration_cannot_be_changed_by_rehashing(self):
+        # Rehashing both the declared manifest and experiment must not make
+        # an unfrozen post-hoc client temperature qualify.
+        for temperature in (0.5, 1.25):
+            with self.subTest(temperature=temperature):
+                changed = copy.deepcopy(self.manifest)
+                changed["noul_temperature"] = temperature
+                changed_sha = qualification.sha256_json(changed)
+                forged = self.signed_result()
+                forged["experiment"]["qualification_manifest_sha256"] = changed_sha
+                forged["experiment_sha256"] = score.evidence_digest(forged["experiment"])
+                with self.assertRaisesRegex(ValueError, "client calibration differs"):
+                    qualification.qualified_report(
+                        forged, changed, changed_sha, self.questions, self.dataset_bytes
+                    )
+
+    def test_http_runner_requires_frozen_client_calibration_before_network(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            manifest_path = here / "manifest.json"
+            manifest_path.write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps({
+                "checkpoint": "pinned", "server_revision": "r1",
+                "backend": "cpu", "dtype": "float32", "temperature": 2.41,
+            }))
+            args = [
+                "--url", "http://127.0.0.1:8009/v1/systemone",
+                "--tag", "calibration",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(manifest_path),
+                "--noul-temperature", "1.25", "B_noul_text",
+            ]
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post") as blocked:
+                    with self.assertRaisesRegex(ValueError, "client calibration differs"):
+                        run_http.main(args)
+                    blocked.assert_not_called()
+                self.assertFalse((here / "results").exists())
+                frozen = copy.deepcopy(self.manifest)
+                frozen["noul_temperature"] = 1.25
+                manifest_path.write_text(json.dumps(frozen))
+                answer = {"answers": {
+                    key: {"type": "noul", "noul": 0.5}
+                    for key in self.questions
+                }}
+                with patch.object(run_http, "post", return_value=(answer, 12.0)) as infer:
+                    run_http.main(args)
+                    self.assertEqual(infer.call_count, 2)
+                result = json.loads(
+                    (here / "results" / "calibration_B_noul_text.json").read_text()
+                )
+                self.assertEqual(result["experiment"]["noul_temperature"], 1.25)
+                self.assertEqual(result["rows"][0]["probs_raw"]["impact"], 0.5)
+                report = qualification.qualified_report(
+                    result, frozen, qualification.sha256_bytes(manifest_path.read_bytes()),
+                    self.questions, self.dataset_bytes,
+                )
+                self.assertFalse(report["qualified_against_frozen_tolerances"])
+
     def test_http_runner_asks_only_pre_frozen_holdout_and_resumes_with_exact_identity(self):
         with tempfile.TemporaryDirectory() as temp:
             here = Path(temp)

From 1e27a2c648654ce433aedd30602ec4c85a6b5eb4 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 21:34:59 -0800
Subject: [PATCH 10/76] fix(evals): refuse non-prefix resumed inference rows

Refs #4535

Only a contiguous prefix of the frozen HTTP case order can be resumed.
Previously a partial log containing a later case ahead of an earlier one
could skip and reorder inference while preserving apparently valid hashes,
misrepresenting the sequence used for cold/warm measurements.

Add a falsifiable HTTP-runner regression for skipped and reversed rows,
no network or output mutation on refusal, and genuine-prefix recovery.
Document the resume invariant. This is partial evaluation integrity only;
DRAFT / NEVER MERGE. No Kev qualification is claimed.
---
 evals/intent_engine/HTTP_RUNNER.md   |  6 ++--
 evals/intent_engine/run_http.py      |  7 +++++
 evals/intent_engine/test_run_http.py | 45 ++++++++++++++++++++++++++++
 3 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 250edc455c9..418d964e80c 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -43,8 +43,10 @@ request fingerprint. The endpoint itself is not copied into the identity.
 An interrupted run resumes only with matching identity. Changed questions,
 question order, renderer output, labels, subset contents, or serving settings
 refuse before any HTTP request or append. Headerless old logs, incomplete rows,
-duplicate/foreign cases, and malformed probability maps also refuse; no migration
-or silent salvage is performed. Choose a new tag for another experiment or an
+duplicate/foreign cases, and malformed probability maps also refuse. A resumable
+log must be a contiguous prefix of the frozen case order: a row for a later case
+cannot be used to skip or reorder earlier inference and distort cold/warm timing.
+No migration or silent salvage is performed. Choose a new tag for another experiment or an
 old unidentifiable log. Completed output files are never overwritten.
 
 An exclusive lock on the partial log rejects concurrent writers for the same tag/configuration. Rows are flushed after each response;
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 8e6ec05ce54..06f46475a31 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -310,6 +310,7 @@ def resume_rows(partial, header, cases, requests, questions, temperature):
     if not first.endswith("\n") or json.loads(first) != header:
         raise ValueError("partial output belongs to another or unrecorded experiment; use a new tag")
     done = {}
+    ordered_ids = tuple(cases)
     for line_number, line in enumerate(partial, 2):
         if not line.endswith("\n"):
             raise ValueError(f"partial output has an incomplete row at line {line_number}")
@@ -317,6 +318,12 @@ def resume_rows(partial, header, cases, requests, questions, temperature):
         case_id = row.get("id") if isinstance(row, dict) else None
         if not isinstance(case_id, str) or case_id not in cases or case_id in done:
             raise ValueError(f"partial output has an unknown or duplicate case at line {line_number}")
+        # A real interrupted run appends only a prefix in request order.
+        # Skipped or reordered rows can misrepresent cold/warm inference order.
+        if case_id != ordered_ids[len(done)]:
+            raise ValueError(
+                f"partial output is not a contiguous prefix of frozen case order at line {line_number}"
+            )
         case = cases[case_id]
         if (row.get("request_sha256") != requests[case_id][1]
                 or row.get("labels") != case["labels"]
diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index 6d5e710698c..7694063fd0e 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -129,6 +129,51 @@ def test_exact_resume_only_requests_the_missing_case(self):
         self.assertFalse(self.partial.exists())
 
 
+    def test_resume_refuses_out_of_order_or_skipped_rows_before_network(self):
+        self.interrupt()
+        original = self.partial.read_bytes()
+        header, first_json = original.decode("utf-8").splitlines()
+        first = json.loads(first_json)
+        second = self.dataset[1]
+        body = {
+            "state": run_http.STATES[self.config](second),
+            "questions": run_http.noul_questions(),
+        }
+        forged_second = {
+            **first,
+            "id": second["id"],
+            "labels": second["labels"],
+            "context_needed": second["context_needed"],
+            "request_sha256": hashlib.sha256(
+                run_http.request_bytes(body)
+            ).hexdigest(),
+        }
+        for name, rows in (
+            ("skipped_first", [forged_second]),
+            ("reversed_prefix", [forged_second, first]),
+        ):
+            with self.subTest(name=name):
+                self.partial.write_text(
+                    header + "\n" + "".join(json.dumps(row) + "\n" for row in rows)
+                )
+                before = self.partial.read_bytes()
+                with patch.object(run_http, "post") as post:
+                    with self.assertRaisesRegex(ValueError, "contiguous prefix"):
+                        self.run_main()
+                    post.assert_not_called()
+                self.assertEqual(self.partial.read_bytes(), before)
+                self.assertFalse(self.output.exists())
+
+        # A genuine interrupted prefix must still be resumable.
+        self.partial.write_bytes(original)
+        with patch.object(run_http, "post", side_effect=self.response) as post:
+            self.run_main()
+        post.assert_called_once()
+        self.assertEqual(
+            [row["id"] for row in json.loads(self.output.read_text())["rows"]],
+            ["first", "second"],
+        )
+
     def test_completed_rows_cannot_be_relabelled_for_scoring(self):
         with patch.object(run_http, "post", side_effect=self.response):
             self.run_main()

From f449431240f9f7e8175af0306792fc34a23da989 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 22:30:37 -0800
Subject: [PATCH 11/76] fix(evals): freeze held-out HTTP evaluation arm

Refs #4535

Holdout qualification now precommits the selected state renderer, optional
model selector, endpoint digest, and entire serving declaration digest.
Reject alternative renderers, model switches and server/endpoint drift before
inference, and independently recheck result identities when scoring.

Add falsifiable regression cases for malformed arms, zero-network refusal
and rehashed result forgery. Preserve the nine-question historical status:
this does not attest server identity or establish Scout runtime parity.
Keep this Kev/decision-model PR DRAFT and NEVER MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md        |   8 ++
 evals/intent_engine/qualification.py      |  23 +++++
 evals/intent_engine/run_http.py           |   6 ++
 evals/intent_engine/test_qualification.py | 115 +++++++++++++++++-----
 4 files changed, 126 insertions(+), 26 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 418d964e80c..0e45cb4f649 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -122,6 +122,14 @@ Create a separate JSON annotation manifest **before running inference** with
 (`dataset_sha256`), and `questions_sha256` equal to the canonical
 `qualification.sha256_json(common.noul_questions())` of the runner's exact
 question dictionary. This last digest must change with question text. Include
+an `arm` object predeclaring exactly one `config` renderer, a nullable
+`requested_model`, `endpoint_sha256` (SHA-256 of the exact URL bytes), and
+`server_config_sha256` (the sorted-JSON digest of the complete serving
+configuration, via `qualification.sha256_json(server_config)`). The runner
+refuses additional renderers or changed serving/model/endpoint declarations
+**before inference**; scoring rechecks these identities. This prevents choosing
+the best evaluation arm only after examining held-out predictions. These are
+operator declarations, not remote checkpoint attestation. Include
 `noul_temperature` as JSON `null` for no client calibration or a finite,
 positive number matching the runner's `--noul-temperature` argument. This
 client-side calibration must be frozen alongside the thresholds: changing it
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 4c9c252b475..6af364b057c 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -40,6 +40,22 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     if manifest.get("questions_sha256") != sha256_json(questions):
         raise ValueError("qualification question vocabulary or instruction mismatch")
 
+    # A held-out split is not independent if the evaluator can try multiple
+    # renderers, model selectors or serving declarations and pick the winner.
+    arm = manifest.get("arm")
+    required_arm = {"config", "requested_model", "endpoint_sha256", "server_config_sha256"}
+    if not isinstance(arm, dict) or set(arm) != required_arm:
+        raise ValueError("qualification requires one frozen HTTP arm")
+    if not isinstance(arm["config"], str) or not arm["config"].strip():
+        raise ValueError("qualification HTTP arm requires a renderer config")
+    model = arm["requested_model"]
+    if model is not None and (not isinstance(model, str) or not model.strip()):
+        raise ValueError("qualification HTTP arm requires a valid requested model or null")
+    for field in ("endpoint_sha256", "server_config_sha256"):
+        value = arm[field]
+        if not isinstance(value, str) or not _DIGEST.fullmatch(value):
+            raise ValueError(f"qualification HTTP arm has invalid {field}")
+
     thresholds = manifest.get("thresholds")
     if not isinstance(thresholds, dict) or set(thresholds) != set(questions):
         raise ValueError("qualification thresholds must cover every requested question")
@@ -147,6 +163,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
                 )
     return {
         "heldout_ids": heldout,
+        "arm": arm,
         "thresholds": thresholds,
         "noul_temperature": client_temperature,
         "error_tolerances": tolerances,
@@ -210,6 +227,12 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     if experiment.get("qualification_manifest_sha256") != manifest_sha:
         raise ValueError("qualification experiment is not bound to this frozen manifest")
     validated = validate_manifest(manifest, dataset_sha, questions, cases)
+    arm = validated["arm"]
+    if (experiment.get("config") != arm["config"]
+            or experiment.get("requested_model") != arm["requested_model"]
+            or experiment.get("endpoint_sha256") != arm["endpoint_sha256"]
+            or sha256_json(server) != arm["server_config_sha256"]):
+        raise ValueError("qualification HTTP arm differs from frozen manifest")
     rows = result.get("rows")
     if not isinstance(rows, list) or [r.get("id") for r in rows if isinstance(r, dict)] != validated["heldout_ids"]:
         raise ValueError("qualification result differs from frozen held-out case order")
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 06f46475a31..1ee3802852b 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -407,6 +407,12 @@ def main(argv=None):
         qualified = validate_manifest(manifest, dataset_sha, questions, dataset)
         if args.noul_temperature != qualified["noul_temperature"]:
             raise ValueError("qualification client calibration differs from frozen manifest")
+        arm = qualified["arm"]
+        if (args.configs != [arm["config"]]
+                or args.send_model != arm["requested_model"]
+                or hashlib.sha256(args.url.encode("utf-8")).hexdigest() != arm["endpoint_sha256"]
+                or digest(server_config) != arm["server_config_sha256"]):
+            raise ValueError("qualification HTTP arm differs from frozen manifest")
         keep = set(qualified["heldout_ids"])
         dataset = [case for case in dataset if case["id"] in keep]
     if args.subset:
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index b135c2c07fd..ea27fc51979 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -47,10 +47,20 @@ def setUp(self):
                 "verified_labels": case["labels"],
                 "verified_context_needed": case["context_needed"],
             })
+        self.endpoint = "http://127.0.0.1:8009/v1/systemone"
+        self.server_config = {
+            "checkpoint": "pinned", "server_revision": "r1",
+            "backend": "cpu", "dtype": "float32", "temperature": 2.41,
+        }
         self.manifest = {
             "version": 1, "application": "historical-intent-evaluation",
             "dataset_sha256": self.dataset_sha,
             "questions_sha256": qualification.sha256_json(self.questions),
+            "arm": {
+                "config": "B_noul_text", "requested_model": None,
+                "endpoint_sha256": qualification.sha256_bytes(self.endpoint.encode("utf-8")),
+                "server_config_sha256": qualification.sha256_json(self.server_config),
+            },
             "noul_temperature": None,
             "thresholds": {key: 0.6 for key in self.questions},
             "error_tolerances": {
@@ -209,6 +219,8 @@ def signed_result(self):
         experiment = {
             "version": 2,
             "config": "B_noul_text",
+            "requested_model": None,
+            "endpoint_sha256": self.manifest["arm"]["endpoint_sha256"],
             "case_ids": [row["id"] for row in rows],
             "dataset_sha256": self.dataset_sha,
             "case_annotations_sha256": score.evidence_digest([
@@ -218,11 +230,7 @@ def signed_result(self):
                 [r["id"], r["request_sha256"]] for r in rows
             ]),
             "noul_temperature": None,
-            "server_config": {
-                "checkpoint": "checkpoint-pinned-by-operator",
-                "server_revision": "immutable-server-revision",
-                "backend": "cpu", "dtype": "float32", "temperature": 2.41,
-            },
+            "server_config": copy.deepcopy(self.server_config),
             "qualification_manifest_sha256": self.manifest_sha,
         }
         return {
@@ -308,10 +316,7 @@ def test_http_runner_requires_frozen_client_calibration_before_network(self):
             (here / "dataset.json").write_bytes(self.dataset_bytes)
             manifest_path = here / "manifest.json"
             manifest_path.write_bytes(self.manifest_bytes)
-            (here / "server.json").write_text(json.dumps({
-                "checkpoint": "pinned", "server_revision": "r1",
-                "backend": "cpu", "dtype": "float32", "temperature": 2.41,
-            }))
+            (here / "server.json").write_text(json.dumps(self.server_config))
             args = [
                 "--url", "http://127.0.0.1:8009/v1/systemone",
                 "--tag", "calibration",
@@ -351,11 +356,7 @@ def test_http_runner_asks_only_pre_frozen_holdout_and_resumes_with_exact_identit
             here = Path(temp)
             (here / "dataset.json").write_bytes(self.dataset_bytes)
             (here / "manifest.json").write_bytes(self.manifest_bytes)
-            (here / "server.json").write_text(json.dumps({
-                "checkpoint": "example-pinned-checkpoint",
-                "server_revision": "example-pinned-server",
-                "backend": "example-backend", "dtype": "float32", "temperature": 2.41,
-            }))
+            (here / "server.json").write_text(json.dumps(self.server_config))
             calls = []
 
             def answer(url, body):
@@ -391,10 +392,7 @@ def test_runner_refuses_group_leakage_without_network_or_partial_output(self):
             changed = copy.deepcopy(self.manifest)
             changed["annotations"][2]["project_id"] = "project-a"
             (here / "manifest.json").write_text(json.dumps(changed))
-            (here / "server.json").write_text(json.dumps({
-                "checkpoint": "pinned", "server_revision": "r1", "backend": "cpu",
-                "dtype": "float32", "temperature": 2.41,
-            }))
+            (here / "server.json").write_text(json.dumps(self.server_config))
             args = [
                 "--url", "http://127.0.0.1:8009/v1/systemone",
                 "--tag", "refused",
@@ -423,10 +421,7 @@ def test_changed_manifest_cannot_resume_or_spend_inference_tokens(self):
             (here / "dataset.json").write_bytes(self.dataset_bytes)
             manifest_path = here / "manifest.json"
             manifest_path.write_bytes(self.manifest_bytes)
-            (here / "server.json").write_text(json.dumps({
-                "checkpoint": "pinned", "server_revision": "r1", "backend": "cpu",
-                "dtype": "float32", "temperature": 2.41,
-            }))
+            (here / "server.json").write_text(json.dumps(self.server_config))
             args = [
                 "--url", "http://127.0.0.1:8009/v1/systemone",
                 "--tag", "interrupted",
@@ -540,10 +535,7 @@ def test_runner_can_consume_explicit_frozen_input_file(self):
             dataset_path = here / "external-cases.json"
             dataset_path.write_bytes(self.dataset_bytes)
             (here / "manifest.json").write_bytes(self.manifest_bytes)
-            (here / "server.json").write_text(json.dumps({
-                "checkpoint": "pinned", "server_revision": "r1",
-                "backend": "cpu", "dtype": "float32", "temperature": 2.41,
-            }))
+            (here / "server.json").write_text(json.dumps(self.server_config))
             answer = {"answers": {
                 key: {"type": "noul", "noul": 0.5} for key in self.questions
             }}
@@ -567,5 +559,76 @@ def test_runner_can_consume_explicit_frozen_input_file(self):
             )
 
 
+
+    def test_frozen_http_arm_is_required_and_well_formed(self):
+        mutations = (
+            (lambda m: m.pop("arm"), "frozen HTTP arm"),
+            (lambda m: m["arm"].pop("config"), "frozen HTTP arm"),
+            (lambda m: m["arm"].update(config=""), "renderer config"),
+            (lambda m: m["arm"].update(requested_model=""), "requested model"),
+            (lambda m: m["arm"].update(endpoint_sha256="x" * 64), "endpoint_sha256"),
+            (lambda m: m["arm"].update(server_config_sha256=None), "server_config_sha256"),
+        )
+        for mutate, error in mutations:
+            with self.subTest(error=error):
+                manifest = copy.deepcopy(self.manifest)
+                mutate(manifest)
+                with self.assertRaisesRegex(ValueError, error):
+                    self.valid(manifest)
+
+    def test_frozen_http_arm_refuses_post_hoc_reselection_before_inference(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            server = here / "server.json"
+            normal = [
+                "--url", self.endpoint, "--tag", "locked-arm",
+                "--server-config", str(server),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "B_noul_text",
+            ]
+            trials = (
+                ("renderer", lambda a: a.__setitem__(-1, "A_noul_prompt")),
+                ("multiple-renderers", lambda a: a.append("A_noul_prompt")),
+                ("model", lambda a: a.__setitem__(slice(-1, -1), ["--send-model", "unfrozen-model"])),
+                ("endpoint", lambda a: a.__setitem__(1, "http://127.0.0.1:8010/v1/systemone")),
+            )
+            with patch.object(run_http, "HERE", str(here)):
+                for label, mutation in trials:
+                    with self.subTest(label=label):
+                        server.write_text(json.dumps(self.server_config))
+                        args = normal.copy()
+                        mutation(args)
+                        with patch.object(run_http, "post") as blocked:
+                            with self.assertRaisesRegex(ValueError, "HTTP arm differs"):
+                                run_http.main(args)
+                            blocked.assert_not_called()
+                        self.assertFalse((here / "results").exists())
+                drifted = copy.deepcopy(self.server_config)
+                drifted["checkpoint"] = "another-checkpoint"
+                server.write_text(json.dumps(drifted))
+                with patch.object(run_http, "post") as blocked:
+                    with self.assertRaisesRegex(ValueError, "HTTP arm differs"):
+                        run_http.main(normal)
+                    blocked.assert_not_called()
+                self.assertFalse((here / "results").exists())
+
+    def test_scorer_refuses_rehashed_unfrozen_http_arm(self):
+        mutations = (
+            lambda e: e.update(endpoint_sha256="f" * 64),
+            lambda e: e["server_config"].update(checkpoint="different-pinned-model"),
+        )
+        for mutate in mutations:
+            with self.subTest(mutate=mutate):
+                result = self.signed_result()
+                mutate(result["experiment"])
+                result["experiment_sha256"] = score.evidence_digest(result["experiment"])
+                with self.assertRaisesRegex(ValueError, "HTTP arm differs"):
+                    qualification.qualified_report(
+                        result, self.manifest, self.manifest_sha,
+                        self.questions, self.dataset_bytes,
+                    )
+
 if __name__ == "__main__":
     unittest.main()

From 3f1b3322a2bee174f634987b3489ebf22b14d772 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 23:12:25 -0800
Subject: [PATCH 12/76] fix(evals): freeze and verify held-out HTTP request
 byte limits

Refs #4535

Precommit a per-request byte ceiling before any inference or output, record
exact serialized input sizes and reconstruct them during qualification.
Refuse forged or omitted counts and oversized frozen inputs, including
bad resumable rows. Report min/p50/max/total request bytes without
conflating them with token counts or claiming runtime parity.

Add nonzero behavioral regressions for invalid budgets, pre-network denial,
rehashed oversize results and tampered byte counts. Decision-model DRAFT,
NO MERGE; real Kev inference and native Bazel verification remain pending.
---
 evals/intent_engine/HTTP_RUNNER.md        | 10 ++-
 evals/intent_engine/qualification.py      | 27 +++++++-
 evals/intent_engine/run_http.py           | 14 +++++
 evals/intent_engine/test_qualification.py | 76 +++++++++++++++++++++--
 4 files changed, 119 insertions(+), 8 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 0e45cb4f649..f5536fe0fa3 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -135,7 +135,12 @@ positive number matching the runner's `--noul-temperature` argument. This
 client-side calibration must be frozen alongside the thresholds: changing it
 after seeing holdout predictions would select a new effective decision boundary
 without changing the threshold manifest. Include
-`thresholds` with one frozen probability threshold per requested question and
+a required `max_request_bytes` positive integer upper bound on the exact UTF-8
+JSON request body. This precommitted byte limit is **not** the model's
+validated 8,192-token context limit: byte counts and token counts are different.
+The runner refuses oversized requests before any result file or model call;
+qualification independently reconstructs and verifies the byte count of every
+held-out request. Include `thresholds` with one frozen probability threshold per requested question and
 `error_tolerances` with all four numbers:
 `max_none_false_hint_rate` (0–1), `max_unwanted_hint_rate` (0–1),
 `min_positive_case_recall` (0–1), and `max_p95_ms` (nonnegative milliseconds).
@@ -178,7 +183,8 @@ bazel test //evals/intent_engine:qualification_test --test_output=errors
 This writes `evals/intent_engine/qualification_summary.json` separately from
 the historical `summary.json`, and reports only **frozen-threshold** held-out
 false hints, all-case unwanted suggestions, missed positive cases, coverage,
-abstention, positive-case recall, p50/p95 measured end-to-end HTTP latency and
+abstention, positive-case recall, reconstructed JSON request sizes
+(min/p50/max/total and the frozen byte ceiling), p50/p95 measured end-to-end HTTP latency and
 the predeclared tolerances. A passing report requires every held-out case to
 report measured zero dropped state tokens. Missing truncation telemetry is
 unknown, and a response measured after truncated input cannot prove the full
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 6af364b057c..ecc3dce05df 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -7,6 +7,7 @@
 import json
 import math
 import re
+import statistics
 
 _DIGEST = re.compile(r"[0-9a-f]{64}\Z")
 
@@ -56,6 +57,11 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
         if not isinstance(value, str) or not _DIGEST.fullmatch(value):
             raise ValueError(f"qualification HTTP arm has invalid {field}")
 
+    request_limit = manifest.get("max_request_bytes")
+    if (isinstance(request_limit, bool) or not isinstance(request_limit, int)
+            or request_limit <= 0):
+        raise ValueError("qualification requires a positive integer max_request_bytes")
+
     thresholds = manifest.get("thresholds")
     if not isinstance(thresholds, dict) or set(thresholds) != set(questions):
         raise ValueError("qualification thresholds must cover every requested question")
@@ -164,6 +170,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     return {
         "heldout_ids": heldout,
         "arm": arm,
+        "max_request_bytes": request_limit,
         "thresholds": thresholds,
         "noul_temperature": client_temperature,
         "error_tolerances": tolerances,
@@ -247,15 +254,24 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     if renderer is None:
         raise ValueError("qualification has an unsupported request renderer")
     case_map = {case["id"]: case for case in cases}
+    request_sizes = []
     for row in rows:
         body = {"state": renderer(case_map[row["id"]]), "questions": questions}
         if experiment.get("requested_model"):
             body["model"] = experiment["requested_model"]
-        expected = sha256_bytes(request_bytes(body))
-        if row.get("request_sha256") != expected:
+        rendered = request_bytes(body)
+        if row.get("request_sha256") != sha256_bytes(rendered):
             raise ValueError(
                 f"qualification request fingerprint disagrees with frozen renderer for {row['id']}"
             )
+        size = len(rendered)
+        if size > validated["max_request_bytes"]:
+            raise ValueError(f"qualification case {row['id']} exceeds frozen request byte limit")
+        recorded_size = row.get("request_size_bytes")
+        if (isinstance(recorded_size, bool) or not isinstance(recorded_size, int)
+                or recorded_size != size):
+            raise ValueError(f"qualification request byte count disagrees for {row['id']}")
+        request_sizes.append(size)
 
     if ("noul_temperature" not in experiment
             or experiment["noul_temperature"] != validated["noul_temperature"]):
@@ -298,6 +314,13 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
         "tolerances": tolerances,
         "fixed_threshold_metrics": fixed,
         "holdout_decisions": decisions,
+        "request_size_bytes": {
+            "min": min(request_sizes),
+            "p50": statistics.median(request_sizes),
+            "max": max(request_sizes),
+            "total": sum(request_sizes),
+            "frozen_limit": validated["max_request_bytes"],
+        },
         "latency": latency,
         "checks": checks,
         "qualified_against_frozen_tolerances": all(checks.values()),
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 1ee3802852b..feb6c6cdfd3 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -329,6 +329,10 @@ def resume_rows(partial, header, cases, requests, questions, temperature):
                 or row.get("labels") != case["labels"]
                 or row.get("context_needed") != case["context_needed"]):
             raise ValueError(f"partial output has mismatched case evidence for {case_id}")
+        if ("qualification_manifest_sha256" in header["experiment"]
+                and (isinstance(row.get("request_size_bytes"), bool)
+                     or row.get("request_size_bytes") != len(request_bytes(requests[case_id][0])))):
+            raise ValueError(f"partial output has mismatched request byte count for {case_id}")
         serialize_result(row, questions, temperature)
         done[case_id] = row
     return done
@@ -427,6 +431,15 @@ def main(argv=None):
     if not dataset:
         raise ValueError("the experiment must contain at least one case")
     cases = {case["id"]: case for case in dataset}
+    if qualification_sha is not None:
+        # Bound exact JSON requests before creating result files or calling the model.
+        renderer = STATES[qualified["arm"]["config"]]
+        for case in dataset:
+            body = {"state": renderer(case), "questions": questions}
+            if args.send_model:
+                body["model"] = args.send_model
+            if len(request_bytes(body)) > qualified["max_request_bytes"]:
+                raise ValueError(f"qualification case {case['id']} exceeds frozen request byte limit")
     os.makedirs(os.path.join(HERE, "results"), exist_ok=True)
     for cfg in args.configs:
         name = f"{args.tag}_{cfg}"
@@ -499,6 +512,7 @@ def main(argv=None):
                     raise ValueError("response usage must be an object or null")
                 row = {
                     "id": case["id"], "request_sha256": request_sha,
+                    "request_size_bytes": len(request_bytes(body)),
                     "labels": case["labels"], "context_needed": case["context_needed"],
                     "ms": ms, "server_ms": server_ms,
                     "input_tokens": usage.get("input_tokens"),
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index ea27fc51979..0d4562ae379 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -62,6 +62,7 @@ def setUp(self):
                 "server_config_sha256": qualification.sha256_json(self.server_config),
             },
             "noul_temperature": None,
+            "max_request_bytes": 65536,
             "thresholds": {key: 0.6 for key in self.questions},
             "error_tolerances": {
                 "max_none_false_hint_rate": 0.0,
@@ -205,16 +206,18 @@ def signed_result(self):
         rows = []
         for case in self.dataset[2:]:
             probs = {cat: (0.9 if cat in case["labels"] else 0.1) for cat in CATS}
+            rendered = run_http.request_bytes({
+                "state": run_http.STATES["B_noul_text"](case),
+                "questions": self.questions,
+            })
             rows.append({
                 "id": case["id"], "labels": case["labels"],
                 "context_needed": case["context_needed"],
                 "probs": probs,
                 "ms": 10 if case["labels"] else 20,
                 "dropped": 0,
-                "request_sha256": hashlib.sha256(run_http.request_bytes({
-                    "state": run_http.STATES["B_noul_text"](case),
-                    "questions": self.questions,
-                })).hexdigest(),
+                "request_sha256": hashlib.sha256(rendered).hexdigest(),
+                "request_size_bytes": len(rendered),
             })
         experiment = {
             "version": 2,
@@ -250,6 +253,10 @@ def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
         self.assertEqual(report["cases"], 2)
         self.assertEqual(report["latency"]["p50_ms"], 15)
         self.assertEqual(report["latency"]["p95_ms"], 20)
+        self.assertEqual(report["request_size_bytes"]["frozen_limit"], 65536)
+        self.assertEqual(report["request_size_bytes"]["total"], sum(
+            row["request_size_bytes"] for row in result["rows"]
+        ))
         self.assertEqual(report["holdout_decisions"]["coverage_rate"], 0.5)
         self.assertEqual(report["holdout_decisions"]["abstention_rate"], 0.5)
         self.assertEqual(report["holdout_decisions"]["unwanted_hint_rate"], 0)
@@ -630,5 +637,66 @@ def test_scorer_refuses_rehashed_unfrozen_http_arm(self):
                         self.questions, self.dataset_bytes,
                     )
 
+
+    def test_frozen_request_byte_budget_must_be_positive_integer(self):
+        for budget in (None, 0, -1, True, 0.5, "4096"):
+            with self.subTest(budget=budget):
+                changed = copy.deepcopy(self.manifest)
+                changed["max_request_bytes"] = budget
+                with self.assertRaisesRegex(ValueError, "positive integer max_request_bytes"):
+                    self.valid(changed)
+
+    def test_frozen_request_byte_budget_refuses_before_network(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            changed = copy.deepcopy(self.manifest)
+            changed["max_request_bytes"] = 1
+            (here / "manifest.json").write_text(json.dumps(changed))
+            args = [
+                "--url", self.endpoint, "--tag", "oversized",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "B_noul_text",
+            ]
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post") as post:
+                    with self.assertRaisesRegex(ValueError, "exceeds frozen request byte limit"):
+                        run_http.main(args)
+                    post.assert_not_called()
+                self.assertFalse((here / "results").exists())
+
+    def test_qualification_reconstructs_measured_request_sizes(self):
+        original = self.signed_result()
+        report = qualification.qualified_report(
+            original, self.manifest, self.manifest_sha,
+            self.questions, self.dataset_bytes,
+        )
+        self.assertGreater(report["request_size_bytes"]["min"], 0)
+        self.assertEqual(
+            report["request_size_bytes"]["max"],
+            max(row["request_size_bytes"] for row in original["rows"]),
+        )
+        for value in (None, True, original["rows"][0]["request_size_bytes"] + 1):
+            with self.subTest(forged_size=value):
+                forged = copy.deepcopy(original)
+                forged["rows"][0]["request_size_bytes"] = value
+                with self.assertRaisesRegex(ValueError, "request byte count disagrees"):
+                    qualification.qualified_report(
+                        forged, self.manifest, self.manifest_sha,
+                        self.questions, self.dataset_bytes,
+                    )
+        tighter = copy.deepcopy(self.manifest)
+        tighter["max_request_bytes"] = 1
+        tighter_sha = qualification.sha256_json(tighter)
+        forged = copy.deepcopy(original)
+        forged["experiment"]["qualification_manifest_sha256"] = tighter_sha
+        forged["experiment_sha256"] = score.evidence_digest(forged["experiment"])
+        with self.assertRaisesRegex(ValueError, "exceeds frozen request byte limit"):
+            qualification.qualified_report(
+                forged, tighter, tighter_sha, self.questions, self.dataset_bytes,
+            )
+
 if __name__ == "__main__":
     unittest.main()

From 990a88bc2299107f51a1e0905ce34039bf6246fb Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sat, 7 Nov 2026 23:13:19 -0800
Subject: [PATCH 13/76] test(evals): exercise measured request sizes on HTTP
 resume

Refs #4535

Assert the runner records exact transmitted JSON byte counts and refuses
a tampered resumable partial before issuing another request or changing
that evidence. Preserve successful interrupted-run recovery and the
existing frozen-manifest refusal checks.

DRAFT / NO MERGE; hosted native Bazel verification remains outstanding.
---
 evals/intent_engine/test_qualification.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 0d4562ae379..b2e7119fdd3 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -384,6 +384,10 @@ def answer(url, body):
                 run_http.main(args)
             self.assertEqual(len(calls), 2, "must exclude training/tune cases from inference")
             result = json.loads((here / "results" / "frozen_B_noul_text.json").read_text())
+            self.assertEqual(
+                [row["request_size_bytes"] for row in result["rows"]],
+                [len(run_http.request_bytes(body)) for body in calls],
+            )
             self.assertEqual([r["id"] for r in result["rows"]], [
                 "holdout-positive", "holdout-negative"
             ])
@@ -450,6 +454,17 @@ def test_changed_manifest_cannot_resume_or_spend_inference_tokens(self):
                 frozen_bytes = partial.read_bytes()
                 self.assertEqual(len(frozen_bytes.splitlines()), 2,
                                  "first accepted prediction must be durably resumable")
+                header_line, row_line = frozen_bytes.splitlines()
+                falsified = json.loads(row_line)
+                falsified["request_size_bytes"] += 1
+                corrupted_bytes = header_line + b"\n" + json.dumps(falsified).encode("utf-8") + b"\n"
+                partial.write_bytes(corrupted_bytes)
+                with patch.object(run_http, "post") as blocked:
+                    with self.assertRaisesRegex(ValueError, "mismatched request byte count"):
+                        run_http.main(args)
+                    blocked.assert_not_called()
+                self.assertEqual(partial.read_bytes(), corrupted_bytes)
+                partial.write_bytes(frozen_bytes)
                 changed = copy.deepcopy(self.manifest)
                 changed["error_tolerances"]["max_p95_ms"] = 21.0
                 manifest_path.write_text(json.dumps(changed))

From ed80f99cef1d5883f6c1161e76ed4faef263977f Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 00:47:04 -0800
Subject: [PATCH 14/76] fix(evals): require consistent server-reported model
 evidence

Refs #4535

Record the model name returned by the HTTP endpoint per evaluated case.
Refuse a contradictory explicit model selector before recording a row;
fail frozen qualification if a model name is absent, changes between
holdout cases, or disagrees with its precommitted requested selector.

Add behavioral regression tests for missing, malformed, switched and
contradictory reported models, plus the passing control. Self-reported
names are not authenticated checkpoint attestations. Decision-model
work remains DRAFT and must never merge.
---
 evals/intent_engine/HTTP_RUNNER.md        | 16 +++--
 evals/intent_engine/qualification.py      | 18 ++++-
 evals/intent_engine/run_http.py           | 11 +++
 evals/intent_engine/test_qualification.py | 83 ++++++++++++++++++++++-
 4 files changed, 122 insertions(+), 6 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index f5536fe0fa3..ac4c9fd3d2d 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -140,8 +140,16 @@ JSON request body. This precommitted byte limit is **not** the model's
 validated 8,192-token context limit: byte counts and token counts are different.
 The runner refuses oversized requests before any result file or model call;
 qualification independently reconstructs and verifies the byte count of every
-held-out request. Include `thresholds` with one frozen probability threshold per requested question and
-`error_tolerances` with all four numbers:
+held-out request. Every HTTP result also records the model name *reported by
+the server*, including explicit null when the endpoint omits it. A frozen-arm
+qualification fails if this name is missing, differs among held-out responses,
+or contradicts a non-null requested model selector. For an explicit selector,
+a contradictory response is rejected before appending the row. The reported
+name is **not** cryptographic server identity or checkpoint attestation; a
+remote service can misreport it.
+
+Include `thresholds` with one frozen probability threshold per requested
+question and `error_tolerances` with all four numbers:
 `max_none_false_hint_rate` (0–1), `max_unwanted_hint_rate` (0–1),
 `min_positive_case_recall` (0–1), and `max_p95_ms` (nonnegative milliseconds).
 The unwanted-hint budget counts an incorrect extra suggestion on a positive
@@ -186,8 +194,8 @@ false hints, all-case unwanted suggestions, missed positive cases, coverage,
 abstention, positive-case recall, reconstructed JSON request sizes
 (min/p50/max/total and the frozen byte ceiling), p50/p95 measured end-to-end HTTP latency and
 the predeclared tolerances. A passing report requires every held-out case to
-report measured zero dropped state tokens. Missing truncation telemetry is
-unknown, and a response measured after truncated input cannot prove the full
+report measured zero dropped state tokens and a consistent self-reported
+model name. Missing truncation telemetry is unknown, and a response measured after truncated input cannot prove the full
 frozen request was evaluated; the measurements remain in the failing report. Scoring reads the original frozen case bytes,
 checks all reviewed labels/splits against them and recreates each HTTP
 request fingerprint with the runner's own renderer. Self-consistent result
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index ecc3dce05df..eb4160ed13d 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -277,6 +277,20 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
             or experiment["noul_temperature"] != validated["noul_temperature"]):
         raise ValueError("qualification client calibration differs from frozen manifest")
 
+    # The serving declaration is operator-supplied. Compare it to the model
+    # reported by each actual HTTP response so a changed serving default or
+    # a mismatched explicit selector cannot silently qualify.
+    observed_models = [row.get("observed_model") for row in rows]
+    reported_model = observed_models[0] if (
+        all(isinstance(value, str) and value.strip()
+            and value == observed_models[0] for value in observed_models)
+    ) else None
+    reported_model_consistent = (
+        reported_model is not None
+        and (arm["requested_model"] is None
+             or reported_model == arm["requested_model"])
+    )
+
     scores = score_config(result, fixed_t=validated["thresholds"])
     fixed = scores["fixed"]["all"]
     latency = scores["latency"]
@@ -303,12 +317,14 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
         "positive_case_recall": rates["positive_case_recall"] >= tolerances["min_positive_case_recall"],
         "p95_ms": rates["p95_ms"] <= tolerances["max_p95_ms"],
         "untruncated_input_evidence": complete_input_evidence,
+        "reported_model_consistency": reported_model_consistent,
     }
     return {
         "application": validated["application"],
         "kind": "declared_independent_labels_heldout_not_runtime_parity",
         "experiment_sha256": result["experiment_sha256"],
         "qualification_manifest_sha256": manifest_sha,
+        "reported_model": reported_model,
         "cases": len(rows),
         "thresholds": validated["thresholds"],
         "tolerances": tolerances,
@@ -324,5 +340,5 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
         "latency": latency,
         "checks": checks,
         "qualified_against_frozen_tolerances": all(checks.values()),
-        "limits": "Reviewer independence is declared, not authenticated; task effectiveness and canonical Scout transcript parity are not proven.",
+        "limits": "Reviewer and serving/model identities are self-declared, not authenticated or checkpoint-attested; task effectiveness and canonical Scout transcript parity are not proven.",
     }
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index feb6c6cdfd3..06038d8243a 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -239,6 +239,12 @@ def serialize_result(row, questions, temperature):
         elif (isinstance(value, bool) or not isinstance(value, (int, float))
               or value < 0 or (isinstance(value, float) and not math.isfinite(value))):
             raise ValueError(f"{field} must be a finite nonnegative number")
+    if "observed_model" in row:
+        observed_model = row["observed_model"]
+        if observed_model is not None and (
+            not isinstance(observed_model, str) or not observed_model.strip()
+        ):
+            raise ValueError("reported HTTP model must be a nonempty string or null")
     if temperature is None:
         if "probs_raw" in row:
             raise ValueError("uncalibrated result must not contain raw probabilities")
@@ -502,6 +508,10 @@ def main(argv=None):
                 else:
                     response, ms = post(args.url, body, bearer_token=bearer_token)
                 probs = noul_probabilities(response, questions)
+                observed_model = response.get("model")
+                if (args.send_model is not None and observed_model is not None
+                        and observed_model != args.send_model):
+                    raise ValueError("reported model differs from requested model")
                 server_ms = response.get("latency_ms")
                 if server_ms is None:
                     server_ms = (response.get("timing") or {}).get("inference_ms")
@@ -518,6 +528,7 @@ def main(argv=None):
                     "input_tokens": usage.get("input_tokens"),
                     "state_tokens": usage.get("state_tokens"),
                     "dropped": usage.get("state_tokens_dropped"), "probs": probs,
+                    "observed_model": observed_model,
                 }
                 if args.noul_temperature is not None:
                     row["probs_raw"] = probs
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index b2e7119fdd3..d9d5dca03e0 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -218,6 +218,7 @@ def signed_result(self):
                 "dropped": 0,
                 "request_sha256": hashlib.sha256(rendered).hexdigest(),
                 "request_size_bytes": len(rendered),
+                "observed_model": "self-reported-test-model",
             })
         experiment = {
             "version": 2,
@@ -261,6 +262,8 @@ def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
         self.assertEqual(report["holdout_decisions"]["abstention_rate"], 0.5)
         self.assertEqual(report["holdout_decisions"]["unwanted_hint_rate"], 0)
         self.assertTrue(report["checks"]["untruncated_input_evidence"])
+        self.assertEqual(report["reported_model"], "self-reported-test-model")
+        self.assertTrue(report["checks"]["reported_model_consistency"])
         self.assertEqual(report["fixed_threshold_metrics"]["none_false_hint_rate"], 0)
         self.assertEqual(report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 1)
 
@@ -271,6 +274,80 @@ def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
             qualification.qualified_report(result, stricter, qualification.sha256_json(stricter), self.questions, self.dataset_bytes)
 
 
+    def test_qualification_reports_missing_or_drifting_model_identity(self):
+        for value in (None, "", True, "another-model"):
+            with self.subTest(observed_model=value):
+                result = self.signed_result()
+                result["rows"][1]["observed_model"] = value
+                report = qualification.qualified_report(
+                    result, self.manifest, self.manifest_sha,
+                    self.questions, self.dataset_bytes,
+                )
+                self.assertFalse(report["qualified_against_frozen_tolerances"])
+                self.assertFalse(report["checks"]["reported_model_consistency"])
+                self.assertIsNone(report["reported_model"])
+
+    def test_qualification_rejects_model_different_from_frozen_selector(self):
+        manifest = copy.deepcopy(self.manifest)
+        manifest["arm"]["requested_model"] = "pinned-model-selector"
+        manifest_sha = qualification.sha256_bytes(
+            json.dumps(manifest).encode("utf-8")
+        )
+        result = self.signed_result()
+        experiment = result["experiment"]
+        experiment["qualification_manifest_sha256"] = manifest_sha
+        experiment["requested_model"] = "pinned-model-selector"
+        for row in result["rows"]:
+            case = next(c for c in self.dataset if c["id"] == row["id"])
+            body = {
+                "state": run_http.STATES["B_noul_text"](case),
+                "questions": self.questions,
+                "model": "pinned-model-selector",
+            }
+            rendered = run_http.request_bytes(body)
+            row["request_sha256"] = hashlib.sha256(rendered).hexdigest()
+            row["request_size_bytes"] = len(rendered)
+        experiment["requests_sha256"] = score.evidence_digest([
+            [row["id"], row["request_sha256"]] for row in result["rows"]
+        ])
+        result["experiment_sha256"] = score.evidence_digest(experiment)
+        report = qualification.qualified_report(
+            result, manifest, manifest_sha, self.questions, self.dataset_bytes
+        )
+        self.assertFalse(report["checks"]["reported_model_consistency"])
+        self.assertFalse(report["qualified_against_frozen_tolerances"])
+
+    def test_runner_refuses_explicit_model_mismatch_before_result_append(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            manifest = copy.deepcopy(self.manifest)
+            manifest["arm"]["requested_model"] = "pinned-model-selector"
+            (here / "manifest.json").write_text(json.dumps(manifest))
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            response = {
+                "model": "wrong-model",
+                "answers": {
+                    key: {"type": "noul", "noul": 0.5}
+                    for key in self.questions
+                },
+            }
+            args = [
+                "--url", self.endpoint,
+                "--tag", "model-mismatch",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "--send-model", "pinned-model-selector", "B_noul_text",
+            ]
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post", return_value=(response, 12.0)) as post:
+                    with self.assertRaisesRegex(ValueError, "reported model differs"):
+                        run_http.main(args)
+                    self.assertEqual(post.call_count, 1)
+            self.assertFalse((here / "results" / "model-mismatch_B_noul_text.json").exists())
+            partial = here / "results" / "model-mismatch_B_noul_text.json.partial"
+            self.assertEqual(len(partial.read_text().splitlines()), 1)
+
     def test_qualification_requires_complete_serving_identity(self):
         for field, change in (
             ("missing_server", lambda r: r["experiment"].pop("server_config")),
@@ -368,7 +445,7 @@ def test_http_runner_asks_only_pre_frozen_holdout_and_resumes_with_exact_identit
 
             def answer(url, body):
                 calls.append(body)
-                return {"answers": {
+                return {"model": "self-reported-test-model", "answers": {
                     key: {"type": "noul", "noul": 0.5}
                     for key in body["questions"]
                 }}, 12.0
@@ -391,6 +468,10 @@ def answer(url, body):
             self.assertEqual([r["id"] for r in result["rows"]], [
                 "holdout-positive", "holdout-negative"
             ])
+            self.assertEqual(
+                [r["observed_model"] for r in result["rows"]],
+                ["self-reported-test-model", "self-reported-test-model"],
+            )
             self.assertEqual(result["experiment"]["qualification_manifest_sha256"], self.manifest_sha)
             self.assertEqual(result["experiment"]["case_ids"], [
                 "holdout-positive", "holdout-negative"

From 2f7a39d6e5ff8843e6d003d51f191264be4b8eb9 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 00:55:57 -0800
Subject: [PATCH 15/76] fix(evals): require measured token budgets for
 qualification

Refs #4535

Freeze a separate input-token ceiling and fail qualification when server
usage is missing, zero, or exceeds the predeclared bound. JSON request bytes
are not interchangeable with tokenizer-dependent context usage.

Report measurement coverage and over-limit cases without inventing zero
telemetry. Extend evaluator and CLI regressions with passing and negative
controls; document server-reported token identity limitations.

This is partial historical evaluation integrity, not Kev inference or
runtime parity. Keep PR DRAFT / NO MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md            | 12 ++++-
 evals/intent_engine/qualification.py          | 29 +++++++++++-
 evals/intent_engine/test_qualification.py     | 44 +++++++++++++++++++
 evals/intent_engine/test_qualification_cli.py |  6 +++
 4 files changed, 88 insertions(+), 3 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index ac4c9fd3d2d..945679ea975 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -138,6 +138,13 @@ without changing the threshold manifest. Include
 a required `max_request_bytes` positive integer upper bound on the exact UTF-8
 JSON request body. This precommitted byte limit is **not** the model's
 validated 8,192-token context limit: byte counts and token counts are different.
+Also freeze a positive integer `max_input_tokens` as an independent **reported**
+input-token ceiling. For the published Kev-4B checkpoint, do not set this above
+its 8,192-token validated context range. Unlike the JSON byte cap, token usage
+is known only after the server replies; missing or zero `usage.input_tokens`,
+or any reported count above this precommitted limit, fails qualification rather
+than being counted as within budget. Reported token counts are unverified server
+telemetry, not an independently reproduced tokenization or attestation.
 The runner refuses oversized requests before any result file or model call;
 qualification independently reconstructs and verifies the byte count of every
 held-out request. Every HTTP result also records the model name *reported by
@@ -192,8 +199,9 @@ This writes `evals/intent_engine/qualification_summary.json` separately from
 the historical `summary.json`, and reports only **frozen-threshold** held-out
 false hints, all-case unwanted suggestions, missed positive cases, coverage,
 abstention, positive-case recall, reconstructed JSON request sizes
-(min/p50/max/total and the frozen byte ceiling), p50/p95 measured end-to-end HTTP latency and
-the predeclared tolerances. A passing report requires every held-out case to
+(min/p50/max/total and the frozen byte ceiling), input-token measurement
+coverage, observed peak and over-budget cases, p50/p95 measured end-to-end HTTP
+latency and the predeclared tolerances. A passing report requires every held-out case to
 report measured zero dropped state tokens and a consistent self-reported
 model name. Missing truncation telemetry is unknown, and a response measured after truncated input cannot prove the full
 frozen request was evaluated; the measurements remain in the failing report. Scoring reads the original frozen case bytes,
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index eb4160ed13d..5b139c2eaf5 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -61,6 +61,10 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     if (isinstance(request_limit, bool) or not isinstance(request_limit, int)
             or request_limit <= 0):
         raise ValueError("qualification requires a positive integer max_request_bytes")
+    token_limit = manifest.get("max_input_tokens")
+    if (isinstance(token_limit, bool) or not isinstance(token_limit, int)
+            or token_limit <= 0):
+        raise ValueError("qualification requires a positive integer max_input_tokens")
 
     thresholds = manifest.get("thresholds")
     if not isinstance(thresholds, dict) or set(thresholds) != set(questions):
@@ -171,6 +175,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
         "heldout_ids": heldout,
         "arm": arm,
         "max_request_bytes": request_limit,
+        "max_input_tokens": token_limit,
         "thresholds": thresholds,
         "noul_temperature": client_temperature,
         "error_tolerances": tolerances,
@@ -296,6 +301,18 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     latency = scores["latency"]
     decisions = holdout_decision_metrics(rows, validated["thresholds"])
     tolerances = validated["error_tolerances"]
+    # A JSON byte ceiling does not bound tokenizer-specific context usage.
+    # An unknown or zero server-reported count is not proof of fitting inside
+    # the model's predeclared validated input-token budget.
+    reported_tokens = [row.get("input_tokens") for row in rows]
+    measured_tokens = [
+        value for value in reported_tokens if type(value) is int and value > 0
+    ]
+    token_evidence_complete = len(measured_tokens) == len(rows)
+    tokens_within_budget = (
+        token_evidence_complete
+        and all(value <= validated["max_input_tokens"] for value in measured_tokens)
+    )
     rates = {
         "none_false_hint_rate": fixed["none_false_hint_rate"],
         "unwanted_hint_rate": decisions["unwanted_hint_rate"],
@@ -318,6 +335,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
         "p95_ms": rates["p95_ms"] <= tolerances["max_p95_ms"],
         "untruncated_input_evidence": complete_input_evidence,
         "reported_model_consistency": reported_model_consistent,
+        "reported_input_token_budget": tokens_within_budget,
     }
     return {
         "application": validated["application"],
@@ -337,8 +355,17 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
             "total": sum(request_sizes),
             "frozen_limit": validated["max_request_bytes"],
         },
+        "input_token_evidence": {
+            "measured_cases": len(measured_tokens),
+            "missing_or_zero_cases": len(rows) - len(measured_tokens),
+            "cases_over_frozen_limit": sum(
+                value > validated["max_input_tokens"] for value in measured_tokens
+            ),
+            "max_reported_tokens": max(measured_tokens, default=None),
+            "frozen_limit": validated["max_input_tokens"],
+        },
         "latency": latency,
         "checks": checks,
         "qualified_against_frozen_tolerances": all(checks.values()),
-        "limits": "Reviewer and serving/model identities are self-declared, not authenticated or checkpoint-attested; task effectiveness and canonical Scout transcript parity are not proven.",
+        "limits": "Reviewer and serving/model identities and token counts are self-reported, not authenticated or checkpoint-attested; task effectiveness and canonical Scout transcript parity are not proven.",
     }
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index d9d5dca03e0..2e460d2b1a9 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -63,6 +63,7 @@ def setUp(self):
             },
             "noul_temperature": None,
             "max_request_bytes": 65536,
+            "max_input_tokens": 64,
             "thresholds": {key: 0.6 for key in self.questions},
             "error_tolerances": {
                 "max_none_false_hint_rate": 0.0,
@@ -215,6 +216,7 @@ def signed_result(self):
                 "context_needed": case["context_needed"],
                 "probs": probs,
                 "ms": 10 if case["labels"] else 20,
+                "input_tokens": 64,
                 "dropped": 0,
                 "request_sha256": hashlib.sha256(rendered).hexdigest(),
                 "request_size_bytes": len(rendered),
@@ -255,6 +257,10 @@ def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
         self.assertEqual(report["latency"]["p50_ms"], 15)
         self.assertEqual(report["latency"]["p95_ms"], 20)
         self.assertEqual(report["request_size_bytes"]["frozen_limit"], 65536)
+        self.assertEqual(report["input_token_evidence"]["frozen_limit"], 64)
+        self.assertEqual(report["input_token_evidence"]["measured_cases"], 2)
+        self.assertEqual(report["input_token_evidence"]["max_reported_tokens"], 64)
+        self.assertTrue(report["checks"]["reported_input_token_budget"])
         self.assertEqual(report["request_size_bytes"]["total"], sum(
             row["request_size_bytes"] for row in result["rows"]
         ))
@@ -734,6 +740,44 @@ def test_scorer_refuses_rehashed_unfrozen_http_arm(self):
                     )
 
 
+    def test_frozen_token_budget_must_be_positive_integer(self):
+        for limit in (None, 0, -1, True, 1.5, "8192"):
+            with self.subTest(limit=limit):
+                manifest = copy.deepcopy(self.manifest)
+                manifest["max_input_tokens"] = limit
+                with self.assertRaisesRegex(ValueError, "positive integer max_input_tokens"):
+                    self.valid(manifest)
+
+    def test_missing_zero_or_overlimit_input_tokens_cannot_qualify(self):
+        for reported in (None, 0, 65):
+            with self.subTest(reported=reported):
+                result = self.signed_result()
+                result["rows"][0]["input_tokens"] = reported
+                report = qualification.qualified_report(
+                    result, self.manifest, self.manifest_sha,
+                    self.questions, self.dataset_bytes,
+                )
+                self.assertFalse(report["qualified_against_frozen_tolerances"])
+                self.assertFalse(report["checks"]["reported_input_token_budget"])
+                evidence = report["input_token_evidence"]
+                self.assertEqual(evidence["measured_cases"], 1 if reported in (None, 0) else 2)
+                self.assertEqual(evidence["missing_or_zero_cases"], 1 if reported in (None, 0) else 0)
+                self.assertEqual(evidence["cases_over_frozen_limit"], int(reported == 65))
+
+    def test_rehashed_tighter_token_budget_does_not_certify_old_measurements(self):
+        restricted = copy.deepcopy(self.manifest)
+        restricted["max_input_tokens"] = 63
+        restricted_sha = qualification.sha256_json(restricted)
+        result = self.signed_result()
+        result["experiment"]["qualification_manifest_sha256"] = restricted_sha
+        result["experiment_sha256"] = score.evidence_digest(result["experiment"])
+        report = qualification.qualified_report(
+            result, restricted, restricted_sha, self.questions, self.dataset_bytes,
+        )
+        self.assertFalse(report["qualified_against_frozen_tolerances"])
+        self.assertFalse(report["checks"]["reported_input_token_budget"])
+        self.assertEqual(report["input_token_evidence"]["cases_over_frozen_limit"], 2)
+
     def test_frozen_request_byte_budget_must_be_positive_integer(self):
         for budget in (None, 0, -1, True, 0.5, "4096"):
             with self.subTest(budget=budget):
diff --git a/evals/intent_engine/test_qualification_cli.py b/evals/intent_engine/test_qualification_cli.py
index d7ceb454db6..9b920b4f3f3 100644
--- a/evals/intent_engine/test_qualification_cli.py
+++ b/evals/intent_engine/test_qualification_cli.py
@@ -68,6 +68,12 @@ def test_failed_checks_exit_nonzero_and_replace_a_previous_passing_report(self):
              lambda r: r["rows"][0].update(dropped=None)),
             ("truncated_input", "untruncated_input_evidence",
              lambda r: r["rows"][0].update(dropped=1)),
+            ("unmeasured_input_tokens", "reported_input_token_budget",
+             lambda r: r["rows"][0].update(input_tokens=None)),
+            ("zero_input_tokens", "reported_input_token_budget",
+             lambda r: r["rows"][0].update(input_tokens=0)),
+            ("over_budget_input_tokens", "reported_input_token_budget",
+             lambda r: r["rows"][0].update(input_tokens=65)),
         )
         for name, failed_check, mutate in failures:
             with self.subTest(name=name):

From 24f004a5d20c57dda8a98deb9510fc7995c24307 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 02:22:13 -0800
Subject: [PATCH 16/76] fix(evals): preserve native Kev truncation evidence
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Refs #4535

Read native Kev's reported state-token and used-token counts with its
truncated flag instead of assuming state_tokens_dropped is exposed. Reject
contradictory telemetry, retain original counters, and reconstruct dropped
tokens during qualified scoring and exact partial resume. Missing counters
remain unknown rather than measured zero.

Add falsifiable runner/scorer/partial-resume regression cases, including
truncated and inconsistent responses and rehashed normalized drop tampering.
No model inference, local Bazel validation or replacement claim is made.
Keep #4797 and mixed #4776 DRAFT — NEVER MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md        |  13 +-
 evals/intent_engine/qualification.py      |   6 +-
 evals/intent_engine/run_http.py           |  78 +++++++++-
 evals/intent_engine/test_qualification.py | 171 ++++++++++++++++++++++
 4 files changed, 265 insertions(+), 3 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 945679ea975..d96846c1d3f 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -203,7 +203,18 @@ abstention, positive-case recall, reconstructed JSON request sizes
 coverage, observed peak and over-budget cases, p50/p95 measured end-to-end HTTP
 latency and the predeclared tolerances. A passing report requires every held-out case to
 report measured zero dropped state tokens and a consistent self-reported
-model name. Missing truncation telemetry is unknown, and a response measured after truncated input cannot prove the full
+model name. Native Kev's optional truncation receipts use a top-level Boolean
+`truncated` and numeric `usage.state_tokens` / `usage.state_tokens_used`,
+not `usage.state_tokens_dropped`. The runner reconstructs their difference
+and rejects inconsistent counts or flags before recording a row. The native
+server's original and used token counts and explicit truncation flags are
+retained in the output; an independent scorer and partial-resume reader reject
+a rewritten normalized drop count. Direct `state_tokens_dropped` counts remain
+supported for servers that expose them.
+Kev emits those paired counters only when its `KEV_TRUNCATE_STATES=1` serving
+setting is enabled; a bare `truncated: false` without numeric counters is
+**unknown, not measured zero**, and cannot pass qualification. Freeze the
+serving configuration; no infrastructure change is implied. Missing truncation telemetry is unknown, and a response measured after truncated input cannot prove the full
 frozen request was evaluated; the measurements remain in the failing report. Scoring reads the original frozen case bytes,
 checks all reviewed labels/splits against them and recreates each HTTP
 request fingerprint with the runner's own renderer. Self-consistent result
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 5b139c2eaf5..b788a47dd5f 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -213,7 +213,7 @@ def holdout_decision_metrics(rows, thresholds):
 def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     """Reproduce qualified predictions from the frozen original dataset."""
     from score import score_config
-    from run_http import STATES, request_bytes, validate_dataset
+    from run_http import STATES, request_bytes, validate_dataset, validate_row_truncation
 
     if not isinstance(dataset_bytes, bytes):
         raise ValueError("qualification requires the original dataset bytes")
@@ -253,6 +253,10 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
         if (row.get("labels") != annotation["verified_labels"]
                 or row.get("context_needed") != annotation["verified_context_needed"]):
             raise ValueError(f"qualification result {row['id']} does not match reviewed labels")
+    # Native truncation counters and the normalized drop count must remain
+    # consistent even if the result JSON was edited after the HTTP response.
+    for row in rows:
+        validate_row_truncation(row)
     # Reconstruct real HTTP request bytes: self-consistent metadata hashes
     # alone cannot prove the frozen prompts were used.
     renderer = STATES.get(experiment.get("config"))
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 06038d8243a..87da5aca14f 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -224,6 +224,70 @@ def noul_probabilities(response, questions):
     return probabilities(values, questions)
 
 
+def dropped_state_tokens(response, usage):
+    """Derive measurable truncation from the native HTTP response contract.
+
+    Kev reports the original and used state-token counts with a top-level
+    truncated flag. Other servers may report state_tokens_dropped directly.
+    A bare false flag is not numeric evidence of zero dropped tokens.
+    """
+    direct = usage.get("state_tokens_dropped")
+    if direct is not None and (type(direct) is not int or direct < 0):
+        raise ValueError("invalid truncation dropped-token count")
+
+    original = usage.get("state_tokens")
+    used = usage.get("state_tokens_used")
+    for value in (original, used):
+        if value is not None and (type(value) is not int or value < 0):
+            raise ValueError("invalid truncation state-token count")
+
+    derived = None
+    if original is not None and used is not None:
+        if used > original:
+            raise ValueError("invalid truncation: used state tokens exceed original")
+        derived = original - used
+        if direct is not None and direct != derived:
+            raise ValueError("inconsistent truncation token counts")
+
+    dropped = derived if derived is not None else direct
+    flags = []
+    for envelope, field in ((response, "truncated"), (usage, "truncated")):
+        if field in envelope:
+            flag = envelope[field]
+            if type(flag) is not bool:
+                raise ValueError("invalid truncation flag")
+            flags.append(flag)
+            if dropped is not None and flag != (dropped > 0):
+                raise ValueError("inconsistent truncation flag and token counts")
+    if len(set(flags)) > 1:
+        raise ValueError("inconsistent truncation flags")
+    return dropped
+
+
+def validate_row_truncation(row):
+    """Recheck persisted server counters; normalized dropped alone is mutable.
+
+    Older rows without raw counter fields still have unknown original source
+    semantics. New rows retain the supplied counts/flags for independent
+    qualification and exact partial-resume validation.
+    """
+    native_fields = ("state_tokens_used", "reported_state_tokens_dropped",
+                     "truncated", "usage_truncated")
+    if not any(field in row for field in native_fields):
+        return
+    usage = {"state_tokens": row.get("state_tokens")}
+    if "state_tokens_used" in row:
+        usage["state_tokens_used"] = row["state_tokens_used"]
+    if "reported_state_tokens_dropped" in row:
+        usage["state_tokens_dropped"] = row["reported_state_tokens_dropped"]
+    if "usage_truncated" in row:
+        usage["truncated"] = row["usage_truncated"]
+    envelope = {"truncated": row["truncated"]} if "truncated" in row else {}
+    expected = dropped_state_tokens(envelope, usage)
+    if row.get("dropped") != expected:
+        raise ValueError("truncation row disagrees with recorded server counters")
+
+
 def serialize_result(row, questions, temperature):
     """Admit fresh and resumed rows before any durable output is changed."""
     probabilities(row.get("probs"), questions)
@@ -252,6 +316,7 @@ def serialize_result(row, questions, temperature):
         raw = probabilities(row.get("probs_raw"), questions)
         if row["probs"] != {key: calibrated(value, temperature) for key, value in raw.items()}:
             raise ValueError("calibrated probabilities disagree with raw probabilities")
+    validate_row_truncation(row)
     # json.loads accepts NaN/Infinity, including in extra nested fields. Reject
     # them while validating the partial log, not after opening the final file.
     return json.dumps(row, allow_nan=False)
@@ -527,9 +592,20 @@ def main(argv=None):
                     "ms": ms, "server_ms": server_ms,
                     "input_tokens": usage.get("input_tokens"),
                     "state_tokens": usage.get("state_tokens"),
-                    "dropped": usage.get("state_tokens_dropped"), "probs": probs,
+                    "dropped": dropped_state_tokens(response, usage), "probs": probs,
                     "observed_model": observed_model,
                 }
+                # Retain the original typed truncation measurements so a
+                # resume and an independent scorer can reconstruct dropped.
+                for field, persisted in (
+                    ("state_tokens_used", "state_tokens_used"),
+                    ("state_tokens_dropped", "reported_state_tokens_dropped"),
+                    ("truncated", "usage_truncated"),
+                ):
+                    if field in usage:
+                        row[persisted] = usage[field]
+                if "truncated" in response:
+                    row["truncated"] = response["truncated"]
                 if args.noul_temperature is not None:
                     row["probs_raw"] = probs
                     row["probs"] = {key: calibrated(value, args.noul_temperature) for key, value in probs.items()}
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 2e460d2b1a9..53d1f24df85 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -145,6 +145,177 @@ def test_unwanted_hints_on_positive_cases_are_not_hidden_by_recall(self):
         self.assertFalse(report["checks"]["unwanted_hint_rate"])
         self.assertFalse(report["qualified_against_frozen_tolerances"])
 
+
+    def test_native_kev_truncation_is_measured_not_assumed_from_a_flag(self):
+        # Native Kev emits state_tokens/state_tokens_used and a top-level
+        # truncated flag, not necessarily state_tokens_dropped.
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            args = [
+                "--url", self.endpoint, "--tag", "native-kev",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "B_noul_text",
+            ]
+            calls = []
+
+            def native_answer(url, body):
+                index = len(calls)
+                calls.append(body)
+                case = self.dataset[index + 2]
+                return {
+                    "model": "self-reported-test-model",
+                    "answers": {
+                        key: {"type": "noul", "noul": 0.9 if key in case["labels"] else 0.1}
+                        for key in self.questions
+                    },
+                    "usage": {
+                        "input_tokens": 64, "state_tokens": 32,
+                        "state_tokens_used": 32 if index == 0 else 24,
+                    },
+                    "truncated": index == 1,
+                    "latency_ms": 5.0,
+                }, 10.0
+
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post", side_effect=native_answer
+            ):
+                run_http.main(args)
+            self.assertEqual(len(calls), 2)
+            result = json.loads(
+                (here / "results" / "native-kev_B_noul_text.json").read_text()
+            )
+            self.assertEqual([r["dropped"] for r in result["rows"]], [0, 8])
+            self.assertEqual([r["state_tokens"] for r in result["rows"]], [32, 32])
+            self.assertEqual([r["state_tokens_used"] for r in result["rows"]], [32, 24])
+            self.assertEqual([r["truncated"] for r in result["rows"]], [False, True])
+            report = qualification.qualified_report(
+                result, self.manifest, self.manifest_sha, self.questions,
+                self.dataset_bytes,
+            )
+            self.assertEqual(report["latency"]["cases_truncated"], 1)
+            self.assertFalse(report["checks"]["untruncated_input_evidence"])
+            self.assertFalse(report["qualified_against_frozen_tolerances"])
+            self.assertTrue(report["checks"]["reported_input_token_budget"])
+            # Rehashing only the declared experiment cannot hide a rewritten
+            # normalized drop count when native server counts are present.
+            forged = copy.deepcopy(result)
+            forged["rows"][1]["dropped"] = 0
+            with self.assertRaisesRegex(ValueError, "truncation row disagrees"):
+                qualification.qualified_report(
+                    forged, self.manifest, self.manifest_sha,
+                    self.questions, self.dataset_bytes,
+                )
+
+    def test_conflicting_native_truncation_is_refused_before_row_append(self):
+        cases = (
+            ("used_exceeds_original", {"truncated": False,
+                "usage": {"state_tokens": 24, "state_tokens_used": 32}}),
+            ("flag_contradicts_counts", {"truncated": False,
+                "usage": {"state_tokens": 32, "state_tokens_used": 24}}),
+            ("direct_contradicts_native", {"truncated": True,
+                "usage": {"state_tokens": 32, "state_tokens_used": 24,
+                          "state_tokens_dropped": 0}}),
+            ("invalid_flag", {"truncated": 0,
+                "usage": {"state_tokens": 32, "state_tokens_used": 32}}),
+            ("flags_disagree", {"truncated": False,
+                "usage": {"truncated": True}}),
+            ("invalid_count", {"truncated": False,
+                "usage": {"state_tokens": True, "state_tokens_used": 32}}),
+        )
+        for label, metadata in cases:
+            with self.subTest(label=label), tempfile.TemporaryDirectory() as temp:
+                here = Path(temp)
+                (here / "dataset.json").write_bytes(self.dataset_bytes)
+                (here / "manifest.json").write_bytes(self.manifest_bytes)
+                (here / "server.json").write_text(json.dumps(self.server_config))
+                args = [
+                    "--url", self.endpoint, "--tag", "bad-native",
+                    "--server-config", str(here / "server.json"),
+                    "--qualification-manifest", str(here / "manifest.json"),
+                    "B_noul_text",
+                ]
+                response = {
+                    "model": "self-reported-test-model",
+                    "answers": {
+                        key: {"type": "noul", "noul": 0.5}
+                        for key in self.questions
+                    },
+                    "usage": {"input_tokens": 64, **metadata["usage"]},
+                    "truncated": metadata["truncated"],
+                }
+                with patch.object(run_http, "HERE", str(here)), patch.object(
+                    run_http, "post", return_value=(response, 10.0)
+                ) as post:
+                    with self.assertRaisesRegex(ValueError, "truncation"):
+                        run_http.main(args)
+                post.assert_called_once()
+                self.assertFalse(
+                    (here / "results" / "bad-native_B_noul_text.json").exists()
+                )
+                partial = here / "results" / "bad-native_B_noul_text.json.partial"
+                self.assertEqual(len(partial.read_text().splitlines()), 1)
+
+
+    def test_native_truncation_partial_tampering_refuses_without_network(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            args = [
+                "--url", self.endpoint, "--tag", "native-partial",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "B_noul_text",
+            ]
+            response = {
+                "model": "self-reported-test-model",
+                "answers": {
+                    key: {"type": "noul", "noul": 0.9}
+                    for key in self.questions
+                },
+                "usage": {
+                    "input_tokens": 64, "state_tokens": 30,
+                    "state_tokens_used": 25,
+                },
+                "truncated": True,
+            }
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post", side_effect=[
+                    (response, 10.0), RuntimeError("network interruption")
+                ]) as first:
+                    with self.assertRaisesRegex(RuntimeError, "network interruption"):
+                        run_http.main(args)
+                    self.assertEqual(first.call_count, 2)
+                partial = here / "results" / "native-partial_B_noul_text.json.partial"
+                header, row = partial.read_text().splitlines()
+                original = json.loads(row)
+                self.assertEqual(original["dropped"], 5)
+                original["dropped"] = 0
+                partial.write_text(header + "\n" + json.dumps(original) + "\n")
+                before = partial.read_bytes()
+                with patch.object(run_http, "post") as blocked:
+                    with self.assertRaisesRegex(ValueError, "truncation row disagrees"):
+                        run_http.main(args)
+                    blocked.assert_not_called()
+                self.assertEqual(partial.read_bytes(), before)
+                self.assertFalse(
+                    (here / "results" / "native-partial_B_noul_text.json").exists()
+                )
+
+    def test_boolean_truncation_without_counts_is_unknown(self):
+        self.assertIsNone(run_http.dropped_state_tokens({"truncated": False}, {}))
+        self.assertIsNone(run_http.dropped_state_tokens({"truncated": True}, {}))
+        self.assertEqual(
+            run_http.dropped_state_tokens(
+                {"truncated": False}, {"state_tokens_dropped": 0}
+            ), 0
+        )
+
     def test_missing_or_dropped_server_tokens_cannot_qualify(self):
         for value in (None, 1):
             with self.subTest(state_tokens_dropped=value):

From c5efdd5b812e8b4dc1819d0ba03319e11c904ea0 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 03:17:18 -0800
Subject: [PATCH 17/76] fix(evals): refuse cross-model frozen qualification
 resumes

Refs #4535

An interrupted qualified evaluation could resume with a forged or stale
server-reported model and spend additional inference on a different model.
Reject resumed rows that contradict the frozen selector or previously seen
model, and refuse a model change during a new held-out run before append.
Retain the original partial unchanged on refusal; valid prefixes still resume.

Add falsifiable regression cases for mid-run drift, missing or substituted
model identity in resumable rows, zero-network refusal and valid recovery.
These are protocol integrity tests, not Kev inference or runtime parity.

DRAFT / NO MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md        |  7 +-
 evals/intent_engine/run_http.py           | 28 +++++++
 evals/intent_engine/test_qualification.py | 92 +++++++++++++++++++++++
 3 files changed, 125 insertions(+), 2 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index d96846c1d3f..5016b167d37 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -151,8 +151,11 @@ held-out request. Every HTTP result also records the model name *reported by
 the server*, including explicit null when the endpoint omits it. A frozen-arm
 qualification fails if this name is missing, differs among held-out responses,
 or contradicts a non-null requested model selector. For an explicit selector,
-a contradictory response is rejected before appending the row. The reported
-name is **not** cryptographic server identity or checkpoint attestation; a
+a contradictory response is rejected before appending the row. A change in the
+server-reported name between cases is also refused before the second append;
+resuming an interrupted frozen arm rejects a stored contradictory name before
+sending more requests. The reported name is **not** cryptographic server
+identity or checkpoint attestation; a
 remote service can misreport it.
 
 Include `thresholds` with one frozen probability threshold per requested
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 87da5aca14f..adda79d907b 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -288,6 +288,19 @@ def validate_row_truncation(row):
         raise ValueError("truncation row disagrees with recorded server counters")
 
 
+def checked_qualified_model(previous, reported, requested):
+    """Refuse a changed reported model within one frozen HTTP arm.
+
+    A server name is not checkpoint attestation, but neither a resumed row
+    nor a new response may contradict the selected or previously seen name.
+    """
+    if requested is not None and reported != requested:
+        raise ValueError("qualified HTTP reported model differs from frozen selector")
+    if previous is not None and reported is not None and reported != previous:
+        raise ValueError("qualified HTTP responses mix reported models")
+    return previous if reported is None else reported
+
+
 def serialize_result(row, questions, temperature):
     """Admit fresh and resumed rows before any durable output is changed."""
     probabilities(row.get("probs"), questions)
@@ -382,6 +395,8 @@ def resume_rows(partial, header, cases, requests, questions, temperature):
         raise ValueError("partial output belongs to another or unrecorded experiment; use a new tag")
     done = {}
     ordered_ids = tuple(cases)
+    qualified = "qualification_manifest_sha256" in header["experiment"]
+    previous_model = None
     for line_number, line in enumerate(partial, 2):
         if not line.endswith("\n"):
             raise ValueError(f"partial output has an incomplete row at line {line_number}")
@@ -405,6 +420,11 @@ def resume_rows(partial, header, cases, requests, questions, temperature):
                      or row.get("request_size_bytes") != len(request_bytes(requests[case_id][0])))):
             raise ValueError(f"partial output has mismatched request byte count for {case_id}")
         serialize_result(row, questions, temperature)
+        if qualified:
+            previous_model = checked_qualified_model(
+                previous_model, row.get("observed_model"),
+                header["experiment"].get("requested_model"),
+            )
         done[case_id] = row
     return done
 
@@ -564,6 +584,10 @@ def main(argv=None):
                 done = {}
             else:
                 done = resume_rows(partial, header, cases, requests, questions, args.noul_temperature)
+            previous_model = next(
+                (row["observed_model"] for row in done.values()
+                 if row.get("observed_model") is not None), None
+            )
             for index, case in enumerate(dataset):
                 if case["id"] in done:
                     continue
@@ -577,6 +601,10 @@ def main(argv=None):
                 if (args.send_model is not None and observed_model is not None
                         and observed_model != args.send_model):
                     raise ValueError("reported model differs from requested model")
+                if qualification_sha is not None:
+                    previous_model = checked_qualified_model(
+                        previous_model, observed_model, args.send_model,
+                    )
                 server_ms = response.get("latency_ms")
                 if server_ms is None:
                     server_ms = (response.get("timing") or {}).get("inference_ms")
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 53d1f24df85..b40a3671a47 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -525,6 +525,98 @@ def test_runner_refuses_explicit_model_mismatch_before_result_append(self):
             partial = here / "results" / "model-mismatch_B_noul_text.json.partial"
             self.assertEqual(len(partial.read_text().splitlines()), 1)
 
+    def test_qualified_inference_refuses_server_model_change_before_second_append(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            args = [
+                "--url", self.endpoint, "--tag", "model-drift",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "B_noul_text",
+            ]
+
+            def reply(model):
+                return {
+                    "model": model,
+                    "answers": {key: {"type": "noul", "noul": 0.5}
+                                for key in self.questions},
+                    "usage": {"input_tokens": 64, "state_tokens": 32,
+                              "state_tokens_used": 32},
+                    "truncated": False,
+                }, 10.0
+
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post", side_effect=[
+                    reply("reported-model-a"), reply("reported-model-b"),
+                ]
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "mix reported models"):
+                    run_http.main(args)
+                self.assertEqual(post.call_count, 2)
+            path = here / "results" / "model-drift_B_noul_text.json"
+            self.assertFalse(path.exists())
+            header, first = (here / "results" / "model-drift_B_noul_text.json.partial").read_text().splitlines()
+            self.assertEqual(json.loads(first)["observed_model"], "reported-model-a")
+            self.assertEqual(json.loads(header)["experiment"]["case_ids"],
+                             ["holdout-positive", "holdout-negative"])
+
+    def test_qualified_resume_refuses_forged_model_before_network(self):
+        for altered_model in (None, "different-model"):
+            with self.subTest(altered_model=altered_model), tempfile.TemporaryDirectory() as temp:
+                here = Path(temp)
+                (here / "dataset.json").write_bytes(self.dataset_bytes)
+                manifest = copy.deepcopy(self.manifest)
+                manifest["arm"]["requested_model"] = "selected-model"
+                (here / "manifest.json").write_text(json.dumps(manifest))
+                (here / "server.json").write_text(json.dumps(self.server_config))
+                args = [
+                    "--url", self.endpoint, "--tag", "model-resume",
+                    "--server-config", str(here / "server.json"),
+                    "--qualification-manifest", str(here / "manifest.json"),
+                    "--send-model", "selected-model", "B_noul_text",
+                ]
+                answer = {
+                    "model": "selected-model",
+                    "answers": {key: {"type": "noul", "noul": 0.5}
+                                for key in self.questions},
+                    "usage": {"input_tokens": 64, "state_tokens": 32,
+                              "state_tokens_used": 32},
+                    "truncated": False,
+                }
+                with patch.object(run_http, "HERE", str(here)):
+                    with patch.object(run_http, "post", side_effect=[
+                        (answer, 10.0), OSError("interrupted"),
+                    ]) as first:
+                        with self.assertRaisesRegex(OSError, "interrupted"):
+                            run_http.main(args)
+                        self.assertEqual(first.call_count, 2)
+                    partial = here / "results" / "model-resume_B_noul_text.json.partial"
+                    original = partial.read_bytes()
+                    header, row = original.decode("utf-8").splitlines()
+                    corrupted = json.loads(row)
+                    corrupted["observed_model"] = altered_model
+                    partial.write_text(header + "\n" + json.dumps(corrupted) + "\n")
+                    before = partial.read_bytes()
+                    with patch.object(run_http, "post") as blocked:
+                        with self.assertRaisesRegex(ValueError, "frozen selector"):
+                            run_http.main(args)
+                        blocked.assert_not_called()
+                    self.assertEqual(partial.read_bytes(), before)
+                    self.assertFalse(
+                        (here / "results" / "model-resume_B_noul_text.json").exists()
+                    )
+                    # The valid original prefix must still resume normally.
+                    partial.write_bytes(original)
+                    with patch.object(run_http, "post", return_value=(answer, 10.0)) as remaining:
+                        run_http.main(args)
+                        remaining.assert_called_once()
+                    self.assertEqual(len(json.loads(
+                        (here / "results" / "model-resume_B_noul_text.json").read_text()
+                    )["rows"]), 2)
+
     def test_qualification_requires_complete_serving_identity(self):
         for field, change in (
             ("missing_server", lambda r: r["experiment"].pop("server_config")),

From 23cd79ce4c3b6b527ca20aff819c63f42a617100 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 04:24:51 -0800
Subject: [PATCH 18/76] fix(evals): reject ambiguous frozen JSON evidence

Refs #4535

Reject duplicate object keys and nonfinite JSON constants in frozen
qualification manifests, source datasets, serving declarations, HTTP
responses, resume logs and qualified CLI results. Last-key-wins parsing
can silently reinterpret reviewed thresholds, labels or model evidence.

Add falsifiable regressions for identical duplicate keys before inference,
ambiguous server responses, untouched partial logs, qualified-report
publication refusal and a valid resumed control. Preserve historical
results without rewriting them. No Kev inference or Bazel pass claimed.

Decision-model work remains DRAFT and NEVER MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md            |  7 +++
 evals/intent_engine/qualification.py          | 21 +++++++-
 evals/intent_engine/run_http.py               | 14 ++---
 evals/intent_engine/score.py                  |  4 +-
 evals/intent_engine/test_qualification.py     | 52 +++++++++++++++++++
 evals/intent_engine/test_qualification_cli.py | 20 +++++++
 evals/intent_engine/test_run_http.py          | 31 +++++++++++
 7 files changed, 138 insertions(+), 11 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 5016b167d37..76c68ff0add 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -24,6 +24,13 @@ name in result evidence. An authenticated remote endpoint must use HTTPS;
 plain HTTP is accepted only for direct loopback addresses. Ambient proxies
 and HTTP redirects are disabled, including for authenticated requests.
 
+Frozen inputs and HTTP responses must be unambiguous JSON: repeated object keys
+(including identical duplicate values) and nonfinite constants such as `NaN`
+are refused. A last-key-wins parse could otherwise make a reviewer see a
+different model, split, threshold or measurement than the evaluator used.
+Malformed `.partial` rows are refused without append, and qualified scoring
+rejects ambiguous original data or result JSON before publishing a report.
+
 The JSON declaration must contain nonempty `checkpoint`, `server_revision`,
 `backend`, and `dtype` strings, and the actual finite, nonnegative server
 `temperature`. Include other settings that affect inference as additional keys.
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index b788a47dd5f..9597d42ef7d 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -12,6 +12,23 @@
 _DIGEST = re.compile(r"[0-9a-f]{64}\Z")
 
 
+def loads_unambiguous_json(raw):
+    """Reject duplicate keys and nonfinite constants in evidence JSON."""
+    def unique_object(pairs):
+        result = {}
+        for key, value in pairs:
+            if key in result:
+                raise ValueError(f"duplicate JSON object key: {key}")
+            result[key] = value
+        return result
+
+    def reject_nonfinite(constant):
+        raise ValueError(f"nonfinite JSON constant: {constant}")
+
+    return json.loads(raw, object_pairs_hook=unique_object,
+                      parse_constant=reject_nonfinite)
+
+
 def sha256_bytes(data):
     return hashlib.sha256(data).hexdigest()
 
@@ -23,7 +40,7 @@ def sha256_json(value):
 def load_manifest(path):
     with open(path, "rb") as source:
         raw = source.read()
-    return json.loads(raw), sha256_bytes(raw)
+    return loads_unambiguous_json(raw), sha256_bytes(raw)
 
 
 def validate_manifest(manifest, dataset_sha, questions, cases=None):
@@ -220,7 +237,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     dataset_sha = sha256_bytes(dataset_bytes)
     if result.get("dataset_sha256") != dataset_sha:
         raise ValueError("qualification result dataset digest differs from supplied evidence")
-    cases = json.loads(dataset_bytes)
+    cases = loads_unambiguous_json(dataset_bytes)
     validate_dataset(cases, questions)
 
     experiment = result.get("experiment")
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index adda79d907b..7dfdc8366da 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -23,7 +23,7 @@
 HERE = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, HERE)
 from common import calibrated, noul_questions, prompt_only, text_window
-from qualification import load_manifest, validate_manifest
+from qualification import load_manifest, loads_unambiguous_json, validate_manifest
 
 STATES = {
     "A_noul_prompt": prompt_only,
@@ -171,13 +171,13 @@ def post(url, body, timeout=1800, bearer_token=None):
     )
     started = time.perf_counter()
     with opener.open(req, timeout=timeout) as response:
-        out = json.loads(response.read())
+        out = loads_unambiguous_json(response.read())
     return out, (time.perf_counter() - started) * 1000
 
 
 def load_server_config(path):
     with open(path, encoding="utf-8") as source:
-        config = json.load(source)
+        config = loads_unambiguous_json(source.read())
     if not isinstance(config, dict):
         raise ValueError("--server-config must contain a JSON object")
     for field in ("checkpoint", "server_revision", "backend", "dtype"):
@@ -391,7 +391,7 @@ def open_partial_for_resume(path):
 def resume_rows(partial, header, cases, requests, questions, temperature):
     """Validate the whole existing log before appending or issuing HTTP work."""
     first = partial.readline()
-    if not first.endswith("\n") or json.loads(first) != header:
+    if not first.endswith("\n") or loads_unambiguous_json(first) != header:
         raise ValueError("partial output belongs to another or unrecorded experiment; use a new tag")
     done = {}
     ordered_ids = tuple(cases)
@@ -400,7 +400,7 @@ def resume_rows(partial, header, cases, requests, questions, temperature):
     for line_number, line in enumerate(partial, 2):
         if not line.endswith("\n"):
             raise ValueError(f"partial output has an incomplete row at line {line_number}")
-        row = json.loads(line)
+        row = loads_unambiguous_json(line)
         case_id = row.get("id") if isinstance(row, dict) else None
         if not isinstance(case_id, str) or case_id not in cases or case_id in done:
             raise ValueError(f"partial output has an unknown or duplicate case at line {line_number}")
@@ -490,7 +490,7 @@ def main(argv=None):
         raise ValueError(f"sample path exists but is not a regular file: {args.sample_out}")
     with open(args.dataset or os.path.join(HERE, "dataset.json"), "rb") as source:
         dataset_bytes = source.read()
-    dataset = json.loads(dataset_bytes)
+    dataset = loads_unambiguous_json(dataset_bytes)
     dataset_sha = hashlib.sha256(dataset_bytes).hexdigest()
     questions = noul_questions()
     case_ids = validate_dataset(dataset, questions)
@@ -512,7 +512,7 @@ def main(argv=None):
         dataset = [case for case in dataset if case["id"] in keep]
     if args.subset:
         with open(args.subset, encoding="utf-8") as source:
-            ids = json.load(source)["ids"]
+            ids = loads_unambiguous_json(source.read())["ids"]
         if (not isinstance(ids, list) or not ids
                 or any(not isinstance(case_id, str) for case_id in ids)
                 or len(set(ids)) != len(ids) or not set(ids).issubset(case_ids)):
diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py
index ded55d94f92..0af47dfedc8 100644
--- a/evals/intent_engine/score.py
+++ b/evals/intent_engine/score.py
@@ -309,13 +309,13 @@ def ch(rs):
     if len(sys.argv) > 1:
         if len(sys.argv) != 5 or sys.argv[1] != "--qualified":
             raise ValueError("usage: score.py [--qualified MANIFEST DATASET RESULT]")
-        from qualification import load_manifest, qualified_report
+        from qualification import load_manifest, loads_unambiguous_json, qualified_report
         from common import noul_questions
         manifest, manifest_sha = load_manifest(sys.argv[2])
         with open(sys.argv[3], "rb") as source:
             dataset_bytes = source.read()
         with open(sys.argv[4], encoding="utf-8") as source:
-            result = json.load(source)
+            result = loads_unambiguous_json(source.read())
         report = qualified_report(result, manifest, manifest_sha, noul_questions(), dataset_bytes)
         publish_summary(os.path.join(here, "qualification_summary.json"), report)
         print(json.dumps(report, indent=2, allow_nan=False))
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index b40a3671a47..40191d289b3 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -83,6 +83,58 @@ def valid(self, manifest=None, cases=None):
             self.dataset if cases is None else cases,
         )
 
+    def test_ambiguous_frozen_input_json_refuses_without_model_calls(self):
+        # Same-value duplicates are deliberately non-vacuous: the old
+        # last-key-wins parser accepted all three frozen artifacts.
+        raw_manifest = self.manifest_bytes.decode("utf-8")
+        raw_dataset = self.dataset_bytes.decode("utf-8")
+        raw_server = json.dumps(self.server_config)
+        variants = (
+            ("manifest", raw_manifest.replace(
+                '"max_input_tokens": 64',
+                '"max_input_tokens": 64, "max_input_tokens": 64', 1)),
+            ("dataset", raw_dataset.replace(
+                '"project_id": "project-b"',
+                '"project_id": "project-b", "project_id": "project-b"', 1)),
+            ("server", raw_server.replace(
+                '"checkpoint": "pinned"',
+                '"checkpoint": "pinned", "checkpoint": "pinned"', 1)),
+        )
+        for changed_file, ambiguous in variants:
+            with self.subTest(changed_file=changed_file):
+                with tempfile.TemporaryDirectory() as temp:
+                    here = Path(temp)
+                    dataset_bytes = (ambiguous.encode("utf-8")
+                                     if changed_file == "dataset" else self.dataset_bytes)
+                    manifest = copy.deepcopy(self.manifest)
+                    manifest["dataset_sha256"] = qualification.sha256_bytes(dataset_bytes)
+                    (here / "dataset.json").write_bytes(dataset_bytes)
+                    (here / "manifest.json").write_text(
+                        ambiguous if changed_file == "manifest" else json.dumps(manifest)
+                    )
+                    (here / "server.json").write_text(
+                        ambiguous if changed_file == "server" else raw_server
+                    )
+                    with patch.object(run_http, "HERE", str(here)), patch.object(
+                        run_http, "post"
+                    ) as post:
+                        with self.assertRaisesRegex(ValueError, "duplicate JSON object key"):
+                            run_http.main([
+                                "--url", self.endpoint, "--tag", "frozen",
+                                "--server-config", str(here / "server.json"),
+                                "--qualification-manifest", str(here / "manifest.json"),
+                                "B_noul_text",
+                            ])
+                        post.assert_not_called()
+                    self.assertFalse((here / "results").exists())
+
+    def test_nonfinite_unused_manifest_field_is_not_reproducible_json(self):
+        with tempfile.TemporaryDirectory() as temp:
+            path = Path(temp) / "manifest.json"
+            path.write_bytes(self.manifest_bytes[:-1] + b', "unreviewed": NaN}')
+            with self.assertRaisesRegex(ValueError, "nonfinite JSON constant"):
+                qualification.load_manifest(path)
+
     def test_positive_and_negative_holdout_is_separate_from_tuning(self):
         verified = self.valid()
         self.assertEqual(verified["heldout_ids"], ["holdout-positive", "holdout-negative"])
diff --git a/evals/intent_engine/test_qualification_cli.py b/evals/intent_engine/test_qualification_cli.py
index 9b920b4f3f3..073e281f0b6 100644
--- a/evals/intent_engine/test_qualification_cli.py
+++ b/evals/intent_engine/test_qualification_cli.py
@@ -47,6 +47,26 @@ def published_report(self, process):
         self.assertFalse((self.root / "summary.json").exists())
         return report
 
+    def test_duplicate_result_keys_cannot_publish_a_passing_report(self):
+        valid = self.fixture.signed_result()
+        raw = json.dumps(valid, allow_nan=False)
+        self.assertIn('"config": "trial_B_noul_text"', raw)
+        (self.root / "result.json").write_text(
+            raw.replace('"config": "trial_B_noul_text"',
+                        '"config": "trial_B_noul_text", "config": "trial_B_noul_text"', 1),
+            encoding="utf-8",
+        )
+        process = subprocess.run(
+            [sys.executable, str(self.root / "score.py"), "--qualified",
+             str(self.root / "manifest.json"), str(self.root / "dataset.json"),
+             str(self.root / "result.json")],
+            cwd=self.root, capture_output=True, text=True, timeout=30,
+        )
+        self.assertNotEqual(process.returncode, 0)
+        self.assertIn("duplicate JSON object key", process.stderr)
+        self.assertEqual(process.stdout, "")
+        self.assertFalse((self.root / "qualification_summary.json").exists())
+
     def test_passing_qualification_exits_zero_with_matching_report(self):
         process = self.run_score(self.fixture.signed_result())
         self.assertEqual(process.returncode, 0, process.stderr)
diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index 7694063fd0e..259df838e89 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -78,6 +78,37 @@ def assert_refused_unchanged(self, extra=()):
         self.assertEqual(self.partial.read_bytes(), before)
         self.assertFalse(self.output.exists())
 
+    def test_ambiguous_http_json_is_rejected_before_response_admission(self):
+        for name, payload, message in (
+            ("duplicate_answer_map", b'{"answers": {}, "answers": {}}',
+             "duplicate JSON object key"),
+            ("nonfinite_unused_field", b'{"answers": {}, "debug": NaN}',
+             "nonfinite JSON constant"),
+        ):
+            with self.subTest(name=name):
+                with patch("urllib.request.build_opener") as opener:
+                    opener.return_value.open.return_value = io.BytesIO(payload)
+                    with self.assertRaisesRegex(ValueError, message):
+                        run_http.post("http://localhost:8009/v1/systemone",
+                                      {"state": "test", "questions": {}})
+
+    def test_ambiguous_partial_row_refuses_without_changing_resumable_evidence(self):
+        self.interrupt()
+        original = self.partial.read_bytes()
+        lines = original.decode("utf-8").splitlines()
+        self.assertEqual(len(lines), 2)
+        self.assertIn('"id": "first"', lines[1])
+        lines[1] = lines[1].replace('"id": "first"',
+                                    '"id": "first", "id": "first"', 1)
+        self.partial.write_text("\n".join(lines) + "\n")
+        self.assert_refused_unchanged()
+        # The original interrupted prefix remains safe to replay.
+        self.partial.write_bytes(original)
+        with patch.object(run_http, "post", side_effect=self.response) as post:
+            self.run_main()
+        post.assert_called_once()
+        self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2)
+
     def test_invalid_dataset_is_refused_before_network_or_partial_output(self):
         malformed = [
             ("not_a_list", {"cases": self.dataset}),

From a45f85a51f8d5a9e14284500cd33dd43b92ee6a7 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 05:32:05 -0800
Subject: [PATCH 19/76] fix(evals): require source counters for zero-truncation
 qualification

Refs #4535

A qualified result could claim dropped=0 on every case without retaining
any numeric counter from the original server response. The aggregate then
reported measured zero truncation, even with no source evidence.

Require each held-out row to retain a numeric direct drop count or native
original/used counter pair before claiming untruncated input. Keep the
failed measurements observable; add positive, missing-source and CLI
regressions without changing frozen thresholds or historical scoring.

DRAFT / NO MERGE: this is partial qualification integrity only; not
canonical Scout parity, independently reviewed labels or Kev inference.
---
 evals/intent_engine/HTTP_RUNNER.md            |  7 ++++--
 evals/intent_engine/qualification.py          | 14 ++++++++++-
 evals/intent_engine/run_http.py               | 13 ++++++++++
 evals/intent_engine/test_qualification.py     | 24 ++++++++++++++++++-
 evals/intent_engine/test_qualification_cli.py |  8 +++++--
 5 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 76c68ff0add..65f1061d5e1 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -212,8 +212,11 @@ abstention, positive-case recall, reconstructed JSON request sizes
 (min/p50/max/total and the frozen byte ceiling), input-token measurement
 coverage, observed peak and over-budget cases, p50/p95 measured end-to-end HTTP
 latency and the predeclared tolerances. A passing report requires every held-out case to
-report measured zero dropped state tokens and a consistent self-reported
-model name. Native Kev's optional truncation receipts use a top-level Boolean
+report measured zero dropped state tokens **backed by original numeric server
+telemetry on every held-out row**, and a consistent self-reported model name.
+An edited/legacy result with `dropped: 0` but no original numeric source
+counter cannot qualify merely because its aggregate looks untruncated.
+Native Kev's optional truncation receipts use a top-level Boolean
 `truncated` and numeric `usage.state_tokens` / `usage.state_tokens_used`,
 not `usage.state_tokens_dropped`. The runner reconstructs their difference
 and rejects inconsistent counts or flags before recording a row. The native
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 9597d42ef7d..2fa90ca9a87 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -230,7 +230,10 @@ def holdout_decision_metrics(rows, thresholds):
 def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     """Reproduce qualified predictions from the frozen original dataset."""
     from score import score_config
-    from run_http import STATES, request_bytes, validate_dataset, validate_row_truncation
+    from run_http import (
+        STATES, has_numeric_truncation_receipt, request_bytes, validate_dataset,
+        validate_row_truncation,
+    )
 
     if not isinstance(dataset_bytes, bytes):
         raise ValueError("qualification requires the original dataset bytes")
@@ -274,6 +277,10 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     # consistent even if the result JSON was edited after the HTTP response.
     for row in rows:
         validate_row_truncation(row)
+    # Normalized dropped=0 can be forged or survive an older partial log with
+    # no original numeric server counters. Such a log may be scored, but it
+    # cannot establish that the model evaluated an untruncated input.
+    source_truncation_cases = sum(has_numeric_truncation_receipt(row) for row in rows)
     # Reconstruct real HTTP request bytes: self-consistent metadata hashes
     # alone cannot prove the frozen prompts were used.
     renderer = STATES.get(experiment.get("config"))
@@ -348,6 +355,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     complete_input_evidence = (
         latency["truncation_measured_cases"] == len(rows)
         and latency["cases_truncated"] == 0
+        and source_truncation_cases == len(rows)
     )
     checks = {
         "none_false_hint_rate": rates["none_false_hint_rate"] <= tolerances["max_none_false_hint_rate"],
@@ -386,6 +394,10 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
             "frozen_limit": validated["max_input_tokens"],
         },
         "latency": latency,
+        "numeric_truncation_source_evidence": {
+            "measured_cases": source_truncation_cases,
+            "missing_cases": len(rows) - source_truncation_cases,
+        },
         "checks": checks,
         "qualified_against_frozen_tolerances": all(checks.values()),
         "limits": "Reviewer and serving/model identities and token counts are self-reported, not authenticated or checkpoint-attested; task effectiveness and canonical Scout transcript parity are not proven.",
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 7dfdc8366da..4346b349d37 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -264,6 +264,19 @@ def dropped_state_tokens(response, usage):
     return dropped
 
 
+def has_numeric_truncation_receipt(row):
+    """A normalized drop count alone is not original server telemetry.
+
+    Native Kev supplies original/used token counts; other servers may emit
+    an explicit dropped-token count. Neither a bare Boolean flag nor a
+    self-authored normalized `dropped` field proves a zero-drop input.
+    """
+    return type(row.get("reported_state_tokens_dropped")) is int or (
+        type(row.get("state_tokens")) is int
+        and type(row.get("state_tokens_used")) is int
+    )
+
+
 def validate_row_truncation(row):
     """Recheck persisted server counters; normalized dropped alone is mutable.
 
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 40191d289b3..5d155c8320e 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -249,6 +249,7 @@ def native_answer(url, body):
                 self.dataset_bytes,
             )
             self.assertEqual(report["latency"]["cases_truncated"], 1)
+            self.assertEqual(report["numeric_truncation_source_evidence"]["measured_cases"], 2)
             self.assertFalse(report["checks"]["untruncated_input_evidence"])
             self.assertFalse(report["qualified_against_frozen_tolerances"])
             self.assertTrue(report["checks"]["reported_input_token_budget"])
@@ -372,7 +373,9 @@ def test_missing_or_dropped_server_tokens_cannot_qualify(self):
         for value in (None, 1):
             with self.subTest(state_tokens_dropped=value):
                 result = self.signed_result()
-                result["rows"][0]["dropped"] = value
+                result["rows"][0].update(
+                    dropped=value, reported_state_tokens_dropped=value,
+                )
                 report = qualification.qualified_report(
                     result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes
                 )
@@ -389,6 +392,24 @@ def test_missing_or_dropped_server_tokens_cannot_qualify(self):
         self.assertTrue(complete["checks"]["untruncated_input_evidence"])
         self.assertTrue(complete["qualified_against_frozen_tolerances"])
 
+    def test_forged_zero_drops_without_server_counter_does_not_qualify(self):
+        result = self.signed_result()
+        del result["rows"][0]["reported_state_tokens_dropped"]
+        report = qualification.qualified_report(
+            result, self.manifest, self.manifest_sha,
+            self.questions, self.dataset_bytes,
+        )
+        # The old aggregate accepted two claimed zeros even though one row
+        # had no original numeric server evidence. That is not measured zero.
+        self.assertEqual(report["latency"]["cases_truncated"], 0)
+        self.assertEqual(report["latency"]["truncation_measured_cases"], 2)
+        self.assertEqual(
+            report["numeric_truncation_source_evidence"],
+            {"measured_cases": 1, "missing_cases": 1},
+        )
+        self.assertFalse(report["checks"]["untruncated_input_evidence"])
+        self.assertFalse(report["qualified_against_frozen_tolerances"])
+
     def test_reviewed_labels_are_not_inferred_from_prediction_or_actions(self):
         for name, mutate, refusal in (
             ("self_label", lambda m: m["annotations"][2].update(reviewer="case-creator"), "self-labelled"),
@@ -441,6 +462,7 @@ def signed_result(self):
                 "ms": 10 if case["labels"] else 20,
                 "input_tokens": 64,
                 "dropped": 0,
+                "reported_state_tokens_dropped": 0,
                 "request_sha256": hashlib.sha256(rendered).hexdigest(),
                 "request_size_bytes": len(rendered),
                 "observed_model": "self-reported-test-model",
diff --git a/evals/intent_engine/test_qualification_cli.py b/evals/intent_engine/test_qualification_cli.py
index 073e281f0b6..d2262ceea4f 100644
--- a/evals/intent_engine/test_qualification_cli.py
+++ b/evals/intent_engine/test_qualification_cli.py
@@ -85,9 +85,13 @@ def test_failed_checks_exit_nonzero_and_replace_a_previous_passing_report(self):
              lambda r: r["rows"][0]["probs"].update(impact=0.1)),
             ("slow_response", "p95_ms", lambda r: r["rows"][1].update(ms=21)),
             ("unknown_truncation", "untruncated_input_evidence",
-             lambda r: r["rows"][0].update(dropped=None)),
+             lambda r: r["rows"][0].update(
+                 dropped=None, reported_state_tokens_dropped=None)),
             ("truncated_input", "untruncated_input_evidence",
-             lambda r: r["rows"][0].update(dropped=1)),
+             lambda r: r["rows"][0].update(
+                 dropped=1, reported_state_tokens_dropped=1)),
+            ("forged_zero_without_numeric_source", "untruncated_input_evidence",
+             lambda r: r["rows"][0].pop("reported_state_tokens_dropped")),
             ("unmeasured_input_tokens", "reported_input_token_budget",
              lambda r: r["rows"][0].update(input_tokens=None)),
             ("zero_input_tokens", "reported_input_token_budget",

From 0b7ead4adb08e391f6e50317cd2b5013f186611b Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 06:27:47 -0800
Subject: [PATCH 20/76] fix(evals): refuse unknown model identity before
 held-out inference

Refs #4535

The frozen scorer correctly rejected a result containing a missing reported
model, but the runner still appended that row and continued held-out model
calls. A partial with an unselected default model could likewise lose its
reported identity and be resumed before the scorer discovered the defect.

Require a nonblank response model for every qualified request, including a
null selector, before durable append or resumed inference. Retain the
unqualified historical behavior. Add falsifiable fresh-run and partial-resume
regressions with a valid recovery control, and align old test fixtures.

DRAFT / NO MERGE. Actual Kev inference, canonical Scout export, production
renderer parity, independent labels and Bazel verification remain pending.
---
 evals/intent_engine/HTTP_RUNNER.md        |  12 +--
 evals/intent_engine/run_http.py           |   2 +
 evals/intent_engine/test_qualification.py | 101 +++++++++++++++++++++-
 3 files changed, 107 insertions(+), 8 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 65f1061d5e1..1857da284cc 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -157,11 +157,13 @@ qualification independently reconstructs and verifies the byte count of every
 held-out request. Every HTTP result also records the model name *reported by
 the server*, including explicit null when the endpoint omits it. A frozen-arm
 qualification fails if this name is missing, differs among held-out responses,
-or contradicts a non-null requested model selector. For an explicit selector,
-a contradictory response is rejected before appending the row. A change in the
-server-reported name between cases is also refused before the second append;
-resuming an interrupted frozen arm rejects a stored contradictory name before
-sending more requests. The reported name is **not** cryptographic server
+or contradicts a non-null requested model selector. A frozen arm now requires a nonblank server-reported model on **each** response,
+even with no explicit model selector. Missing identities are rejected before
+appending the response; missing or blank identities in an existing partial are
+rejected before sending more requests. For an explicit selector, a contradictory
+response is likewise rejected before appending. A change in the server-reported
+name between cases is refused before the second append; valid interrupted runs
+still resume with their recorded model identity. The reported name is **not** cryptographic server
 identity or checkpoint attestation; a
 remote service can misreport it.
 
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 4346b349d37..2afa785a370 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -307,6 +307,8 @@ def checked_qualified_model(previous, reported, requested):
     A server name is not checkpoint attestation, but neither a resumed row
     nor a new response may contradict the selected or previously seen name.
     """
+    if not isinstance(reported, str) or not reported.strip():
+        raise ValueError("qualified HTTP response requires a server-reported model")
     if requested is not None and reported != requested:
         raise ValueError("qualified HTTP reported model differs from frozen selector")
     if previous is not None and reported is not None and reported != previous:
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 5d155c8320e..8853d5c7d8e 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -691,6 +691,101 @@ def test_qualified_resume_refuses_forged_model_before_network(self):
                         (here / "results" / "model-resume_B_noul_text.json").read_text()
                     )["rows"]), 2)
 
+    def test_frozen_arm_refuses_missing_reported_model_before_appending(self):
+        # With no explicit model selector, an absent identity used to be
+        # appended and the runner continued spending holdout calls.
+        good = {
+            "model": "observed-checkpoint",
+            "answers": {key: {"type": "noul", "noul": 0.5}
+                        for key in self.questions},
+            "usage": {"input_tokens": 64, "state_tokens": 32,
+                      "state_tokens_used": 32},
+            "truncated": False,
+        }
+        for index in (0, 1):
+            with self.subTest(failing_case=index), tempfile.TemporaryDirectory() as temp:
+                here = Path(temp)
+                (here / "dataset.json").write_bytes(self.dataset_bytes)
+                (here / "manifest.json").write_bytes(self.manifest_bytes)
+                (here / "server.json").write_text(json.dumps(self.server_config))
+                replies = [copy.deepcopy(good), copy.deepcopy(good)]
+                del replies[index]["model"]
+                args = [
+                    "--url", self.endpoint, "--tag", "missing-model",
+                    "--server-config", str(here / "server.json"),
+                    "--qualification-manifest", str(here / "manifest.json"),
+                    "B_noul_text",
+                ]
+                with patch.object(run_http, "HERE", str(here)), patch.object(
+                    run_http, "post", side_effect=[(reply, 10.0) for reply in replies]
+                ) as post:
+                    with self.assertRaisesRegex(ValueError, "requires a server-reported model"):
+                        run_http.main(args)
+                    self.assertEqual(post.call_count, index + 1)
+                completed = here / "results" / "missing-model_B_noul_text.json"
+                self.assertFalse(completed.exists())
+                partial = Path(str(completed) + ".partial")
+                recorded = partial.read_text().splitlines()
+                self.assertEqual(len(recorded), index + 1)
+                if index:
+                    self.assertEqual(json.loads(recorded[1])["observed_model"],
+                                     "observed-checkpoint")
+
+    def test_frozen_resume_refuses_missing_default_model_before_network(self):
+        # No --send-model: absent/null/blank names must still be refused before
+        # another held-out inference. An unchanged valid prefix can resume.
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            args = [
+                "--url", self.endpoint, "--tag", "missing-resume-model",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "B_noul_text",
+            ]
+            reply = {
+                "model": "observed-checkpoint",
+                "answers": {key: {"type": "noul", "noul": 0.5}
+                            for key in self.questions},
+                "usage": {"input_tokens": 64, "state_tokens": 32,
+                          "state_tokens_used": 32},
+                "truncated": False,
+            }
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post", side_effect=[
+                    (reply, 10.0), OSError("interrupted")
+                ]):
+                    with self.assertRaisesRegex(OSError, "interrupted"):
+                        run_http.main(args)
+                completed = here / "results" / "missing-resume-model_B_noul_text.json"
+                partial = Path(str(completed) + ".partial")
+                original = partial.read_bytes()
+                header, row = original.decode("utf-8").splitlines()
+                for changed in ("absent", "null", "blank"):
+                    with self.subTest(changed=changed):
+                        forged = json.loads(row)
+                        if changed == "absent":
+                            del forged["observed_model"]
+                        else:
+                            forged["observed_model"] = None if changed == "null" else " "
+                        partial.write_text(header + "\n" + json.dumps(forged) + "\n")
+                        before = partial.read_bytes()
+                        with patch.object(run_http, "post") as blocked:
+                            with self.assertRaisesRegex(ValueError, "requires a server-reported model"):
+                                run_http.main(args)
+                            blocked.assert_not_called()
+                        self.assertEqual(partial.read_bytes(), before)
+                        self.assertFalse(completed.exists())
+                partial.write_bytes(original)
+                with patch.object(run_http, "post", return_value=(reply, 10.0)) as remaining:
+                    run_http.main(args)
+                    remaining.assert_called_once()
+                self.assertEqual([r["observed_model"] for r in json.loads(
+                    completed.read_text()
+                )["rows"]], ["observed-checkpoint", "observed-checkpoint"])
+
     def test_qualification_requires_complete_serving_identity(self):
         for field, change in (
             ("missing_server", lambda r: r["experiment"].pop("server_config")),
@@ -760,7 +855,7 @@ def test_http_runner_requires_frozen_client_calibration_before_network(self):
                 frozen = copy.deepcopy(self.manifest)
                 frozen["noul_temperature"] = 1.25
                 manifest_path.write_text(json.dumps(frozen))
-                answer = {"answers": {
+                answer = {"model": "self-reported-test-model", "answers": {
                     key: {"type": "noul", "noul": 0.5}
                     for key in self.questions
                 }}
@@ -864,7 +959,7 @@ def test_changed_manifest_cannot_resume_or_spend_inference_tokens(self):
                 "--qualification-manifest", str(manifest_path),
                 "B_noul_text",
             ]
-            answer = {"answers": {
+            answer = {"model": "self-reported-test-model", "answers": {
                 key: {"type": "noul", "noul": 0.5} for key in self.questions
             }}
             with patch.object(run_http, "HERE", str(here)):
@@ -982,7 +1077,7 @@ def test_runner_can_consume_explicit_frozen_input_file(self):
             dataset_path.write_bytes(self.dataset_bytes)
             (here / "manifest.json").write_bytes(self.manifest_bytes)
             (here / "server.json").write_text(json.dumps(self.server_config))
-            answer = {"answers": {
+            answer = {"model": "self-reported-test-model", "answers": {
                 key: {"type": "noul", "noul": 0.5} for key in self.questions
             }}
             with patch.object(run_http, "HERE", str(here)):

From d8588131550fe833238858c735cbcf7ca91406eb Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 07:20:03 -0800
Subject: [PATCH 21/76] feat(evals): score frozen nonhistorical Noul
 vocabularies

Refs #4535

Allow a declared, digest-frozen question file to drive the existing HTTP
runner and qualification scorer without assuming historical nine-category
metrics. Reject unsupported or changed question files before inference.
This is preparation for canonical Scout parity, not parity evidence.
---
 evals/intent_engine/HTTP_RUNNER.md        | 17 +++++
 evals/intent_engine/qualification.py      | 30 +++++++-
 evals/intent_engine/run_http.py           |  7 +-
 evals/intent_engine/score.py              | 48 ++++++------
 evals/intent_engine/test_qualification.py | 93 +++++++++++++++++++++++
 5 files changed, 169 insertions(+), 26 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 1857da284cc..af0d9128351 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -239,3 +239,20 @@ alone demonstrate task success, ignored/acted outcomes, cold/warm cost, native
 runtime renderer/settings parity or a real Kev checkpoint. All those require
 canonical transcript export, independent review, real host inference and
 application-specific trials before any replacement or merge decision.
+
+## Declared question vocabulary (qualification only)
+
+The historical benchmark uses nine fixed categories. A frozen held-out run
+can instead use `--questions-file /path/to/questions.json` alongside
+`--qualification-manifest`. The JSON object maps each configured key to
+`{"type":"noul","instructions":"..."}`; the manifest's `questions_sha256`
+must pin the exact question object. Score with
+`score.py --qualified MANIFEST DATASET RESULT --questions-file FILE`.
+Both commands reject changed or non-Noul instructions; historical scoring
+retains its original nine-category behavior.
+
+**Not runtime parity:** An operator-supplied file does not establish that the
+actual configured Scout questions, settings digest, window renderer or
+rehydrated transcript were used. Obtain that evidence from the production
+settings and owning transcript authority before making any application
+replacement decision. The runner itself does not export canonical Scout data.
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 2fa90ca9a87..dcc599a16bb 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -29,6 +29,33 @@ def reject_nonfinite(constant):
                       parse_constant=reject_nonfinite)
 
 
+def load_questions(path):
+    """Read a frozen Noul question vocabulary supplied by the settings owner.
+
+    A question file is only a declared contract. Its provenance and the Scout
+    renderer/window digest must be established separately before runtime parity.
+    """
+    with open(path, "rb") as source:
+        questions = loads_unambiguous_json(source.read())
+    return validate_questions(questions)
+
+
+def validate_questions(questions):
+    if not isinstance(questions, dict) or not questions:
+        raise ValueError("qualification requires a nonempty Noul question vocabulary")
+    for key, question in questions.items():
+        if not isinstance(key, str) or not key.strip():
+            raise ValueError("qualification question keys must be nonempty strings")
+        if (not isinstance(question, dict)
+                or set(question) != {"type", "instructions"}
+                or question["type"] != "noul"
+                or not isinstance(question["instructions"], str)
+                or not question["instructions"].strip()
+                or len(question["instructions"].encode("utf-8")) > 512):
+            raise ValueError(f"qualification question {key!r} must be a bounded Noul instruction")
+    return questions
+
+
 def sha256_bytes(data):
     return hashlib.sha256(data).hexdigest()
 
@@ -55,6 +82,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
         raise ValueError("invalid qualification dataset digest")
     if manifest.get("dataset_sha256") != dataset_sha:
         raise ValueError("qualification dataset digest mismatch")
+    validate_questions(questions)
     if manifest.get("questions_sha256") != sha256_json(questions):
         raise ValueError("qualification question vocabulary or instruction mismatch")
 
@@ -324,7 +352,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
              or reported_model == arm["requested_model"])
     )
 
-    scores = score_config(result, fixed_t=validated["thresholds"])
+    scores = score_config(result, fixed_t=validated["thresholds"], cats=tuple(questions))
     fixed = scores["fixed"]["all"]
     latency = scores["latency"]
     decisions = holdout_decision_metrics(rows, validated["thresholds"])
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 2afa785a370..e64466f5f58 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -23,7 +23,7 @@
 HERE = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, HERE)
 from common import calibrated, noul_questions, prompt_only, text_window
-from qualification import load_manifest, loads_unambiguous_json, validate_manifest
+from qualification import load_manifest, load_questions, loads_unambiguous_json, validate_manifest
 
 STATES = {
     "A_noul_prompt": prompt_only,
@@ -485,6 +485,7 @@ def main(argv=None):
     ap.add_argument("--subset")
     ap.add_argument("--dataset", help="original JSON cases; frozen digest recorded in each run")
     ap.add_argument("--qualification-manifest", help="freeze reviewed held-out labels, groups, thresholds and tolerances")
+    ap.add_argument("--questions-file", help="declared typed Noul questions; qualified runs only, requires separate runtime provenance")
     ap.add_argument("--send-model")
     ap.add_argument("--bearer-token-env", metavar="NAME",
                     help="name of an environment variable holding a bearer token (never recorded)")
@@ -507,7 +508,9 @@ def main(argv=None):
         dataset_bytes = source.read()
     dataset = loads_unambiguous_json(dataset_bytes)
     dataset_sha = hashlib.sha256(dataset_bytes).hexdigest()
-    questions = noul_questions()
+    if args.questions_file and not args.qualification_manifest:
+        raise ValueError("--questions-file requires --qualification-manifest")
+    questions = load_questions(args.questions_file) if args.questions_file else noul_questions()
     case_ids = validate_dataset(dataset, questions)
     qualification_sha = None
     if args.qualification_manifest:
diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py
index 0af47dfedc8..06c112fe1ff 100644
--- a/evals/intent_engine/score.py
+++ b/evals/intent_engine/score.py
@@ -26,11 +26,11 @@ def publish_summary(path, summary):
             os.unlink(staged)
 
 
-def validate_rows(rows, config):
+def validate_rows(rows, config, cats=CATS):
     """Refuse invalid measured evidence before any threshold or AUC arithmetic."""
     if not isinstance(rows, list) or not rows:
         raise ValueError("configuration %r has no rows to score" % config)
-    categories = set(CATS)
+    categories = set(cats)
     seen = set()
     for row in rows:
         if not isinstance(row, dict):
@@ -172,7 +172,7 @@ def case_metrics(rows, ths):
     none_rows = [r for r in rows if not r["labels"]]
     none_fh = 0
     for r in rows:
-        fired = {c for c in CATS if r["probs"][c] >= ths[c]}
+        fired = {c for c in ths if r["probs"][c] >= ths[c]}
         exact += fired == set(r["labels"])
         passing = [(r["probs"][c] - ths[c], c) for c in fired]
         pick = max(passing)[1] if passing else None
@@ -225,13 +225,13 @@ def lat(rows):
         result[f"{field}_measured_cases"] = len(values)
     return result
 
-def score_config(res, fixed_t=None):
+def score_config(res, fixed_t=None, cats=CATS):
     if "experiment" in res:
         experiment = res["experiment"]
         if not isinstance(experiment, dict) or experiment.get("version") != 2:
             raise ValueError("HTTP result telemetry version is unsupported; rerun with a new tag")
     rows = res["rows"]
-    validate_rows(rows, res.get("config"))
+    validate_rows(rows, res.get("config"), cats)
     validate_calibration(res, rows)
     validate_http_provenance(res, rows)
     choices = ["choice" in row for row in rows]
@@ -239,7 +239,7 @@ def score_config(res, fixed_t=None):
         raise ValueError("configuration %r has incomplete choice coverage" % res.get("config"))
     ctx = [r for r in rows if r["context_needed"]]; nctx = [r for r in rows if not r["context_needed"]]
     per = {}
-    for c in CATS:
+    for c in cats:
         pos = [r["probs"][c] for r in rows if c in r["labels"]]
         neg = [r["probs"][c] for r in rows if c not in r["labels"]]
         sweep = {str(t): prf(rows, c, t) for t in TH}
@@ -249,7 +249,7 @@ def score_config(res, fixed_t=None):
             "sweep": sweep, "best_t": best, "best_f1": sweep[str(best)]["f1"]}
     # A singleton has no training rows after holding out its only case.
     loo_cases = len(rows) if len(rows) > 1 else 0
-    for c in CATS:
+    for c in cats:
         if not loo_cases:
             per[c].update(loo_f1=None, loo_p=None, loo_r=None)
             continue
@@ -278,19 +278,19 @@ def training_f1(t):
         per[c]["loo_p"] = tp / (tp + fp) if tp + fp else None
         per[c]["loo_r"] = tp / (tp + fn) if tp + fn else None
     out = {"per_cat": per, "loo_evaluated_cases": loo_cases,
-           "macro_loo_f1": statistics.mean(per[c]["loo_f1"] or 0 for c in CATS) if loo_cases else None}
-    for name, t in [("t0.5", {c: 0.5 for c in CATS}), ("t0.6", {c: 0.6 for c in CATS}),
-                    ("t0.7", {c: 0.7 for c in CATS}),
-                    ("best_t_in_sample", {c: per[c]["best_t"] for c in CATS})] + ([("fixed", fixed_t)] if fixed_t else []):
-        mf = statistics.mean(prf(rows, c, t[c])["f1"] or 0 for c in CATS)
+           "macro_loo_f1": statistics.mean(per[c]["loo_f1"] or 0 for c in cats) if loo_cases else None}
+    for name, t in [("t0.5", {c: 0.5 for c in cats}), ("t0.6", {c: 0.6 for c in cats}),
+                    ("t0.7", {c: 0.7 for c in cats}),
+                    ("best_t_in_sample", {c: per[c]["best_t"] for c in cats})] + ([("fixed", fixed_t)] if fixed_t else []):
+        mf = statistics.mean(prf(rows, c, t[c])["f1"] or 0 for c in cats)
         out[name] = {"macro_f1": mf,
-                     "macro_f1_ctx": statistics.mean(prf(ctx, c, t[c])["f1"] or 0 for c in CATS) if ctx else None,
+                     "macro_f1_ctx": statistics.mean(prf(ctx, c, t[c])["f1"] or 0 for c in cats) if ctx else None,
                      "all": case_metrics(rows, t), "ctx": case_metrics(ctx, t), "no_ctx": case_metrics(nctx, t)}
-    out["macro_auc"] = mean_defined(per[c]["auc"] for c in CATS)
-    out["macro_auc_ctx"] = mean_defined(per[c]["auc_ctx"] for c in CATS)
-    out["macro_auc_n_categories"] = sum(per[c]["auc"] is not None for c in CATS)
-    out["macro_auc_ctx_n_categories"] = sum(per[c]["auc_ctx"] is not None for c in CATS)
-    out["reliability"] = reliability(rows)
+    out["macro_auc"] = mean_defined(per[c]["auc"] for c in cats)
+    out["macro_auc_ctx"] = mean_defined(per[c]["auc_ctx"] for c in cats)
+    out["macro_auc_n_categories"] = sum(per[c]["auc"] is not None for c in cats)
+    out["macro_auc_ctx_n_categories"] = sum(per[c]["auc_ctx"] is not None for c in cats)
+    out["reliability"] = reliability(rows, cats)
     out["latency"] = lat(rows)
     if all(choices):
         def ch(rs):
@@ -301,22 +301,24 @@ def ch(rs):
         out["choice_argmax_per_cat"] = {c: {
             "tp": sum(r["choice"] == c and c in r["labels"] for r in rows),
             "fp": sum(r["choice"] == c and c not in r["labels"] for r in rows),
-            "fn": sum(r["choice"] != c and c in r["labels"] for r in rows)} for c in CATS}
+            "fn": sum(r["choice"] != c and c in r["labels"] for r in rows)} for c in cats}
     return out
 
 if __name__ == "__main__":
     here = os.path.dirname(os.path.abspath(__file__))
     if len(sys.argv) > 1:
-        if len(sys.argv) != 5 or sys.argv[1] != "--qualified":
-            raise ValueError("usage: score.py [--qualified MANIFEST DATASET RESULT]")
-        from qualification import load_manifest, loads_unambiguous_json, qualified_report
+        if (sys.argv[1] != "--qualified" or len(sys.argv) not in (5, 7)
+                or (len(sys.argv) == 7 and sys.argv[5] != "--questions-file")):
+            raise ValueError("usage: score.py [--qualified MANIFEST DATASET RESULT [--questions-file FILE]]")
+        from qualification import load_manifest, load_questions, loads_unambiguous_json, qualified_report
         from common import noul_questions
         manifest, manifest_sha = load_manifest(sys.argv[2])
         with open(sys.argv[3], "rb") as source:
             dataset_bytes = source.read()
         with open(sys.argv[4], encoding="utf-8") as source:
             result = loads_unambiguous_json(source.read())
-        report = qualified_report(result, manifest, manifest_sha, noul_questions(), dataset_bytes)
+        questions = load_questions(sys.argv[6]) if len(sys.argv) == 7 else noul_questions()
+        report = qualified_report(result, manifest, manifest_sha, questions, dataset_bytes)
         publish_summary(os.path.join(here, "qualification_summary.json"), report)
         print(json.dumps(report, indent=2, allow_nan=False))
         # A completed evaluation is not necessarily a qualified model. Keep
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 8853d5c7d8e..d43e6acdeed 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1270,5 +1270,98 @@ def test_qualification_reconstructs_measured_request_sizes(self):
                 forged, tighter, tighter_sha, self.questions, self.dataset_bytes,
             )
 
+
+    def test_frozen_ten_question_vocabulary_runs_and_scores_without_historical_categories(self):
+        # This fixture exercises a declared ten-question contract; it is not a
+        # canonical Scout export and never claims Kev inference or parity.
+        questions = {
+            key: value for key, value in self.questions.items()
+            if key != "refactor_edit"
+        }
+        questions["atomic_edit"] = {
+            "type": "noul", "instructions": "Is the user requesting a mechanical edit?"
+        }
+        questions["build_diagnostics"] = {
+            "type": "noul", "instructions": "Does the user need a build error fixed?"
+        }
+        self.assertEqual(len(questions), 10)
+        manifest = copy.deepcopy(self.manifest)
+        manifest["questions_sha256"] = qualification.sha256_json(questions)
+        manifest["thresholds"] = {key: 0.6 for key in questions}
+        manifest_sha = qualification.sha256_bytes(json.dumps(manifest).encode())
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_text(json.dumps(manifest))
+            (here / "questions.json").write_text(json.dumps(questions))
+            (here / "server.json").write_text(json.dumps(self.server_config))
+
+            def inference(_url, body):
+                self.assertEqual(set(body["questions"]), set(questions))
+                probability = 0.9 if "What changes?" in body["state"] else 0.1
+                return ({
+                    "model": "kev-declared",
+                    "answers": {
+                        key: {"noul": probability if key == "impact" else 0.1}
+                        for key in body["questions"]
+                    },
+                    "usage": {"input_tokens": 42, "state_tokens_dropped": 0},
+                    "latency_ms": 5.0,
+                }, 5.0)
+
+            args = [
+                "--url", self.endpoint, "--tag", "ten-questions",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "--questions-file", str(here / "questions.json"),
+                "B_noul_text",
+            ]
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post", side_effect=inference) as post:
+                    run_http.main(args)
+                self.assertEqual(post.call_count, 2)
+                output = here / "results" / "ten-questions_B_noul_text.json"
+                result = json.loads(output.read_text())
+                report = qualification.qualified_report(
+                    result, manifest, manifest_sha, questions, self.dataset_bytes,
+                )
+                self.assertTrue(report["qualified_against_frozen_tolerances"])
+                self.assertEqual(report["cases"], 2)
+                self.assertEqual(len(score.score_config(
+                    result, fixed_t=manifest["thresholds"], cats=tuple(questions)
+                )["per_cat"]), 10)
+                self.assertEqual(set(report["thresholds"]), set(questions))
+                self.assertNotIn("refactor_edit", report["thresholds"])
+                # The default historical vocabulary must not certify ten questions.
+                with self.assertRaisesRegex(ValueError, "question vocabulary"):
+                    qualification.qualified_report(
+                        result, manifest, manifest_sha,
+                        self.questions, self.dataset_bytes,
+                    )
+                altered = copy.deepcopy(questions)
+                altered["atomic_edit"]["instructions"] = "A changed instruction"
+                (here / "questions.json").write_text(json.dumps(altered))
+                args[3] = "another-tag"
+                with patch.object(run_http, "post") as blocked:
+                    with self.assertRaisesRegex(ValueError, "question vocabulary"):
+                        run_http.main(args)
+                    blocked.assert_not_called()
+                self.assertFalse((here / "results" / "another-tag_B_noul_text.json.partial").exists())
+
+    def test_qualified_question_files_refuse_non_noul_or_ambiguous_vocabulary(self):
+        with tempfile.TemporaryDirectory() as temp:
+            path = Path(temp) / "questions.json"
+            for body, expected in (
+                ('{"q": {"type": "choice", "instructions": "Pick"}}', "Noul"),
+                ('{"q": {"type": "noul", "instructions": ""}}', "Noul"),
+                ('{"q": {"type": "noul", "instructions": "Good"}, '
+                 '"q": {"type": "noul", "instructions": "Good"}}', "duplicate JSON"),
+                ('{"q": {"type": "noul", "instructions": "Good", "extra": 1}}', "Noul"),
+            ):
+                with self.subTest(body=body):
+                    path.write_text(body)
+                    with self.assertRaisesRegex(ValueError, expected):
+                        qualification.load_questions(path)
+
 if __name__ == "__main__":
     unittest.main()

From 6ca5cfacb5863474442b1fd67fed8e1ad2cbc494 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 07:56:26 -0800
Subject: [PATCH 22/76] fix(evals): freeze Noul question order before held-out
 scoring

Refs #4535

Question manifests previously hashed sorted JSON objects, even though the
HTTP runner sends questions in insertion order. Reordering the same frozen
vocabulary could select a second held-out model arm without changing the
manifest's question digest. Bind the actual ordered question sequence, retain
field-order normalization, and reject reordered inference/scoring in tests.

This is a qualification-integrity fix, not canonical Scout or Kev evidence.
---
 evals/intent_engine/HTTP_RUNNER.md        | 13 +++--
 evals/intent_engine/qualification.py      | 14 ++++-
 evals/intent_engine/test_qualification.py | 63 ++++++++++++++++++++++-
 3 files changed, 83 insertions(+), 7 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index af0d9128351..92a90fc2704 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -126,9 +126,11 @@ metrics; it cannot reconstruct a canonical Scout transcript or attest a model.
 
 Create a separate JSON annotation manifest **before running inference** with
 `version: 1`, `application`, the SHA-256 of the original dataset file
-(`dataset_sha256`), and `questions_sha256` equal to the canonical
-`qualification.sha256_json(common.noul_questions())` of the runner's exact
-question dictionary. This last digest must change with question text. Include
+(`dataset_sha256`), and `questions_sha256` equal to
+`qualification.sha256_ordered_questions(common.noul_questions())` for the
+runner's exact question dictionary. This digest freezes **question key order
+as emitted in the HTTP body**, as well as question names/instructions; a
+sorted-object digest does not prevent trying reordered held-out arms. Include
 an `arm` object predeclaring exactly one `config` renderer, a nullable
 `requested_model`, `endpoint_sha256` (SHA-256 of the exact URL bytes), and
 `server_config_sha256` (the sorted-JSON digest of the complete serving
@@ -246,7 +248,10 @@ The historical benchmark uses nine fixed categories. A frozen held-out run
 can instead use `--questions-file /path/to/questions.json` alongside
 `--qualification-manifest`. The JSON object maps each configured key to
 `{"type":"noul","instructions":"..."}`; the manifest's `questions_sha256`
-must pin the exact question object. Score with
+must pin the ordered question contract via
+`qualification.sha256_ordered_questions(questions)`. A changed insertion
+order of question keys is a different arm and is refused before inference.
+Score with
 `score.py --qualified MANIFEST DATASET RESULT --questions-file FILE`.
 Both commands reject changed or non-Noul instructions; historical scoring
 retains its original nine-category behavior.
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index dcc599a16bb..1481f19c498 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -64,6 +64,18 @@ def sha256_json(value):
     return sha256_bytes(json.dumps(value, sort_keys=True, allow_nan=False).encode("utf-8"))
 
 
+def sha256_ordered_questions(questions):
+    """Freeze the actual question order sent to the HTTP model.
+
+    Sorted-object digests cannot bind order: two question files with identical
+    keys/instructions but different insertion orders produce different HTTP
+    request bodies and can produce different predictions on the same holdout.
+    Normalize only each question's own field order, not the sequence of keys.
+    """
+    validate_questions(questions)
+    return sha256_json([[key, question] for key, question in questions.items()])
+
+
 def load_manifest(path):
     with open(path, "rb") as source:
         raw = source.read()
@@ -83,7 +95,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     if manifest.get("dataset_sha256") != dataset_sha:
         raise ValueError("qualification dataset digest mismatch")
     validate_questions(questions)
-    if manifest.get("questions_sha256") != sha256_json(questions):
+    if manifest.get("questions_sha256") != sha256_ordered_questions(questions):
         raise ValueError("qualification question vocabulary or instruction mismatch")
 
     # A held-out split is not independent if the evaluator can try multiple
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index d43e6acdeed..5d463627982 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -55,7 +55,7 @@ def setUp(self):
         self.manifest = {
             "version": 1, "application": "historical-intent-evaluation",
             "dataset_sha256": self.dataset_sha,
-            "questions_sha256": qualification.sha256_json(self.questions),
+            "questions_sha256": qualification.sha256_ordered_questions(self.questions),
             "arm": {
                 "config": "B_noul_text", "requested_model": None,
                 "endpoint_sha256": qualification.sha256_bytes(self.endpoint.encode("utf-8")),
@@ -1271,6 +1271,65 @@ def test_qualification_reconstructs_measured_request_sizes(self):
             )
 
 
+    def test_frozen_question_order_refuses_a_second_holdout_arm_before_inference(self):
+        # Sorted-key JSON digests collide across question reordering although
+        # the runner sends dict entries in insertion order. That let the same
+        # frozen manifest authorize multiple held-out request sequences.
+        reordered = dict(reversed(list(self.questions.items())))
+        self.assertEqual(set(reordered), set(self.questions))
+        self.assertEqual(
+            qualification.sha256_json(reordered),
+            qualification.sha256_json(self.questions),
+        )
+        self.assertNotEqual(
+            qualification.sha256_ordered_questions(reordered),
+            qualification.sha256_ordered_questions(self.questions),
+        )
+        # Reordering type/instructions fields within one question does not
+        # change the contract: only actual HTTP question sequence is frozen.
+        equivalent = {
+            key: {"instructions": item["instructions"], "type": item["type"]}
+            for key, item in self.questions.items()
+        }
+        self.assertEqual(
+            qualification.sha256_ordered_questions(equivalent),
+            qualification.sha256_ordered_questions(self.questions),
+        )
+        with self.assertRaisesRegex(ValueError, "question vocabulary or instruction"):
+            qualification.validate_manifest(
+                self.manifest, self.dataset_sha, reordered, self.dataset,
+            )
+
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "questions.json").write_text(json.dumps(reordered))
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post") as inference:
+                    with self.assertRaisesRegex(ValueError, "question vocabulary or instruction"):
+                        run_http.main([
+                            "--url", self.endpoint, "--tag", "reordered-holdout",
+                            "--dataset", str(here / "dataset.json"),
+                            "--server-config", str(here / "server.json"),
+                            "--qualification-manifest", str(here / "manifest.json"),
+                            "--questions-file", str(here / "questions.json"),
+                            "B_noul_text",
+                        ])
+                    inference.assert_not_called()
+                self.assertFalse(
+                    (here / "results" / "reordered-holdout_B_noul_text.json.partial").exists()
+                )
+
+        # An original run must not be requalified using the altered order,
+        # even though the question set and old sorted-object digest agree.
+        with self.assertRaisesRegex(ValueError, "question vocabulary or instruction"):
+            qualification.qualified_report(
+                self.signed_result(), self.manifest, self.manifest_sha,
+                reordered, self.dataset_bytes,
+            )
+
     def test_frozen_ten_question_vocabulary_runs_and_scores_without_historical_categories(self):
         # This fixture exercises a declared ten-question contract; it is not a
         # canonical Scout export and never claims Kev inference or parity.
@@ -1286,7 +1345,7 @@ def test_frozen_ten_question_vocabulary_runs_and_scores_without_historical_categ
         }
         self.assertEqual(len(questions), 10)
         manifest = copy.deepcopy(self.manifest)
-        manifest["questions_sha256"] = qualification.sha256_json(questions)
+        manifest["questions_sha256"] = qualification.sha256_ordered_questions(questions)
         manifest["thresholds"] = {key: 0.6 for key in questions}
         manifest_sha = qualification.sha256_bytes(json.dumps(manifest).encode())
         with tempfile.TemporaryDirectory() as temp:

From d5527caf30a411f8966092c214ab0a52c0b5e007 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 09:34:30 -0800
Subject: [PATCH 23/76] fix(evals): precommit frozen held-out request bodies

---
 evals/intent_engine/HTTP_RUNNER.md        | 12 ++++-
 evals/intent_engine/qualification.py      |  8 +++
 evals/intent_engine/run_http.py           | 10 +++-
 evals/intent_engine/test_qualification.py | 59 +++++++++++++++++++++++
 4 files changed, 86 insertions(+), 3 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 92a90fc2704..e693789aa8f 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -131,7 +131,17 @@ Create a separate JSON annotation manifest **before running inference** with
 runner's exact question dictionary. This digest freezes **question key order
 as emitted in the HTTP body**, as well as question names/instructions; a
 sorted-object digest does not prevent trying reordered held-out arms. Include
-an `arm` object predeclaring exactly one `config` renderer, a nullable
+`holdout_requests_sha256`, a precommitted SHA-256 of the ordered
+`[[case_id, request_sha256], ...]` pairs for the held-out cases. Construct each
+request SHA-256 from the exact serialized JSON body (the selected renderer
+state, ordered questions and optional model selector) **before** evaluating
+any holdout. The runner refuses a changed renderer output under an unchanged
+manifest before model calls or output creation. Qualified scoring independently
+checks the result experiment's request digest against this precommitment.
+Generate a new frozen manifest for a changed renderer; a renderer config name
+alone is not a code or output identity. This remains declared benchmark
+evidence, **not** production Scout transcript rehydration or runtime parity.
+Include an `arm` object predeclaring exactly one `config` renderer, a nullable
 `requested_model`, `endpoint_sha256` (SHA-256 of the exact URL bytes), and
 `server_config_sha256` (the sorted-JSON digest of the complete serving
 configuration, via `qualification.sha256_json(server_config)`). The runner
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 1481f19c498..e8af39b4f27 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -97,6 +97,11 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     validate_questions(questions)
     if manifest.get("questions_sha256") != sha256_ordered_questions(questions):
         raise ValueError("qualification question vocabulary or instruction mismatch")
+    # Renderer names are not immutable code identities. Freeze actual held-out
+    # HTTP request bytes before seeing any model predictions.
+    frozen_requests = manifest.get("holdout_requests_sha256")
+    if not isinstance(frozen_requests, str) or not _DIGEST.fullmatch(frozen_requests):
+        raise ValueError("qualification requires a frozen held-out request digest")
 
     # A held-out split is not independent if the evaluator can try multiple
     # renderers, model selectors or serving declarations and pick the winner.
@@ -230,6 +235,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
                 )
     return {
         "heldout_ids": heldout,
+        "holdout_requests_sha256": frozen_requests,
         "arm": arm,
         "max_request_bytes": request_limit,
         "max_input_tokens": token_limit,
@@ -305,6 +311,8 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
             or experiment.get("endpoint_sha256") != arm["endpoint_sha256"]
             or sha256_json(server) != arm["server_config_sha256"]):
         raise ValueError("qualification HTTP arm differs from frozen manifest")
+    if experiment.get("requests_sha256") != validated["holdout_requests_sha256"]:
+        raise ValueError("qualification rendered held-out requests differ from frozen manifest")
     rows = result.get("rows")
     if not isinstance(rows, list) or [r.get("id") for r in rows if isinstance(r, dict)] != validated["heldout_ids"]:
         raise ValueError("qualification result differs from frozen held-out case order")
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index e64466f5f58..d8e29562692 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -541,14 +541,20 @@ def main(argv=None):
         raise ValueError("the experiment must contain at least one case")
     cases = {case["id"]: case for case in dataset}
     if qualification_sha is not None:
-        # Bound exact JSON requests before creating result files or calling the model.
+        # Bound and verify precommitted request bodies before opening any
+        # output. A stable config name does not imply a stable renderer.
         renderer = STATES[qualified["arm"]["config"]]
+        rendered_requests = []
         for case in dataset:
             body = {"state": renderer(case), "questions": questions}
             if args.send_model:
                 body["model"] = args.send_model
-            if len(request_bytes(body)) > qualified["max_request_bytes"]:
+            encoded = request_bytes(body)
+            if len(encoded) > qualified["max_request_bytes"]:
                 raise ValueError(f"qualification case {case['id']} exceeds frozen request byte limit")
+            rendered_requests.append([case["id"], hashlib.sha256(encoded).hexdigest()])
+        if digest(rendered_requests) != qualified["holdout_requests_sha256"]:
+            raise ValueError("qualification rendered held-out requests differ from frozen manifest")
     os.makedirs(os.path.join(HERE, "results"), exist_ok=True)
     for cfg in args.configs:
         name = f"{args.tag}_{cfg}"
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 5d463627982..ea37e29366a 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -56,6 +56,7 @@ def setUp(self):
             "version": 1, "application": "historical-intent-evaluation",
             "dataset_sha256": self.dataset_sha,
             "questions_sha256": qualification.sha256_ordered_questions(self.questions),
+            "holdout_requests_sha256": self.holdout_request_digest(),
             "arm": {
                 "config": "B_noul_text", "requested_model": None,
                 "endpoint_sha256": qualification.sha256_bytes(self.endpoint.encode("utf-8")),
@@ -76,6 +77,19 @@ def setUp(self):
         self.manifest_bytes = json.dumps(self.manifest).encode("utf-8")
         self.manifest_sha = qualification.sha256_bytes(self.manifest_bytes)
 
+    def holdout_request_digest(self, questions=None, model=None, config="B_noul_text"):
+        questions = self.questions if questions is None else questions
+        fingerprints = []
+        for case in self.dataset[2:]:
+            body = {"state": run_http.STATES[config](case), "questions": questions}
+            if model is not None:
+                body["model"] = model
+            fingerprints.append([
+                case["id"],
+                hashlib.sha256(run_http.request_bytes(body)).hexdigest(),
+            ])
+        return score.evidence_digest(fingerprints)
+
     def valid(self, manifest=None, cases=None):
         return qualification.validate_manifest(
             self.manifest if manifest is None else manifest,
@@ -541,6 +555,7 @@ def test_qualification_reports_missing_or_drifting_model_identity(self):
     def test_qualification_rejects_model_different_from_frozen_selector(self):
         manifest = copy.deepcopy(self.manifest)
         manifest["arm"]["requested_model"] = "pinned-model-selector"
+        manifest["holdout_requests_sha256"] = self.holdout_request_digest(model="pinned-model-selector")
         manifest_sha = qualification.sha256_bytes(
             json.dumps(manifest).encode("utf-8")
         )
@@ -574,6 +589,7 @@ def test_runner_refuses_explicit_model_mismatch_before_result_append(self):
             (here / "dataset.json").write_bytes(self.dataset_bytes)
             manifest = copy.deepcopy(self.manifest)
             manifest["arm"]["requested_model"] = "pinned-model-selector"
+            manifest["holdout_requests_sha256"] = self.holdout_request_digest(model="pinned-model-selector")
             (here / "manifest.json").write_text(json.dumps(manifest))
             (here / "server.json").write_text(json.dumps(self.server_config))
             response = {
@@ -644,6 +660,7 @@ def test_qualified_resume_refuses_forged_model_before_network(self):
                 (here / "dataset.json").write_bytes(self.dataset_bytes)
                 manifest = copy.deepcopy(self.manifest)
                 manifest["arm"]["requested_model"] = "selected-model"
+                manifest["holdout_requests_sha256"] = self.holdout_request_digest(model="selected-model")
                 (here / "manifest.json").write_text(json.dumps(manifest))
                 (here / "server.json").write_text(json.dumps(self.server_config))
                 args = [
@@ -1271,6 +1288,47 @@ def test_qualification_reconstructs_measured_request_sizes(self):
             )
 
 
+    def test_frozen_rendered_requests_refuse_renderer_drift_before_inference(self):
+        # The renderer's name, source cases, and questions may all remain
+        # unchanged while the actual outbound model prompt changes.
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            with patch.object(run_http, "HERE", str(here)), patch.dict(
+                run_http.STATES, {"B_noul_text": lambda case: "changed renderer output"},
+            ), patch.object(run_http, "post") as inference:
+                with self.assertRaisesRegex(ValueError, "rendered held-out requests differ"):
+                    run_http.main([
+                        "--url", self.endpoint, "--tag", "renderer-drift",
+                        "--server-config", str(here / "server.json"),
+                        "--qualification-manifest", str(here / "manifest.json"),
+                        "B_noul_text",
+                    ])
+                inference.assert_not_called()
+            self.assertFalse((here / "results").exists())
+
+    def test_frozen_rendered_request_digest_refuses_rehashed_result(self):
+        for bad_digest in (None, 42, "f" * 64):
+            with self.subTest(frozen_digest=bad_digest):
+                changed = copy.deepcopy(self.manifest)
+                changed["holdout_requests_sha256"] = bad_digest
+                if bad_digest is None or bad_digest == 42:
+                    with self.assertRaisesRegex(ValueError, "frozen held-out request digest"):
+                        self.valid(changed)
+                    continue
+                changed_sha = qualification.sha256_json(changed)
+                forged = self.signed_result()
+                forged["experiment"]["qualification_manifest_sha256"] = changed_sha
+                forged["experiment_sha256"] = score.evidence_digest(forged["experiment"])
+                # Every result/experiment fingerprint still agrees internally.
+                # The independently frozen request body digest must also match.
+                with self.assertRaisesRegex(ValueError, "rendered held-out requests differ"):
+                    qualification.qualified_report(
+                        forged, changed, changed_sha, self.questions, self.dataset_bytes,
+                    )
+
     def test_frozen_question_order_refuses_a_second_holdout_arm_before_inference(self):
         # Sorted-key JSON digests collide across question reordering although
         # the runner sends dict entries in insertion order. That let the same
@@ -1347,6 +1405,7 @@ def test_frozen_ten_question_vocabulary_runs_and_scores_without_historical_categ
         manifest = copy.deepcopy(self.manifest)
         manifest["questions_sha256"] = qualification.sha256_ordered_questions(questions)
         manifest["thresholds"] = {key: 0.6 for key in questions}
+        manifest["holdout_requests_sha256"] = self.holdout_request_digest(questions=questions)
         manifest_sha = qualification.sha256_bytes(json.dumps(manifest).encode())
         with tempfile.TemporaryDirectory() as temp:
             here = Path(temp)

From eb09b12d5b6331aa678538cd5f0767c6a3418ee5 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 10:06:49 -0800
Subject: [PATCH 24/76] fix(evals): keep Scout HTTP evidence private on disk

---
 evals/intent_engine/HTTP_RUNNER.md   |  7 ++++++
 evals/intent_engine/run_http.py      | 21 +++++++++++++-----
 evals/intent_engine/test_run_http.py | 32 +++++++++++++++++++++++++---
 3 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index e693789aa8f..bb14ae89705 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -40,6 +40,13 @@ provenance, **not remote attestation**: update it whenever the deployment change
 The optional `--noul-temperature` is a separate client-side calibration step;
 it must be finite and strictly positive.
 
+New raw-response samples, resumable `.partial` logs and completed result
+files are created owner-readable/writable only (mode 0600 on POSIX), even
+under a permissive process umask. A safely locked, older permissive partial
+has its group/other permissions removed on resume without changing evidence
+bytes. Samples can contain the full model input, including private session
+content: do not put them on a shared filesystem with weaker access semantics.
+
 The existing `.partial` log starts with the experiment identity. This binds the
 whole dataset hash, selected case IDs and order, the original labels and
 context-needed annotations, the exact rendered HTTP request bytes for each
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index d8e29562692..5810775a3fb 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -50,10 +50,10 @@ def publish_sample(path, serialized):
     linking the completed file keeps the original exclusive-create behavior.
     """
     directory = os.path.dirname(os.path.abspath(path))
-    # mkstemp forces mode 0600. Match open(path, "x") by applying the
-    # process umask at creation without changing the process-wide umask.
+    # Samples include the raw request, potentially containing Scout session
+    # transcripts. The caller's umask must not make that evidence public.
     staged = os.path.join(directory, f".noul-sample-{secrets.token_hex(16)}")
-    descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666)
+    descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
     try:
         with os.fdopen(descriptor, "w", encoding="utf-8") as output:
             output.write(serialized)
@@ -111,7 +111,7 @@ def publish_final_result(path, document):
     serialized = json.dumps(document, indent=1, allow_nan=False)
     directory = os.path.dirname(os.path.abspath(path))
     staged = os.path.join(directory, f".noul-result-{secrets.token_hex(16)}")
-    descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666)
+    descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
     try:
         with os.fdopen(descriptor, "w", encoding="utf-8") as output:
             output.write(serialized)
@@ -376,6 +376,10 @@ def lock_partial_writer(partial, path):
     opened = os.fstat(partial.fileno())
     if (current.st_dev, current.st_ino) != (opened.st_dev, opened.st_ino):
         raise ValueError(f"partial output was replaced or redirected: {path}")
+    # Older interrupted logs may predate private evidence permissions.
+    # Tighten the opened, locked inode before reading or continuing inference.
+    if os.name == "posix" and stat.S_IMODE(opened.st_mode) & 0o077:
+        os.fchmod(partial.fileno(), stat.S_IMODE(opened.st_mode) & 0o700)
     partial.seek(0)
 
 
@@ -592,7 +596,14 @@ def main(argv=None):
         header = {"experiment": identity, "experiment_sha256": digest(identity)}
         partial_path = path + ".partial"
         try:
-            partial = open(partial_path, "x+", encoding="utf-8")
+            # Default umask 022 (or an explicit umask 000) must not expose
+            # result labels, cases or response telemetry to other users.
+            descriptor = os.open(partial_path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
+            try:
+                partial = os.fdopen(descriptor, "r+", encoding="utf-8")
+            except BaseException:
+                os.close(descriptor)
+                raise
             created = True
         except FileExistsError:
             partial = open_partial_for_resume(partial_path)
diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index 259df838e89..cddc83b108a 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -933,17 +933,43 @@ def test_dangling_sample_link_refuses_without_inference_or_partial_output(self):
         self.assertEqual(list(self.root.glob(".noul-sample-*")), [])
 
     @unittest.skipUnless(os.name == "posix", "POSIX umask semantics")
-    def test_published_sample_matches_normal_exclusive_creation_mode(self):
+    def test_private_samples_and_results_ignore_permissive_umask(self):
         sample = self.root / "first-response.json"
-        previous_umask = os.umask(0o022)
+        previous_umask = os.umask(0)
         try:
             with patch.object(run_http, "post", side_effect=self.response):
                 self.run_main(["--sample-out", str(sample)])
         finally:
             os.umask(previous_umask)
-        self.assertEqual(stat.S_IMODE(sample.stat().st_mode), 0o644)
+        self.assertEqual(stat.S_IMODE(sample.stat().st_mode), 0o600)
+        self.assertEqual(stat.S_IMODE(self.output.stat().st_mode), 0o600)
         self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2)
 
+    @unittest.skipUnless(os.name == "posix", "POSIX file permissions")
+    def test_private_partial_creation_and_tightening_on_resume(self):
+        previous_umask = os.umask(0)
+        try:
+            self.interrupt()
+            self.assertEqual(stat.S_IMODE(self.partial.stat().st_mode), 0o600)
+            # A partial written by the previous evaluator may have been
+            # group/world-readable. Preserve its bytes and resume safely.
+            frozen_bytes = self.partial.read_bytes()
+            os.chmod(self.partial, 0o666)
+
+            def assert_private_before_network(url, body):
+                self.assertEqual(stat.S_IMODE(self.partial.stat().st_mode), 0o600)
+                self.assertEqual(self.partial.read_bytes(), frozen_bytes)
+                return self.response(url, body)
+
+            with patch.object(run_http, "post", side_effect=assert_private_before_network) as post:
+                self.run_main()
+            post.assert_called_once()
+            self.assertFalse(self.partial.exists())
+            self.assertEqual(stat.S_IMODE(self.output.stat().st_mode), 0o600)
+            self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2)
+        finally:
+            os.umask(previous_umask)
+
     def test_calibration_remains_finite_at_extreme_positive_temperatures(self):
         for temperature in (1e-300, 1.0, 1e300):
             for probability in (0, 1e-300, 0.25, 0.5, 0.75, 1):

From 8af44e0f375dfeaa68328ac7242b1a4c40cf4a03 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 11:05:47 -0800
Subject: [PATCH 25/76] feat(evals): replay frozen Scout HTTP request bytes

Refs #4535

Partial evidence transport only. Keep #4797 DRAFT, never merge.
---
 evals/intent_engine/HTTP_RUNNER.md            |  51 ++++++
 evals/intent_engine/qualification.py          |  73 ++++++++-
 evals/intent_engine/run_http.py               |  60 ++++++--
 evals/intent_engine/score.py                  |  23 ++-
 evals/intent_engine/test_qualification.py     | 145 ++++++++++++++++++
 evals/intent_engine/test_qualification_cli.py |  48 ++++++
 6 files changed, 373 insertions(+), 27 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index bb14ae89705..09e6e858074 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -278,3 +278,54 @@ actual configured Scout questions, settings digest, window renderer or
 rehydrated transcript were used. Obtain that evidence from the production
 settings and owning transcript authority before making any application
 replacement decision. The runner itself does not export canonical Scout data.
+
+## Captured Scout request arm (qualified only)
+
+The historical `A/B/B2` renderers do **not** reproduce production Scout
+requests. A separately captured, byte-exact request file can now be evaluated
+without rewriting those requests in Python. This is a new arm,
+`S_scout_captured`, not a historical benchmark result.
+
+Use a JSON array containing **only the ordered held-out cases**, each with
+`{"id":"case-id","body":"<complete JSON HTTP request as a string>"}`.
+The inner `body` string must be the literal UTF-8 JSON that the runtime sent:
+`{"state": "...", "questions": {"intent": {"type": "noul",
+"instructions": "..."}}}` with optional `"model"` if the frozen model
+selector is non-null. Keep its exact byte encoding, escapes, whitespace and
+field order: the evaluator sends those bytes directly instead of generating
+new ones with the historical renderer.
+
+The manifest must set `arm.config` to `S_scout_captured`,
+`captured_requests_sha256` to the SHA-256 of the entire **original capture
+file**, and `holdout_requests_sha256` to the existing ordered
+`[[case_id, sha256(inner_body_bytes)], ...]` commitment. Other manifest
+identity, split, serving, calibration, token-budget, and tolerance constraints
+are unchanged. For example:
+
+```sh
+python evals/intent_engine/run_http.py \
+  --url https://private-model-host.example/v1/systemone \
+  --tag scout-capture-1 \
+  --server-config /path/to/serving.json \
+  --qualification-manifest /path/to/manifest.json \
+  --questions-file /path/to/frozen-questions.json \
+  --captured-requests /path/to/scout-requests.json \
+  S_scout_captured
+
+python evals/intent_engine/score.py --qualified \
+  /path/to/manifest.json /path/to/original-dataset.json \
+  evals/intent_engine/results/scout-capture-1_S_scout_captured.json \
+  --questions-file /path/to/frozen-questions.json \
+  --captured-requests /path/to/scout-requests.json
+```
+
+Both the runner and independent scorer refuse altered capture bytes, missing
+or reordered held-out cases, invalid Noul questions, differing model selectors,
+extra request keys, request-size violations, or a capture attached to an
+historical renderer. A capture is an **operator-supplied input**, not proof that
+its bytes came from Scout. The owning transcript's original store-id bounds,
+canonical window digest, production settings, real question instructions, and
+actual request serialization must still be independently verified by the
+missing Scout exporter. Do not claim production parity, real Kev inference,
+reviewed labels, or a replacement result merely because a captured-arm
+fixture passes tests.
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index e8af39b4f27..06cf8a469af 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -29,6 +29,39 @@ def reject_nonfinite(constant):
                       parse_constant=reject_nonfinite)
 
 
+CAPTURED_SCOUT_CONFIG = "S_scout_captured"
+
+
+def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes):
+    """Replay exact, operator-supplied Scout HTTP request bytes without rendering.
+
+    Captures are declarations until independently exported from a digest-checked
+    owning transcript window. This method does not attest their origin.
+    """
+    records = loads_unambiguous_json(raw)
+    if (not isinstance(records, list) or len(records) != len(case_ids)
+            or not all(isinstance(record, dict)
+                       and set(record) == {"id", "body"} for record in records)
+            or [record["id"] for record in records] != list(case_ids)):
+        raise ValueError("captured Scout requests must cover held-out cases in order")
+    expected_keys = {"state", "questions"} | ({"model"} if requested_model is not None else set())
+    bodies = {}
+    for record in records:
+        if not isinstance(record["body"], str):
+            raise ValueError(f"captured request {record['id']} must be JSON text")
+        encoded = record["body"].encode("utf-8")
+        if len(encoded) > max_bytes:
+            raise ValueError(f"captured request {record['id']} exceeds frozen request byte limit")
+        request = loads_unambiguous_json(encoded)
+        if (not isinstance(request, dict) or set(request) != expected_keys
+                or not isinstance(request["state"], str) or not request["state"].strip()
+                or request["questions"] != questions
+                or (requested_model is not None and request.get("model") != requested_model)):
+            raise ValueError(f"captured Scout request {record['id']} differs from frozen Noul contract")
+        bodies[record["id"]] = encoded
+    return bodies
+
+
 def load_questions(path):
     """Read a frozen Noul question vocabulary supplied by the settings owner.
 
@@ -119,6 +152,13 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
         if not isinstance(value, str) or not _DIGEST.fullmatch(value):
             raise ValueError(f"qualification HTTP arm has invalid {field}")
 
+    captured_sha = manifest.get("captured_requests_sha256")
+    if arm["config"] == CAPTURED_SCOUT_CONFIG:
+        if not isinstance(captured_sha, str) or not _DIGEST.fullmatch(captured_sha):
+            raise ValueError("qualification requires a frozen captured Scout request file")
+    elif captured_sha is not None:
+        raise ValueError("only captured Scout arms may declare captured request evidence")
+
     request_limit = manifest.get("max_request_bytes")
     if (isinstance(request_limit, bool) or not isinstance(request_limit, int)
             or request_limit <= 0):
@@ -237,6 +277,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
         "heldout_ids": heldout,
         "holdout_requests_sha256": frozen_requests,
         "arm": arm,
+        "captured_requests_sha256": captured_sha,
         "max_request_bytes": request_limit,
         "max_input_tokens": token_limit,
         "thresholds": thresholds,
@@ -273,7 +314,7 @@ def holdout_decision_metrics(rows, thresholds):
     }
 
 
-def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
+def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, captured_requests_bytes=None):
     """Reproduce qualified predictions from the frozen original dataset."""
     from score import score_config
     from run_http import (
@@ -331,16 +372,32 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes):
     source_truncation_cases = sum(has_numeric_truncation_receipt(row) for row in rows)
     # Reconstruct real HTTP request bytes: self-consistent metadata hashes
     # alone cannot prove the frozen prompts were used.
-    renderer = STATES.get(experiment.get("config"))
-    if renderer is None:
-        raise ValueError("qualification has an unsupported request renderer")
+    captured = None
+    if arm["config"] == CAPTURED_SCOUT_CONFIG:
+        if (not isinstance(captured_requests_bytes, bytes)
+                or sha256_bytes(captured_requests_bytes) != validated["captured_requests_sha256"]):
+            raise ValueError("qualification requires the original frozen Scout request capture")
+        captured = captured_request_bodies(
+            captured_requests_bytes, validated["heldout_ids"], questions,
+            arm["requested_model"], validated["max_request_bytes"],
+        )
+        renderer = None
+    else:
+        if captured_requests_bytes is not None:
+            raise ValueError("historical renderers cannot import Scout request captures")
+        renderer = STATES.get(experiment.get("config"))
+        if renderer is None:
+            raise ValueError("qualification has an unsupported request renderer")
     case_map = {case["id"]: case for case in cases}
     request_sizes = []
     for row in rows:
-        body = {"state": renderer(case_map[row["id"]]), "questions": questions}
-        if experiment.get("requested_model"):
-            body["model"] = experiment["requested_model"]
-        rendered = request_bytes(body)
+        if captured is not None:
+            rendered = captured[row["id"]]
+        else:
+            body = {"state": renderer(case_map[row["id"]]), "questions": questions}
+            if experiment.get("requested_model"):
+                body["model"] = experiment["requested_model"]
+            rendered = request_bytes(body)
         if row.get("request_sha256") != sha256_bytes(rendered):
             raise ValueError(
                 f"qualification request fingerprint disagrees with frozen renderer for {row['id']}"
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 5810775a3fb..16a1abe68eb 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -23,7 +23,10 @@
 HERE = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, HERE)
 from common import calibrated, noul_questions, prompt_only, text_window
-from qualification import load_manifest, load_questions, loads_unambiguous_json, validate_manifest
+from qualification import (
+    CAPTURED_SCOUT_CONFIG, captured_request_bodies, load_manifest, load_questions,
+    loads_unambiguous_json, validate_manifest,
+)
 
 STATES = {
     "A_noul_prompt": prompt_only,
@@ -39,7 +42,9 @@ def digest(value):
 
 
 def request_bytes(body):
-    # Preserve question and request-field order in both HTTP and resume evidence.
+    # Never reserialize a Rust-produced Scout request through Python's encoder.
+    if isinstance(body, bytes):
+        return body
     return json.dumps(body, allow_nan=False).encode("utf-8")
 
 
@@ -490,12 +495,13 @@ def main(argv=None):
     ap.add_argument("--dataset", help="original JSON cases; frozen digest recorded in each run")
     ap.add_argument("--qualification-manifest", help="freeze reviewed held-out labels, groups, thresholds and tolerances")
     ap.add_argument("--questions-file", help="declared typed Noul questions; qualified runs only, requires separate runtime provenance")
+    ap.add_argument("--captured-requests", help="frozen verbatim Scout HTTP requests, captured qualification arm only")
     ap.add_argument("--send-model")
     ap.add_argument("--bearer-token-env", metavar="NAME",
                     help="name of an environment variable holding a bearer token (never recorded)")
     ap.add_argument("--sample-out", help="write the first raw response here")
     ap.add_argument("--noul-temperature", type=float, help="client calibration temperature; raw probabilities are retained")
-    ap.add_argument("configs", nargs="+", choices=tuple(STATES))
+    ap.add_argument("configs", nargs="+", choices=(*STATES, CAPTURED_SCOUT_CONFIG))
     args = ap.parse_args(argv)
     if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.tag):
         raise ValueError("--tag must be a filename component starting with a letter or digit")
@@ -514,9 +520,14 @@ def main(argv=None):
     dataset_sha = hashlib.sha256(dataset_bytes).hexdigest()
     if args.questions_file and not args.qualification_manifest:
         raise ValueError("--questions-file requires --qualification-manifest")
+    if args.captured_requests and not args.qualification_manifest:
+        raise ValueError("--captured-requests requires --qualification-manifest")
+    if CAPTURED_SCOUT_CONFIG in args.configs and not args.qualification_manifest:
+        raise ValueError("captured Scout requests require frozen qualification")
     questions = load_questions(args.questions_file) if args.questions_file else noul_questions()
     case_ids = validate_dataset(dataset, questions)
     qualification_sha = None
+    captured = None
     if args.qualification_manifest:
         if args.subset:
             raise ValueError("qualification selects its own frozen holdout; --subset is not allowed")
@@ -530,6 +541,17 @@ def main(argv=None):
                 or hashlib.sha256(args.url.encode("utf-8")).hexdigest() != arm["endpoint_sha256"]
                 or digest(server_config) != arm["server_config_sha256"]):
             raise ValueError("qualification HTTP arm differs from frozen manifest")
+        if (arm["config"] == CAPTURED_SCOUT_CONFIG) != bool(args.captured_requests):
+            raise ValueError("captured Scout arms require --captured-requests and no historical renderer")
+        if args.captured_requests:
+            with open(args.captured_requests, "rb") as source:
+                capture_bytes = source.read()
+            if hashlib.sha256(capture_bytes).hexdigest() != qualified["captured_requests_sha256"]:
+                raise ValueError("captured Scout request evidence differs from frozen manifest")
+            captured = captured_request_bodies(
+                capture_bytes, qualified["heldout_ids"], questions,
+                args.send_model, qualified["max_request_bytes"],
+            )
         keep = set(qualified["heldout_ids"])
         dataset = [case for case in dataset if case["id"] in keep]
     if args.subset:
@@ -547,13 +569,16 @@ def main(argv=None):
     if qualification_sha is not None:
         # Bound and verify precommitted request bodies before opening any
         # output. A stable config name does not imply a stable renderer.
-        renderer = STATES[qualified["arm"]["config"]]
+        renderer = STATES.get(qualified["arm"]["config"])
         rendered_requests = []
         for case in dataset:
-            body = {"state": renderer(case), "questions": questions}
-            if args.send_model:
-                body["model"] = args.send_model
-            encoded = request_bytes(body)
+            if captured is not None:
+                encoded = captured[case["id"]]
+            else:
+                body = {"state": renderer(case), "questions": questions}
+                if args.send_model:
+                    body["model"] = args.send_model
+                encoded = request_bytes(body)
             if len(encoded) > qualified["max_request_bytes"]:
                 raise ValueError(f"qualification case {case['id']} exceeds frozen request byte limit")
             rendered_requests.append([case["id"], hashlib.sha256(encoded).hexdigest()])
@@ -570,9 +595,12 @@ def main(argv=None):
             require_exclusive_publication(os.path.dirname(os.path.abspath(args.sample_out)))
         requests = {}
         for case in dataset:
-            body = {"state": STATES[cfg](case), "questions": questions}
-            if args.send_model:
-                body["model"] = args.send_model
+            if captured is not None:
+                body = captured[case["id"]]
+            else:
+                body = {"state": STATES[cfg](case), "questions": questions}
+                if args.send_model:
+                    body["model"] = args.send_model
             requests[case["id"]] = (body, hashlib.sha256(request_bytes(body)).hexdigest())
         identity = {
             "version": 2,
@@ -677,9 +705,13 @@ def main(argv=None):
                 # that a later valid retry mistakes for a completed capture.
                 serialized_row = serialize_result(row, questions, args.noul_temperature)
                 if args.sample_out and not os.path.exists(args.sample_out):
-                    serialized_sample = json.dumps(
-                        {"request": body, "response": response}, indent=1, allow_nan=False
-                    )
+                    sample = {
+                        "request": loads_unambiguous_json(body) if isinstance(body, bytes) else body,
+                        "response": response,
+                    }
+                    if isinstance(body, bytes):
+                        sample["request_json"] = body.decode("utf-8")
+                    serialized_sample = json.dumps(sample, indent=1, allow_nan=False)
                     publish_sample(args.sample_out, serialized_sample)
                 partial.write(serialized_row + "\n")
                 partial.flush()
diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py
index 06c112fe1ff..2bf0ca12201 100644
--- a/evals/intent_engine/score.py
+++ b/evals/intent_engine/score.py
@@ -307,9 +307,16 @@ def ch(rs):
 if __name__ == "__main__":
     here = os.path.dirname(os.path.abspath(__file__))
     if len(sys.argv) > 1:
-        if (sys.argv[1] != "--qualified" or len(sys.argv) not in (5, 7)
-                or (len(sys.argv) == 7 and sys.argv[5] != "--questions-file")):
-            raise ValueError("usage: score.py [--qualified MANIFEST DATASET RESULT [--questions-file FILE]]")
+        usage = ("usage: score.py [--qualified MANIFEST DATASET RESULT "
+                 "[--questions-file FILE] [--captured-requests FILE]]")
+        if sys.argv[1] != "--qualified" or len(sys.argv) < 5 or len(sys.argv[5:]) % 2:
+            raise ValueError(usage)
+        options = {}
+        for index in range(5, len(sys.argv), 2):
+            flag = sys.argv[index]
+            if flag not in ("--questions-file", "--captured-requests") or flag in options:
+                raise ValueError(usage)
+            options[flag] = sys.argv[index + 1]
         from qualification import load_manifest, load_questions, loads_unambiguous_json, qualified_report
         from common import noul_questions
         manifest, manifest_sha = load_manifest(sys.argv[2])
@@ -317,8 +324,14 @@ def ch(rs):
             dataset_bytes = source.read()
         with open(sys.argv[4], encoding="utf-8") as source:
             result = loads_unambiguous_json(source.read())
-        questions = load_questions(sys.argv[6]) if len(sys.argv) == 7 else noul_questions()
-        report = qualified_report(result, manifest, manifest_sha, questions, dataset_bytes)
+        questions = load_questions(options["--questions-file"]) if "--questions-file" in options else noul_questions()
+        captured_requests_bytes = None
+        if "--captured-requests" in options:
+            with open(options["--captured-requests"], "rb") as source:
+                captured_requests_bytes = source.read()
+        report = qualified_report(
+            result, manifest, manifest_sha, questions, dataset_bytes, captured_requests_bytes,
+        )
         publish_summary(os.path.join(here, "qualification_summary.json"), report)
         print(json.dumps(report, indent=2, allow_nan=False))
         # A completed evaluation is not necessarily a qualified model. Keep
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index ea37e29366a..98ddb7fedb7 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1481,5 +1481,150 @@ def test_qualified_question_files_refuse_non_noul_or_ambiguous_vocabulary(self):
                     with self.assertRaisesRegex(ValueError, expected):
                         qualification.load_questions(path)
 
+    def captured_scout_fixture(self):
+        # The fixture deliberately uses Rust-style compact JSON and a request
+        # field order different from the historical Python renderer.
+        records = []
+        for case in self.dataset[2:]:
+            body = {"questions": self.questions, "state": "User: " + case["current_prompt"]}
+            records.append({
+                "id": case["id"],
+                "body": json.dumps(body, ensure_ascii=False, separators=(",", ":")),
+            })
+        capture_bytes = json.dumps(records, ensure_ascii=False).encode("utf-8")
+        manifest = copy.deepcopy(self.manifest)
+        manifest["arm"]["config"] = qualification.CAPTURED_SCOUT_CONFIG
+        manifest["captured_requests_sha256"] = qualification.sha256_bytes(capture_bytes)
+        manifest["holdout_requests_sha256"] = score.evidence_digest([
+            [record["id"], hashlib.sha256(record["body"].encode("utf-8")).hexdigest()]
+            for record in records
+        ])
+        return capture_bytes, manifest
+
+    def test_captured_scout_requests_replay_exact_bytes_and_score(self):
+        # This proves a captured arm is byte-exact, not that the synthetic
+        # fixture came from real Scout or that its labels were reviewed.
+        capture_bytes, manifest = self.captured_scout_fixture()
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_text(json.dumps(manifest))
+            (here / "captured.json").write_bytes(capture_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            transmitted = []
+
+            def inference(_url, body):
+                self.assertIsInstance(body, bytes, "must not reserialize the Rust request")
+                transmitted.append(body)
+                request = qualification.loads_unambiguous_json(body)
+                probability = 0.9 if "What changes?" in request["state"] else 0.1
+                return ({
+                    "model": "reported-scout-fixture",
+                    "answers": {
+                        key: {"noul": probability if key == "impact" else 0.1}
+                        for key in self.questions
+                    },
+                    "usage": {"input_tokens": 42, "state_tokens_dropped": 0},
+                }, 5.0)
+
+            args = [
+                "--url", self.endpoint, "--tag", "captured",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "--captured-requests", str(here / "captured.json"),
+                qualification.CAPTURED_SCOUT_CONFIG,
+            ]
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post", side_effect=inference) as post:
+                    run_http.main(args)
+                self.assertEqual(post.call_count, 2, "only frozen heldout cases may be inferred")
+                expected = [
+                    record["body"].encode("utf-8")
+                    for record in qualification.loads_unambiguous_json(capture_bytes)
+                ]
+                self.assertEqual(transmitted, expected)
+                result = json.loads((here / "results" /
+                                    "captured_S_scout_captured.json").read_text())
+                report = qualification.qualified_report(
+                    result, manifest, qualification.sha256_bytes(
+                        (here / "manifest.json").read_bytes()),
+                    self.questions, self.dataset_bytes, capture_bytes,
+                )
+                self.assertTrue(report["qualified_against_frozen_tolerances"])
+                self.assertEqual(report["cases"], 2)
+                self.assertEqual([row["request_size_bytes"] for row in result["rows"]],
+                                 [len(body) for body in expected])
+                with self.assertRaisesRegex(ValueError, "original frozen Scout request capture"):
+                    qualification.qualified_report(
+                        result, manifest, qualification.sha256_bytes(
+                            (here / "manifest.json").read_bytes()),
+                        self.questions, self.dataset_bytes,
+                    )
+                altered = capture_bytes.replace(b"What changes?", b"What changed?")
+                with self.assertRaisesRegex(ValueError, "original frozen Scout request capture"):
+                    qualification.qualified_report(
+                        result, manifest, qualification.sha256_bytes(
+                            (here / "manifest.json").read_bytes()),
+                        self.questions, self.dataset_bytes, altered,
+                    )
+
+    def test_capture_drift_and_invalid_contract_refuse_before_network(self):
+        capture_bytes, manifest = self.captured_scout_fixture()
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_text(json.dumps(manifest))
+            (here / "captured.json").write_bytes(capture_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            args = [
+                "--url", self.endpoint, "--tag", "capture-refusal",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "--captured-requests", str(here / "captured.json"),
+                qualification.CAPTURED_SCOUT_CONFIG,
+            ]
+            records = qualification.loads_unambiguous_json(capture_bytes)
+            invalid = []
+            reordered = copy.deepcopy(records)
+            reordered.reverse()
+            invalid.append(("ordering", reordered))
+            injected = copy.deepcopy(records)
+            injected[0]["body"] = injected[0]["body"].replace(
+                '"type":"noul"', '"type":"choice"', 1
+            )
+            invalid.append(("question contract", injected))
+            extra = copy.deepcopy(records)
+            extra[0]["body"] = extra[0]["body"].replace(
+                '{"questions":', '{"debug":"injected","questions":', 1
+            )
+            invalid.append(("extra request field", extra))
+            for name, changed in invalid:
+                with self.subTest(name=name):
+                    changed_bytes = json.dumps(changed, ensure_ascii=False).encode("utf-8")
+                    with self.assertRaises(ValueError):
+                        qualification.captured_request_bodies(
+                            changed_bytes, ["holdout-positive", "holdout-negative"],
+                            self.questions, None, manifest["max_request_bytes"],
+                        )
+            for name, changed in [
+                ("changed text", capture_bytes.replace(b"What changes?", b"What changed?")),
+                ("unexpected case", json.dumps(records[:-1]).encode("utf-8")),
+            ]:
+                with self.subTest(name=name):
+                    (here / "captured.json").write_bytes(changed)
+                    with patch.object(run_http, "HERE", str(here)):
+                        with patch.object(run_http, "post") as post:
+                            with self.assertRaises(ValueError):
+                                run_http.main(args)
+                            post.assert_not_called()
+                    self.assertFalse((here / "results" /
+                                      "capture-refusal_S_scout_captured.json.partial").exists())
+            (here / "captured.json").write_bytes(capture_bytes)
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post") as post:
+                    with self.assertRaisesRegex(ValueError, "--captured-requests"):
+                        run_http.main(args[:-3] + [qualification.CAPTURED_SCOUT_CONFIG])
+                    post.assert_not_called()
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/evals/intent_engine/test_qualification_cli.py b/evals/intent_engine/test_qualification_cli.py
index d2262ceea4f..b964d03f8f6 100644
--- a/evals/intent_engine/test_qualification_cli.py
+++ b/evals/intent_engine/test_qualification_cli.py
@@ -47,6 +47,54 @@ def published_report(self, process):
         self.assertFalse((self.root / "summary.json").exists())
         return report
 
+    def test_captured_arm_cli_requires_original_request_bytes(self):
+        # Synthetic captured requests demonstrate byte-identity enforcement,
+        # not production Scout provenance or real Kev qualification.
+        capture_bytes, manifest = self.fixture.captured_scout_fixture()
+        manifest_bytes = json.dumps(manifest).encode("utf-8")
+        (self.root / "manifest.json").write_bytes(manifest_bytes)
+        (self.root / "captured.json").write_bytes(capture_bytes)
+        records = fixtures.qualification.loads_unambiguous_json(capture_bytes)
+        result = self.fixture.signed_result()
+        for row, record in zip(result["rows"], records, strict=True):
+            raw = record["body"].encode("utf-8")
+            row["request_sha256"] = fixtures.qualification.sha256_bytes(raw)
+            row["request_size_bytes"] = len(raw)
+        experiment = result["experiment"]
+        experiment["config"] = fixtures.qualification.CAPTURED_SCOUT_CONFIG
+        experiment["requests_sha256"] = fixtures.score.evidence_digest([
+            [row["id"], row["request_sha256"]] for row in result["rows"]
+        ])
+        experiment["qualification_manifest_sha256"] = (
+            fixtures.qualification.sha256_bytes(manifest_bytes)
+        )
+        result["config"] = "trial_S_scout_captured"
+        result["experiment_sha256"] = fixtures.score.evidence_digest(experiment)
+        (self.root / "result.json").write_text(json.dumps(result), encoding="utf-8")
+        argv = [
+            sys.executable, str(self.root / "score.py"), "--qualified",
+            str(self.root / "manifest.json"), str(self.root / "dataset.json"),
+            str(self.root / "result.json"),
+        ]
+        missing = subprocess.run(argv, cwd=self.root, capture_output=True, text=True, timeout=30)
+        self.assertNotEqual(missing.returncode, 0)
+        self.assertIn("original frozen Scout request capture", missing.stderr)
+        self.assertFalse((self.root / "qualification_summary.json").exists())
+
+        complete = argv + ["--captured-requests", str(self.root / "captured.json")]
+        passing = subprocess.run(complete, cwd=self.root, capture_output=True, text=True, timeout=30)
+        self.assertEqual(passing.returncode, 0, passing.stderr)
+        report = self.published_report(passing)
+        self.assertTrue(report["qualified_against_frozen_tolerances"])
+        frozen_report = (self.root / "qualification_summary.json").read_bytes()
+        (self.root / "captured.json").write_bytes(
+            capture_bytes.replace(b"What changes?", b"What changed?")
+        )
+        rejected = subprocess.run(complete, cwd=self.root, capture_output=True, text=True, timeout=30)
+        self.assertNotEqual(rejected.returncode, 0)
+        self.assertIn("original frozen Scout request capture", rejected.stderr)
+        self.assertEqual((self.root / "qualification_summary.json").read_bytes(), frozen_report)
+
     def test_duplicate_result_keys_cannot_publish_a_passing_report(self):
         valid = self.fixture.signed_result()
         raw = json.dumps(valid, allow_nan=False)

From 331fc44dd465809b66d395f99d59739633d555fc Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 12:03:57 -0800
Subject: [PATCH 26/76] fix(evals): validate resumed and frozen qualification
 evidence

---
 evals/intent_engine/qualification.py      |  5 ++++-
 evals/intent_engine/run_http.py           | 13 ++++++-------
 evals/intent_engine/test_qualification.py | 13 ++++++++++---
 3 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 06cf8a469af..71a932d868d 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -325,6 +325,8 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
     if not isinstance(dataset_bytes, bytes):
         raise ValueError("qualification requires the original dataset bytes")
     dataset_sha = sha256_bytes(dataset_bytes)
+    if not isinstance(result, dict):
+        raise ValueError("qualification result must be an object")
     if result.get("dataset_sha256") != dataset_sha:
         raise ValueError("qualification result dataset digest differs from supplied evidence")
     cases = loads_unambiguous_json(dataset_bytes)
@@ -355,7 +357,8 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
     if experiment.get("requests_sha256") != validated["holdout_requests_sha256"]:
         raise ValueError("qualification rendered held-out requests differ from frozen manifest")
     rows = result.get("rows")
-    if not isinstance(rows, list) or [r.get("id") for r in rows if isinstance(r, dict)] != validated["heldout_ids"]:
+    if (not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows)
+            or [row.get("id") for row in rows] != validated["heldout_ids"]):
         raise ValueError("qualification result differs from frozen held-out case order")
     for row in rows:
         annotation = validated["annotations"][row["id"]]
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 16a1abe68eb..7ac026c874f 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -316,9 +316,9 @@ def checked_qualified_model(previous, reported, requested):
         raise ValueError("qualified HTTP response requires a server-reported model")
     if requested is not None and reported != requested:
         raise ValueError("qualified HTTP reported model differs from frozen selector")
-    if previous is not None and reported is not None and reported != previous:
+    if previous is not None and reported != previous:
         raise ValueError("qualified HTTP responses mix reported models")
-    return previous if reported is None else reported
+    return reported
 
 
 def serialize_result(row, questions, temperature):
@@ -439,16 +439,15 @@ def resume_rows(partial, header, cases, requests, questions, temperature):
                 or row.get("labels") != case["labels"]
                 or row.get("context_needed") != case["context_needed"]):
             raise ValueError(f"partial output has mismatched case evidence for {case_id}")
-        if ("qualification_manifest_sha256" in header["experiment"]
-                and (isinstance(row.get("request_size_bytes"), bool)
-                     or row.get("request_size_bytes") != len(request_bytes(requests[case_id][0])))):
-            raise ValueError(f"partial output has mismatched request byte count for {case_id}")
-        serialize_result(row, questions, temperature)
         if qualified:
+            recorded_size = row.get("request_size_bytes")
+            if type(recorded_size) is not int or recorded_size != len(request_bytes(requests[case_id][0])):
+                raise ValueError(f"partial output has mismatched request byte count for {case_id}")
             previous_model = checked_qualified_model(
                 previous_model, row.get("observed_model"),
                 header["experiment"].get("requested_model"),
             )
+        serialize_result(row, questions, temperature)
         done[case_id] = row
     return done
 
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 98ddb7fedb7..aea7c823d8c 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -692,7 +692,8 @@ def test_qualified_resume_refuses_forged_model_before_network(self):
                     partial.write_text(header + "\n" + json.dumps(corrupted) + "\n")
                     before = partial.read_bytes()
                     with patch.object(run_http, "post") as blocked:
-                        with self.assertRaisesRegex(ValueError, "frozen selector"):
+                        expected = "requires a server-reported model" if altered_model is None else "frozen selector"
+                        with self.assertRaisesRegex(ValueError, expected):
                             run_http.main(args)
                         blocked.assert_not_called()
                     self.assertEqual(partial.read_bytes(), before)
@@ -823,6 +824,7 @@ def test_qualified_reporting_refuses_tampered_labels_and_unbound_results(self):
         for field, mutation, pattern in (
             ("label", lambda r: r["rows"][0].update(labels=[]), "reviewed labels"),
             ("order", lambda r: r["rows"].reverse(), "held-out case order"),
+            ("non_object_row", lambda r: r["rows"].append(None), "held-out case order"),
             ("identity", lambda r: r["experiment"].pop("qualification_manifest_sha256"), "not bound"),
             ("forged_review", lambda r: r["rows"][1].update(context_needed=True), "reviewed labels"),
         ):
@@ -1080,11 +1082,16 @@ def test_qualified_requires_original_cases_and_reproducible_requests(self):
         altered["experiment"]["requests_sha256"] = score.evidence_digest([
             [row["id"], row["request_sha256"]] for row in altered["rows"]
         ])
+        # Keep the declaration and the result internally consistent so this
+        # exercises reconstruction against the original requests.
+        manifest = copy.deepcopy(self.manifest)
+        manifest["holdout_requests_sha256"] = altered["experiment"]["requests_sha256"]
+        manifest_sha = qualification.sha256_bytes(json.dumps(manifest).encode("utf-8"))
+        altered["experiment"]["qualification_manifest_sha256"] = manifest_sha
         altered["experiment_sha256"] = score.evidence_digest(altered["experiment"])
         with self.assertRaisesRegex(ValueError, "request fingerprint disagrees"):
             qualification.qualified_report(
-                altered, self.manifest, self.manifest_sha,
-                self.questions, self.dataset_bytes,
+                altered, manifest, manifest_sha, self.questions, self.dataset_bytes,
             )
 
     def test_runner_can_consume_explicit_frozen_input_file(self):

From 51cad35613a6dc9286a8bc6bd28ba073325930f4 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 12:42:52 -0800
Subject: [PATCH 27/76] fix(evals): preserve frozen question order in Scout
 replay

Refs #4535

Reject captured requests whose model-visible question key ordering differs
from the frozen Noul vocabulary, even with internally rehashed evidence.
Add zero-network runner refusal and independent scorer regressions.
---
 evals/intent_engine/HTTP_RUNNER.md        |  3 +-
 evals/intent_engine/qualification.py      |  3 ++
 evals/intent_engine/test_qualification.py | 66 +++++++++++++++++++++++
 3 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 09e6e858074..9dbe2457cee 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -320,7 +320,8 @@ python evals/intent_engine/score.py --qualified \
 ```
 
 Both the runner and independent scorer refuse altered capture bytes, missing
-or reordered held-out cases, invalid Noul questions, differing model selectors,
+or reordered held-out cases, invalid Noul questions, question key ordering that
+differs from the frozen vocabulary (even when map values match), differing model selectors,
 extra request keys, request-size violations, or a capture attached to an
 historical renderer. A capture is an **operator-supplied input**, not proof that
 its bytes came from Scout. The owning transcript's original store-id bounds,
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 71a932d868d..2691a8f5735 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -56,6 +56,9 @@ def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes
         if (not isinstance(request, dict) or set(request) != expected_keys
                 or not isinstance(request["state"], str) or not request["state"].strip()
                 or request["questions"] != questions
+                # Object equality ignores insertion order. The model sees the
+                # actual wire order, so it must match the frozen vocabulary.
+                or list(request["questions"]) != list(questions)
                 or (requested_model is not None and request.get("model") != requested_model)):
             raise ValueError(f"captured Scout request {record['id']} differs from frozen Noul contract")
         bodies[record["id"]] = encoded
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index aea7c823d8c..bc891ece918 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1633,5 +1633,71 @@ def test_capture_drift_and_invalid_contract_refuse_before_network(self):
                         run_http.main(args[:-3] + [qualification.CAPTURED_SCOUT_CONFIG])
                     post.assert_not_called()
 
+    def test_captured_question_order_cannot_be_changed_even_when_all_hashes_are_refrozen(self):
+        # Dict equality ignores insertion order, but the model receives ordered
+        # question bytes. Rehash every capture/manifest/result identity to
+        # isolate the missing order check rather than a digest mismatch.
+        capture, manifest = self.captured_scout_fixture()
+        records = qualification.loads_unambiguous_json(capture)
+        for record in records:
+            body = qualification.loads_unambiguous_json(record["body"])
+            body["questions"] = dict(reversed(list(body["questions"].items())))
+            self.assertEqual(body["questions"], self.questions)
+            self.assertNotEqual(list(body["questions"]), list(self.questions))
+            record["body"] = json.dumps(body, ensure_ascii=False, separators=(",", ":"))
+        reordered = json.dumps(records, ensure_ascii=False).encode("utf-8")
+        manifest["captured_requests_sha256"] = qualification.sha256_bytes(reordered)
+        manifest["holdout_requests_sha256"] = score.evidence_digest([
+            [record["id"], qualification.sha256_bytes(record["body"].encode("utf-8"))]
+            for record in records
+        ])
+        self.valid(manifest)
+        with self.assertRaisesRegex(ValueError, "question order"):
+            qualification.captured_request_bodies(
+                reordered, ["holdout-positive", "holdout-negative"],
+                self.questions, None, manifest["max_request_bytes"],
+            )
+
+        # Runner refusal precedes network traffic and evidence-file creation.
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            manifest_bytes = json.dumps(manifest).encode("utf-8")
+            (here / "manifest.json").write_bytes(manifest_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            (here / "captured.json").write_bytes(reordered)
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post") as post:
+                    with self.assertRaisesRegex(ValueError, "question order"):
+                        run_http.main([
+                            "--url", self.endpoint, "--tag", "reordered-capture",
+                            "--server-config", str(here / "server.json"),
+                            "--qualification-manifest", str(here / "manifest.json"),
+                            "--captured-requests", str(here / "captured.json"),
+                            qualification.CAPTURED_SCOUT_CONFIG,
+                        ])
+                    post.assert_not_called()
+            self.assertFalse((here / "results").exists())
+
+        # An independently supplied result with matching rewritten request
+        # fingerprints and experiment SHA must be refused by the scorer too.
+        result = self.signed_result()
+        for row, record in zip(result["rows"], records, strict=True):
+            body_bytes = record["body"].encode("utf-8")
+            row["request_sha256"] = qualification.sha256_bytes(body_bytes)
+            row["request_size_bytes"] = len(body_bytes)
+        result["config"] = "trial_" + qualification.CAPTURED_SCOUT_CONFIG
+        experiment = result["experiment"]
+        experiment["config"] = qualification.CAPTURED_SCOUT_CONFIG
+        experiment["requests_sha256"] = manifest["holdout_requests_sha256"]
+        experiment["qualification_manifest_sha256"] = qualification.sha256_bytes(manifest_bytes)
+        result["experiment_sha256"] = score.evidence_digest(experiment)
+        with self.assertRaisesRegex(ValueError, "question order"):
+            qualification.qualified_report(
+                result, manifest, qualification.sha256_bytes(manifest_bytes),
+                self.questions, self.dataset_bytes, reordered,
+            )
+
+
 if __name__ == "__main__":
     unittest.main()

From 6ea4efe469e446567edc1a93d2501033b4c2dd90 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 12:43:43 -0800
Subject: [PATCH 28/76] fix(evals): report captured question order drift
 precisely

Refs #4535

Keep contract-shape diagnostics separate from the model-visible
key-order refusal so the new regression asserts the intended failure.
---
 evals/intent_engine/qualification.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 2691a8f5735..3a65d0919f0 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -56,11 +56,12 @@ def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes
         if (not isinstance(request, dict) or set(request) != expected_keys
                 or not isinstance(request["state"], str) or not request["state"].strip()
                 or request["questions"] != questions
-                # Object equality ignores insertion order. The model sees the
-                # actual wire order, so it must match the frozen vocabulary.
-                or list(request["questions"]) != list(questions)
                 or (requested_model is not None and request.get("model") != requested_model)):
             raise ValueError(f"captured Scout request {record['id']} differs from frozen Noul contract")
+        # Dict equality ignores insertion order, but the model receives
+        # ordered wire bytes. The declared vocabulary freezes this order.
+        if list(request["questions"]) != list(questions):
+            raise ValueError(f"captured Scout request {record['id']} question order differs from frozen vocabulary")
         bodies[record["id"]] = encoded
     return bodies
 

From a35cfe60e3068feaf323fefeef0813e07d0e9960 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 12:52:57 -0800
Subject: [PATCH 29/76] fix(evals): bound remote decision response memory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Refs #4535

Cap typed HTTP response bytes before JSON parsing or sample publication.
Regression covers exact-limit success, oversized refusal with bounded reads,
and resume after rejection without publishing invalid evidence.

DRAFT / NO MERGE — qualification remains incomplete.
---
 evals/intent_engine/run_http.py      |  9 +++++-
 evals/intent_engine/test_run_http.py | 44 ++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 7ac026c874f..99e2d9e9693 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -28,6 +28,10 @@
     loads_unambiguous_json, validate_manifest,
 )
 
+# A decision response contains only a bounded set of typed answers and usage.
+# Refuse unexpectedly large remote bodies before materializing or parsing them.
+MAX_HTTP_RESPONSE_BYTES = 1024 * 1024
+
 STATES = {
     "A_noul_prompt": prompt_only,
     "B_noul_text": text_window,
@@ -176,7 +180,10 @@ def post(url, body, timeout=1800, bearer_token=None):
     )
     started = time.perf_counter()
     with opener.open(req, timeout=timeout) as response:
-        out = loads_unambiguous_json(response.read())
+        raw = response.read(MAX_HTTP_RESPONSE_BYTES + 1)
+        if len(raw) > MAX_HTTP_RESPONSE_BYTES:
+            raise ValueError("decision HTTP response exceeds bounded byte limit")
+        out = loads_unambiguous_json(raw)
     return out, (time.perf_counter() - started) * 1000
 
 
diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index cddc83b108a..a55bfe3f978 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -92,6 +92,50 @@ def test_ambiguous_http_json_is_rejected_before_response_admission(self):
                         run_http.post("http://localhost:8009/v1/systemone",
                                       {"state": "test", "questions": {}})
 
+    def test_http_response_limit_reads_only_bounded_bytes_and_accepts_boundary(self):
+        body = b'{"answers":{}}'
+        max_bytes = run_http.MAX_HTTP_RESPONSE_BYTES
+        for size, accepted in ((max_bytes, True), (max_bytes + 1, False)):
+            with self.subTest(size=size):
+                payload = body + b" " * (size - len(body))
+                with patch("urllib.request.build_opener") as opener:
+                    response = opener.return_value.open.return_value
+                    response.read.side_effect = io.BytesIO(payload).read
+                    if accepted:
+                        result, latency = run_http.post(
+                            "http://localhost:8009/v1/systemone",
+                            {"state": "test", "questions": {}},
+                        )
+                        self.assertEqual(result, {"answers": {}})
+                        self.assertGreaterEqual(latency, 0)
+                    else:
+                        with self.assertRaisesRegex(ValueError, "bounded byte limit"):
+                            run_http.post(
+                                "http://localhost:8009/v1/systemone",
+                                {"state": "test", "questions": {}},
+                            )
+                    response.read.assert_called_once_with(max_bytes + 1)
+
+    def test_oversized_response_preserves_resumable_evidence_without_sample(self):
+        sample = self.root / "sample.json"
+        payload = b" " * (run_http.MAX_HTTP_RESPONSE_BYTES + 1)
+        with patch("urllib.request.build_opener") as opener:
+            opener.return_value.open.return_value = io.BytesIO(payload)
+            with self.assertRaisesRegex(ValueError, "bounded byte limit"):
+                self.run_main(("--sample-out", str(sample)))
+        self.assertFalse(sample.exists())
+        self.assertFalse(self.output.exists())
+        self.assertTrue(self.partial.exists())
+        self.assertEqual(len(self.partial.read_text().splitlines()), 1)
+        initial = self.partial.read_bytes()
+        with patch.object(run_http, "post", side_effect=self.response) as post:
+            self.run_main(("--sample-out", str(sample)))
+        self.assertEqual(post.call_count, 2)
+        self.assertTrue(sample.exists())
+        self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2)
+        self.assertFalse(self.partial.exists())
+        self.assertTrue(initial.endswith(b"\\n"))
+
     def test_ambiguous_partial_row_refuses_without_changing_resumable_evidence(self):
         self.interrupt()
         original = self.partial.read_bytes()

From ff0838b37a4294c76eb268e0a39b17cfdba5c6f1 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 12:53:52 -0800
Subject: [PATCH 30/76] test(evals): assert the actual partial header newline

Refs #4535

Correct the byte-literal escape in the new oversized-response
recovery regression. Preserve a non-vacuous newline assertion.

DRAFT / NO MERGE.
---
 evals/intent_engine/test_run_http.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index a55bfe3f978..4a0e694bfa8 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -134,7 +134,7 @@ def test_oversized_response_preserves_resumable_evidence_without_sample(self):
         self.assertTrue(sample.exists())
         self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2)
         self.assertFalse(self.partial.exists())
-        self.assertTrue(initial.endswith(b"\\n"))
+        self.assertTrue(initial.endswith(b"\n"))
 
     def test_ambiguous_partial_row_refuses_without_changing_resumable_evidence(self):
         self.interrupt()

From 7923a3adcc699509205c5bc6128771911229b395 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 12:54:16 -0800
Subject: [PATCH 31/76] test(evals): assert bounded read on entered HTTP stream

Refs #4535

The HTTP test must stub and inspect the context manager's entered
response, not the unentered mock. Keep the max-size failure falsifiable.

DRAFT / NO MERGE.
---
 evals/intent_engine/test_run_http.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index 4a0e694bfa8..4f22af95b8b 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -99,7 +99,9 @@ def test_http_response_limit_reads_only_bounded_bytes_and_accepts_boundary(self)
             with self.subTest(size=size):
                 payload = body + b" " * (size - len(body))
                 with patch("urllib.request.build_opener") as opener:
-                    response = opener.return_value.open.return_value
+                    # The HTTP body is read from the context manager's
+                    # entered stream, not the mock returned by opener.open().
+                    response = opener.return_value.open.return_value.__enter__.return_value
                     response.read.side_effect = io.BytesIO(payload).read
                     if accepted:
                         result, latency = run_http.post(

From 12445c6bcda0faed3c19d40e8b4d1a3f49e14610 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 13:55:00 -0800
Subject: [PATCH 32/76] fix(evals): match captured Scout question wire order

Refs #4535

Compare a captured request's question-map order against the canonical
serde_json map serialization, not settings Vec iteration order. Keep
original capture bytes and manifest digests exact. Add positive and
negative regression coverage for the distinction.
---
 evals/intent_engine/HTTP_RUNNER.md        | 13 +++++++++---
 evals/intent_engine/qualification.py      | 10 +++++----
 evals/intent_engine/test_qualification.py | 25 ++++++++++++++++++-----
 3 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 9dbe2457cee..5d037eb039d 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -293,7 +293,14 @@ The inner `body` string must be the literal UTF-8 JSON that the runtime sent:
 "instructions": "..."}}}` with optional `"model"` if the frozen model
 selector is non-null. Keep its exact byte encoding, escapes, whitespace and
 field order: the evaluator sends those bytes directly instead of generating
-new ones with the historical renderer.
+new ones with the historical renderer. In the pinned Bazel Rust runtime,
+`serde_json::Map` serializes question object keys in lexical order even when
+the effective settings' `questions` vector is configured in a different order.
+For this arm, `questions_sha256` binds the declared settings vocabulary and
+its order, while `captured_requests_sha256` and `holdout_requests_sha256`
+bind the actual ordered **wire bytes**. The capture validator requires the
+canonical sorted question-map wire order, not a match to the settings vector's
+iteration order.
 
 The manifest must set `arm.config` to `S_scout_captured`,
 `captured_requests_sha256` to the SHA-256 of the entire **original capture
@@ -320,8 +327,8 @@ python evals/intent_engine/score.py --qualified \
 ```
 
 Both the runner and independent scorer refuse altered capture bytes, missing
-or reordered held-out cases, invalid Noul questions, question key ordering that
-differs from the frozen vocabulary (even when map values match), differing model selectors,
+or reordered held-out cases, invalid Noul questions, noncanonical Scout wire
+question key ordering (even when map values match), differing model selectors,
 extra request keys, request-size violations, or a capture attached to an
 historical renderer. A capture is an **operator-supplied input**, not proof that
 its bytes came from Scout. The owning transcript's original store-id bounds,
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 3a65d0919f0..b028eb069dc 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -58,10 +58,12 @@ def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes
                 or request["questions"] != questions
                 or (requested_model is not None and request.get("model") != requested_model)):
             raise ValueError(f"captured Scout request {record['id']} differs from frozen Noul contract")
-        # Dict equality ignores insertion order, but the model receives
-        # ordered wire bytes. The declared vocabulary freezes this order.
-        if list(request["questions"]) != list(questions):
-            raise ValueError(f"captured Scout request {record['id']} question order differs from frozen vocabulary")
+        # The production Rust serde_json::Map sorts keys on the Bazel
+        # build. The settings Vec order is not the serialized wire order.
+        # Capture bytes are pinned independently; reject any question-map
+        # order other than the production encoder's canonical sorted order.
+        if list(request["questions"]) != sorted(questions):
+            raise ValueError(f"captured Scout request {record['id']} question order differs from canonical Scout wire order")
         bodies[record["id"]] = encoded
     return bodies
 
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index bc891ece918..dbc8ccf6370 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1489,14 +1489,14 @@ def test_qualified_question_files_refuse_non_noul_or_ambiguous_vocabulary(self):
                         qualification.load_questions(path)
 
     def captured_scout_fixture(self):
-        # The fixture deliberately uses Rust-style compact JSON and a request
-        # field order different from the historical Python renderer.
+        # Rust serde_json::Map emits lexicographically sorted object keys in
+        # the Bazel runtime. The settings Vec uses a different question order.
         records = []
         for case in self.dataset[2:]:
             body = {"questions": self.questions, "state": "User: " + case["current_prompt"]}
             records.append({
                 "id": case["id"],
-                "body": json.dumps(body, ensure_ascii=False, separators=(",", ":")),
+                "body": json.dumps(body, ensure_ascii=False, separators=(",", ":"), sort_keys=True),
             })
         capture_bytes = json.dumps(records, ensure_ascii=False).encode("utf-8")
         manifest = copy.deepcopy(self.manifest)
@@ -1508,6 +1508,21 @@ def captured_scout_fixture(self):
         ])
         return capture_bytes, manifest
 
+    def test_captured_scout_accepts_canonical_wire_order_not_settings_vector_order(self):
+        capture, manifest = self.captured_scout_fixture()
+        self.assertNotEqual(list(self.questions), sorted(self.questions),
+                            "fixture must distinguish settings order from canonical wire order")
+        records = qualification.loads_unambiguous_json(capture)
+        wire_questions = qualification.loads_unambiguous_json(records[0]["body"])["questions"]
+        self.assertEqual(list(wire_questions), sorted(self.questions))
+        self.assertNotEqual(list(wire_questions), list(self.questions))
+        bodies = qualification.captured_request_bodies(
+            capture, ["holdout-positive", "holdout-negative"],
+            self.questions, None, manifest["max_request_bytes"],
+        )
+        self.assertEqual(bodies["holdout-positive"], records[0]["body"].encode("utf-8"))
+        self.assertEqual(bodies["holdout-negative"], records[1]["body"].encode("utf-8"))
+
     def test_captured_scout_requests_replay_exact_bytes_and_score(self):
         # This proves a captured arm is byte-exact, not that the synthetic
         # fixture came from real Scout or that its labels were reviewed.
@@ -1643,7 +1658,7 @@ def test_captured_question_order_cannot_be_changed_even_when_all_hashes_are_refr
             body = qualification.loads_unambiguous_json(record["body"])
             body["questions"] = dict(reversed(list(body["questions"].items())))
             self.assertEqual(body["questions"], self.questions)
-            self.assertNotEqual(list(body["questions"]), list(self.questions))
+            self.assertNotEqual(list(body["questions"]), sorted(self.questions))
             record["body"] = json.dumps(body, ensure_ascii=False, separators=(",", ":"))
         reordered = json.dumps(records, ensure_ascii=False).encode("utf-8")
         manifest["captured_requests_sha256"] = qualification.sha256_bytes(reordered)
@@ -1652,7 +1667,7 @@ def test_captured_question_order_cannot_be_changed_even_when_all_hashes_are_refr
             for record in records
         ])
         self.valid(manifest)
-        with self.assertRaisesRegex(ValueError, "question order"):
+        with self.assertRaisesRegex(ValueError, "canonical Scout wire order"):
             qualification.captured_request_bodies(
                 reordered, ["holdout-positive", "holdout-negative"],
                 self.questions, None, manifest["max_request_bytes"],

From 846b01ae9d9d9ae8ca6674a926c26227677d0537 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 15:50:23 -0800
Subject: [PATCH 33/76] fix(evals): reject contradictory Kev token usage
 evidence

---
 evals/intent_engine/HTTP_RUNNER.md        |  8 +++
 evals/intent_engine/run_http.py           | 19 ++++++++
 evals/intent_engine/test_qualification.py | 30 ++++++++++++
 evals/intent_engine/test_run_http.py      | 59 +++++++++++++++++++++++
 4 files changed, 116 insertions(+)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 5d037eb039d..9fce4269f97 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -86,6 +86,14 @@ Each result records `usage.input_tokens` as `input_tokens`,
 `dropped`. These are distinct server measurements; total input tokens are never
 substituted for state tokens. Omitted or null telemetry stays JSON `null`.
 Reported zeroes stay zero, and reported counts must be nonnegative integers.
+Impossible cross-counter evidence is refused before a fresh row is appended,
+before an interrupted log is resumed, and by the independent scorer: dropped
+state tokens may not exceed the original state count, and the number of state
+tokens actually consumed may not exceed the server's total input-token count.
+When no explicit consumed-state count exists, an original-minus-dropped count
+is checked where both measurements are present. These checks catch
+contradictory self-reports; they do not authenticate the server or replace
+independently verified token accounting.
 
 `score.py` computes token statistics only from measured cases and reports
 `input_tokens_measured_cases`, `state_tokens_measured_cases`, and
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 99e2d9e9693..f401646e724 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -253,6 +253,12 @@ def dropped_state_tokens(response, usage):
         if value is not None and (type(value) is not int or value < 0):
             raise ValueError("invalid truncation state-token count")
 
+    # A directly reported drop count cannot exceed the original state.
+    # Without this check, a malformed server can claim more dropped tokens
+    # than were ever sent and still leave a self-consistent evidence row.
+    if direct is not None and original is not None and direct > original:
+        raise ValueError("invalid truncation: dropped state tokens exceed original")
+
     derived = None
     if original is not None and used is not None:
         if used > original:
@@ -296,6 +302,19 @@ def validate_row_truncation(row):
     semantics. New rows retain the supplied counts/flags for independent
     qualification and exact partial-resume validation.
     """
+    # Kev's total input count includes every state token the model actually
+    # consumed, even when some original state tokens were truncated. A low
+    # reported input count cannot establish context fit if the used-state
+    # counter already exceeds it. Check fresh, resumed and scored rows alike.
+    original = row.get("state_tokens")
+    dropped = row.get("dropped")
+    used = row.get("state_tokens_used")
+    if used is None and type(original) is int and type(dropped) is int:
+        used = original - dropped
+    input_tokens = row.get("input_tokens")
+    if type(input_tokens) is int and type(used) is int and used > input_tokens:
+        raise ValueError("inconsistent usage: used state tokens exceed total input tokens")
+
     native_fields = ("state_tokens_used", "reported_state_tokens_dropped",
                      "truncated", "usage_truncated")
     if not any(field in row for field in native_fields):
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index dbc8ccf6370..6fbb770e51c 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -374,6 +374,36 @@ def test_native_truncation_partial_tampering_refuses_without_network(self):
                     (here / "results" / "native-partial_B_noul_text.json").exists()
                 )
 
+    def test_impossible_token_counters_cannot_qualify_even_with_rehashed_evidence(self):
+        control = qualification.qualified_report(
+            self.signed_result(), self.manifest, self.manifest_sha,
+            self.questions, self.dataset_bytes,
+        )
+        self.assertTrue(control["qualified_against_frozen_tolerances"])
+        # The old scorer accepted input_tokens=1 as proof of context fit
+        # even when Kev reported using 32 state tokens on that request.
+        forged = self.signed_result()
+        forged["rows"][0].update(
+            input_tokens=1, state_tokens=32, state_tokens_used=32, dropped=0,
+        )
+        with self.assertRaisesRegex(ValueError, "used state tokens exceed total input tokens"):
+            qualification.qualified_report(
+                forged, self.manifest, self.manifest_sha,
+                self.questions, self.dataset_bytes,
+            )
+
+        # A second contradictory server receipt also cannot be interpreted
+        # as a legitimate, zero-truncation or measured-drop result.
+        forged = self.signed_result()
+        forged["rows"][0].update(
+            state_tokens=3, reported_state_tokens_dropped=4, dropped=4,
+        )
+        with self.assertRaisesRegex(ValueError, "dropped state tokens exceed original"):
+            qualification.qualified_report(
+                forged, self.manifest, self.manifest_sha,
+                self.questions, self.dataset_bytes,
+            )
+
     def test_boolean_truncation_without_counts_is_unknown(self):
         self.assertIsNone(run_http.dropped_state_tokens({"truncated": False}, {}))
         self.assertIsNone(run_http.dropped_state_tokens({"truncated": True}, {}))
diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index 4f22af95b8b..f3a908c0350 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -437,6 +437,65 @@ def test_reported_usage_and_zero_counts_survive_resume(self):
         self.assertEqual(timing["truncation_measured_cases"], 2)
         self.assertEqual(timing["cases_truncated"], 1)
 
+    def test_inconsistent_state_usage_is_rejected_before_resume_or_new_append(self):
+        self.interrupt()
+        header, row = self.partial.read_text().splitlines()
+        forged = json.loads(row)
+        forged.update(
+            input_tokens=1, state_tokens=32, state_tokens_used=32, dropped=0,
+        )
+        self.partial.write_text(header + "\n" + json.dumps(forged) + "\n")
+        before = self.partial.read_bytes()
+        with patch.object(run_http, "post") as post:
+            with self.assertRaisesRegex(ValueError, "used state tokens exceed total input tokens"):
+                self.run_main()
+            post.assert_not_called()
+        self.assertEqual(self.partial.read_bytes(), before)
+        self.assertFalse(self.output.exists())
+
+        # The same impossible values in a fresh server response must not
+        # append a row or poison the original interrupted evidence.
+        valid, elapsed = self.response("", {"questions": run_http.noul_questions()})
+        impossible = copy.deepcopy(valid)
+        impossible["usage"] = {
+            "input_tokens": 1, "state_tokens": 32, "state_tokens_used": 32,
+        }
+        impossible["truncated"] = False
+        # Restore a genuine first row before exercising the new response.
+        self.partial.write_text(header + "\n" + row + "\n")
+        genuine_prefix = self.partial.read_bytes()
+        with patch.object(run_http, "post", return_value=(impossible, elapsed)) as post:
+            with self.assertRaisesRegex(ValueError, "used state tokens exceed total input tokens"):
+                self.run_main()
+            post.assert_called_once()
+        self.assertEqual(self.partial.read_bytes(), genuine_prefix)
+        self.assertFalse(self.output.exists())
+
+        with patch.object(run_http, "post", return_value=(valid, elapsed)) as post:
+            self.run_main()
+            post.assert_called_once()
+        self.assertTrue(self.output.exists())
+        self.assertFalse(self.partial.exists())
+        self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2)
+
+    def test_dropped_count_exceeding_original_state_is_refused(self):
+        with self.assertRaisesRegex(ValueError, "dropped state tokens exceed original"):
+            run_http.dropped_state_tokens(
+                {}, {"state_tokens": 3, "state_tokens_dropped": 4},
+            )
+        self.interrupt()
+        before = self.partial.read_bytes()
+        impossible, elapsed = self.response("", {"questions": run_http.noul_questions()})
+        impossible["usage"] = {
+            "input_tokens": 12, "state_tokens": 3, "state_tokens_dropped": 4,
+        }
+        with patch.object(run_http, "post", return_value=(impossible, elapsed)) as post:
+            with self.assertRaisesRegex(ValueError, "dropped state tokens exceed original"):
+                self.run_main()
+            post.assert_called_once()
+        self.assertEqual(self.partial.read_bytes(), before)
+        self.assertFalse(self.output.exists())
+
     def test_missing_usage_survives_resume_and_scoring(self):
         response, elapsed = self.response("", {"questions": run_http.noul_questions()})
         response["usage"] = None

From 77556fabe459186438aecc8b0cf1aa7f9642e777 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 16:10:18 -0800
Subject: [PATCH 34/76] fix(evals): reject non-Noul fields in qualified
 evidence

---
 evals/intent_engine/qualification.py      | 23 ++++++++++++++++
 evals/intent_engine/test_qualification.py | 32 +++++++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index b028eb069dc..33d4a9ccdab 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -371,6 +371,29 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
         if (row.get("labels") != annotation["verified_labels"]
                 or row.get("context_needed") != annotation["verified_context_needed"]):
             raise ValueError(f"qualification result {row['id']} does not match reviewed labels")
+    # Historical scoring accepts Choice results and optional "none" scores.
+    # Qualified Scout runs are Noul-only: the original HTTP runner cannot emit
+    # a Choice pick or an extra probability outside its frozen question set.
+    # Reject mixed-type edited evidence before computing qualification checks.
+    expected_questions = set(questions)
+    for row in rows:
+        probabilities = row.get("probs")
+        raw_probabilities = row.get("probs_raw")
+        if (
+            "choice" in row
+            or not isinstance(probabilities, dict)
+            or set(probabilities) != expected_questions
+            or (
+                raw_probabilities is not None
+                and (
+                    not isinstance(raw_probabilities, dict)
+                    or set(raw_probabilities) != expected_questions
+                )
+            )
+        ):
+            raise ValueError(
+                f"qualification row {row['id']} must contain only frozen Noul probabilities"
+            )
     # Native truncation counters and the normalized drop count must remain
     # consistent even if the result JSON was edited after the HTTP response.
     for row in rows:
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 6fbb770e51c..d22a19d5334 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -569,6 +569,38 @@ def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
             qualification.qualified_report(result, stricter, qualification.sha256_json(stricter), self.questions, self.dataset_bytes)
 
 
+    def test_qualified_scorer_rejects_edited_choice_and_extra_none_evidence(self):
+        # These edits previously passed the generic historical scorer. An
+        # independently scored Noul-only result must reject them even when
+        # the request hashes, experiment SHA and verified labels still agree.
+        control = self.signed_result()
+        self.assertTrue(qualification.qualified_report(
+            control, self.manifest, self.manifest_sha,
+            self.questions, self.dataset_bytes,
+        )["qualified_against_frozen_tolerances"])
+
+        for variant in ("extra_none_probability", "choice_on_every_case"):
+            with self.subTest(variant=variant):
+                forged = copy.deepcopy(control)
+                if variant == "extra_none_probability":
+                    forged["rows"][0]["probs"]["none"] = 0.95
+                else:
+                    for row in forged["rows"]:
+                        row["choice"] = "none"
+
+                # An unqualified historical result may legitimately have
+                # these fields. This passing control proves the old scored
+                # path would not refuse our adversarial evidence.
+                self.assertIn(
+                    "latency",
+                    score.score_config(forged, cats=tuple(self.questions)),
+                )
+                with self.assertRaisesRegex(ValueError, "only frozen Noul probabilities"):
+                    qualification.qualified_report(
+                        forged, self.manifest, self.manifest_sha,
+                        self.questions, self.dataset_bytes,
+                    )
+
     def test_qualification_reports_missing_or_drifting_model_identity(self):
         for value in (None, "", True, "another-model"):
             with self.subTest(observed_model=value):

From 86b95c25e3cd84c8d93af94afa87b13df2179729 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 17:45:02 -0800
Subject: [PATCH 35/76] fix(evals): freeze and enforce qualified HTTP timeouts

Refs #4535

Bind a per-call transport timeout to the held-out manifest and HTTP
experiment identity. Fail closed on an altered deadline before inference
or independent scoring; retain historical defaults. Add falsifiable
positive and forged-evidence regression tests.

Keep #4797 and #4776 DRAFT; never merge.
---
 evals/intent_engine/HTTP_RUNNER.md        | 10 ++++-
 evals/intent_engine/qualification.py      |  6 ++-
 evals/intent_engine/run_http.py           | 15 ++++++-
 evals/intent_engine/test_qualification.py | 52 +++++++++++++++++++++--
 4 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 9fce4269f97..ceb1916e038 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -11,6 +11,13 @@ python evals/intent_engine/run_http.py \
   B_noul_text
 ```
 
+The `--timeout-ms` CLI flag must match frozen `arm.timeout_ms` for qualified
+experiments. The frozen value (1 to 1,800,000 milliseconds) is enforced on
+each network request, not just checked as a p95 scoring tolerance afterward.
+This is a transport deadline, not proof of the production Scout pass's
+end-to-end cancellation or resource costs. Historical unqualified runs
+retain their original 30-minute default.
+
 The endpoint must use HTTP or HTTPS with a hostname. File URLs, data URLs,
 other non-HTTP schemes, malformed ports and URL-embedded credentials are
 refused before any evaluation output is opened. Otherwise urllib can read
@@ -157,7 +164,8 @@ Generate a new frozen manifest for a changed renderer; a renderer config name
 alone is not a code or output identity. This remains declared benchmark
 evidence, **not** production Scout transcript rehydration or runtime parity.
 Include an `arm` object predeclaring exactly one `config` renderer, a nullable
-`requested_model`, `endpoint_sha256` (SHA-256 of the exact URL bytes), and
+`requested_model`, `timeout_ms` (the enforced per-request timeout),
+`endpoint_sha256` (SHA-256 of the exact URL bytes), and
 `server_config_sha256` (the sorted-JSON digest of the complete serving
 configuration, via `qualification.sha256_json(server_config)`). The runner
 refuses additional renderers or changed serving/model/endpoint declarations
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 33d4a9ccdab..d4fde283d7a 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -145,11 +145,14 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     # A held-out split is not independent if the evaluator can try multiple
     # renderers, model selectors or serving declarations and pick the winner.
     arm = manifest.get("arm")
-    required_arm = {"config", "requested_model", "endpoint_sha256", "server_config_sha256"}
+    required_arm = {"config", "requested_model", "endpoint_sha256", "server_config_sha256", "timeout_ms"}
     if not isinstance(arm, dict) or set(arm) != required_arm:
         raise ValueError("qualification requires one frozen HTTP arm")
     if not isinstance(arm["config"], str) or not arm["config"].strip():
         raise ValueError("qualification HTTP arm requires a renderer config")
+    timeout_ms = arm["timeout_ms"]
+    if type(timeout_ms) is not int or not 0 < timeout_ms <= 1_800_000:
+        raise ValueError("qualification HTTP arm requires timeout_ms in (0, 1800000]")
     model = arm["requested_model"]
     if model is not None and (not isinstance(model, str) or not model.strip()):
         raise ValueError("qualification HTTP arm requires a valid requested model or null")
@@ -357,6 +360,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
     arm = validated["arm"]
     if (experiment.get("config") != arm["config"]
             or experiment.get("requested_model") != arm["requested_model"]
+            or experiment.get("timeout_ms") != arm["timeout_ms"]
             or experiment.get("endpoint_sha256") != arm["endpoint_sha256"]
             or sha256_json(server) != arm["server_config_sha256"]):
         raise ValueError("qualification HTTP arm differs from frozen manifest")
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index f401646e724..1300c212882 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -522,6 +522,8 @@ def main(argv=None):
     ap.add_argument("--questions-file", help="declared typed Noul questions; qualified runs only, requires separate runtime provenance")
     ap.add_argument("--captured-requests", help="frozen verbatim Scout HTTP requests, captured qualification arm only")
     ap.add_argument("--send-model")
+    ap.add_argument("--timeout-ms", type=int, default=1_800_000,
+                    help="per-request HTTP timeout in milliseconds (1..1800000); frozen for qualified runs")
     ap.add_argument("--bearer-token-env", metavar="NAME",
                     help="name of an environment variable holding a bearer token (never recorded)")
     ap.add_argument("--sample-out", help="write the first raw response here")
@@ -533,6 +535,8 @@ def main(argv=None):
     if (args.noul_temperature is not None
             and (not math.isfinite(args.noul_temperature) or args.noul_temperature <= 0)):
         raise ValueError("--noul-temperature must be finite and greater than zero")
+    if not 0 < args.timeout_ms <= 1_800_000:
+        raise ValueError("--timeout-ms must be in (0, 1800000]")
     require_http_endpoint(args.url)
     server_config = load_server_config(args.server_config)
     bearer_token = (load_bearer_token(args.bearer_token_env, args.url)
@@ -563,6 +567,7 @@ def main(argv=None):
         arm = qualified["arm"]
         if (args.configs != [arm["config"]]
                 or args.send_model != arm["requested_model"]
+                or args.timeout_ms != arm["timeout_ms"]
                 or hashlib.sha256(args.url.encode("utf-8")).hexdigest() != arm["endpoint_sha256"]
                 or digest(server_config) != arm["server_config_sha256"]):
             raise ValueError("qualification HTTP arm differs from frozen manifest")
@@ -646,6 +651,7 @@ def main(argv=None):
         }
         if qualification_sha is not None:
             identity["qualification_manifest_sha256"] = qualification_sha
+            identity["timeout_ms"] = args.timeout_ms
         header = {"experiment": identity, "experiment_sha256": digest(identity)}
         partial_path = path + ".partial"
         try:
@@ -680,10 +686,15 @@ def main(argv=None):
                 if case["id"] in done:
                     continue
                 body, request_sha = requests[case["id"]]
+                # Match the precommitted model-serving deadline; a post-hoc
+                # p95 threshold cannot substitute for a real request timeout.
+                timeout_arg = ({"timeout": args.timeout_ms / 1000}
+                               if qualification_sha is not None or args.timeout_ms != 1_800_000
+                               else {})
                 if bearer_token is None:
-                    response, ms = post(args.url, body)
+                    response, ms = post(args.url, body, **timeout_arg)
                 else:
-                    response, ms = post(args.url, body, bearer_token=bearer_token)
+                    response, ms = post(args.url, body, bearer_token=bearer_token, **timeout_arg)
                 probs = noul_probabilities(response, questions)
                 observed_model = response.get("model")
                 if (args.send_model is not None and observed_model is not None
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index d22a19d5334..1493c0035e9 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -61,6 +61,7 @@ def setUp(self):
                 "config": "B_noul_text", "requested_model": None,
                 "endpoint_sha256": qualification.sha256_bytes(self.endpoint.encode("utf-8")),
                 "server_config_sha256": qualification.sha256_json(self.server_config),
+                "timeout_ms": 1_800_000,
             },
             "noul_temperature": None,
             "max_request_bytes": 65536,
@@ -516,6 +517,7 @@ def signed_result(self):
             "config": "B_noul_text",
             "requested_model": None,
             "endpoint_sha256": self.manifest["arm"]["endpoint_sha256"],
+            "timeout_ms": self.manifest["arm"]["timeout_ms"],
             "case_ids": [row["id"] for row in rows],
             "dataset_sha256": self.dataset_sha,
             "case_annotations_sha256": score.evidence_digest([
@@ -962,7 +964,7 @@ def test_http_runner_asks_only_pre_frozen_holdout_and_resumes_with_exact_identit
             (here / "server.json").write_text(json.dumps(self.server_config))
             calls = []
 
-            def answer(url, body):
+            def answer(url, body, **kwargs):
                 calls.append(body)
                 return {"model": "self-reported-test-model", "answers": {
                     key: {"type": "noul", "noul": 0.5}
@@ -1187,6 +1189,45 @@ def test_runner_can_consume_explicit_frozen_input_file(self):
 
 
 
+    def test_qualified_timeout_is_enforced_and_scored_against_frozen_arm(self):
+        manifest = copy.deepcopy(self.manifest)
+        manifest["arm"]["timeout_ms"] = 250
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_text(json.dumps(manifest))
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            args = [
+                "--url", self.endpoint, "--tag", "deadline",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "--timeout-ms", "250", "B_noul_text",
+            ]
+            response = {
+                "model": "self-reported-test-model",
+                "answers": {key: {"noul": 0.9} for key in self.questions},
+                "usage": {"input_tokens": 32, "state_tokens_dropped": 0},
+            }
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post", return_value=(response, 5.0)) as post:
+                    run_http.main(args)
+                self.assertEqual(post.call_count, 2)
+                self.assertTrue(all(call.kwargs == {"timeout": 0.25}
+                                    for call in post.call_args_list))
+                result = json.loads(
+                    (here / "results" / "deadline_B_noul_text.json").read_text()
+                )
+                self.assertEqual(result["experiment"]["timeout_ms"], 250)
+                # Rehashing result metadata cannot change the frozen deadline.
+                result["experiment"]["timeout_ms"] = 1_800_000
+                result["experiment_sha256"] = score.evidence_digest(result["experiment"])
+                with self.assertRaisesRegex(ValueError, "HTTP arm differs"):
+                    qualification.qualified_report(
+                        result, manifest,
+                        qualification.sha256_bytes((here / "manifest.json").read_bytes()),
+                        self.questions, self.dataset_bytes,
+                    )
+
     def test_frozen_http_arm_is_required_and_well_formed(self):
         mutations = (
             (lambda m: m.pop("arm"), "frozen HTTP arm"),
@@ -1195,6 +1236,10 @@ def test_frozen_http_arm_is_required_and_well_formed(self):
             (lambda m: m["arm"].update(requested_model=""), "requested model"),
             (lambda m: m["arm"].update(endpoint_sha256="x" * 64), "endpoint_sha256"),
             (lambda m: m["arm"].update(server_config_sha256=None), "server_config_sha256"),
+            (lambda m: m["arm"].pop("timeout_ms"), "frozen HTTP arm"),
+            (lambda m: m["arm"].update(timeout_ms=0), "timeout_ms"),
+            (lambda m: m["arm"].update(timeout_ms=1_800_001), "timeout_ms"),
+            (lambda m: m["arm"].update(timeout_ms=True), "timeout_ms"),
         )
         for mutate, error in mutations:
             with self.subTest(error=error):
@@ -1220,6 +1265,7 @@ def test_frozen_http_arm_refuses_post_hoc_reselection_before_inference(self):
                 ("multiple-renderers", lambda a: a.append("A_noul_prompt")),
                 ("model", lambda a: a.__setitem__(slice(-1, -1), ["--send-model", "unfrozen-model"])),
                 ("endpoint", lambda a: a.__setitem__(1, "http://127.0.0.1:8010/v1/systemone")),
+                ("timeout", lambda a: a.__setitem__(slice(-1, -1), ["--timeout-ms", "250"])),
             )
             with patch.object(run_http, "HERE", str(here)):
                 for label, mutation in trials:
@@ -1483,7 +1529,7 @@ def test_frozen_ten_question_vocabulary_runs_and_scores_without_historical_categ
             (here / "questions.json").write_text(json.dumps(questions))
             (here / "server.json").write_text(json.dumps(self.server_config))
 
-            def inference(_url, body):
+            def inference(_url, body, **kwargs):
                 self.assertEqual(set(body["questions"]), set(questions))
                 probability = 0.9 if "What changes?" in body["state"] else 0.1
                 return ({
@@ -1597,7 +1643,7 @@ def test_captured_scout_requests_replay_exact_bytes_and_score(self):
             (here / "server.json").write_text(json.dumps(self.server_config))
             transmitted = []
 
-            def inference(_url, body):
+            def inference(_url, body, **kwargs):
                 self.assertIsInstance(body, bytes, "must not reserialize the Rust request")
                 transmitted.append(body)
                 request = qualification.loads_unambiguous_json(body)

From 398ca9dbda32bff04ce8f7701facfc76304cc992 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 18:36:28 -0800
Subject: [PATCH 36/76] fix(evals): reject noncanonical captured Scout HTTP
 JSON

Refs #4535

Refuse semantically equivalent but unproducible Rust wire encodings,
even when captures and experiment digests are coherently rehashed.
Exercise direct validation, zero-network runner refusal and independent
scoring rejection for top-level order and whitespace drift.

Keep #4797 and #4776 DRAFT; never merge.
---
 evals/intent_engine/HTTP_RUNNER.md        |  9 ++-
 evals/intent_engine/qualification.py      | 13 ++++
 evals/intent_engine/test_qualification.py | 76 +++++++++++++++++++++++
 3 files changed, 97 insertions(+), 1 deletion(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index ceb1916e038..77861c25204 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -316,7 +316,14 @@ For this arm, `questions_sha256` binds the declared settings vocabulary and
 its order, while `captured_requests_sha256` and `holdout_requests_sha256`
 bind the actual ordered **wire bytes**. The capture validator requires the
 canonical sorted question-map wire order, not a match to the settings vector's
-iteration order.
+iteration order. It also checks the full compact Rust `serde_json::Value` wire
+encoding (lexically ordered object keys, no optional JSON whitespace and
+UTF-8 text rather than alternative Unicode escapes). Semantically equivalent
+JSON with a different top-level order or pretty-printing is not a legitimate
+capture of this pinned runtime serializer, even if every evidence hash has
+been recomputed. Only the inner request must be canonical; the outer evidence
+file preserves its original hashed bytes. This wire-shape check does **not**
+attest that the bytes actually originated from Scout.
 
 The manifest must set `arm.config` to `S_scout_captured`,
 `captured_requests_sha256` to the SHA-256 of the entire **original capture
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index d4fde283d7a..04dd87d795e 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -64,6 +64,19 @@ def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes
         # order other than the production encoder's canonical sorted order.
         if list(request["questions"]) != sorted(questions):
             raise ValueError(f"captured Scout request {record['id']} question order differs from canonical Scout wire order")
+        # The production client passes a serde_json::Value to ureq::send_json.
+        # Its default Bazel serializer emits compact JSON with lexically sorted
+        # object keys, not just a sorted questions map. Otherwise an invented
+        # (but self-consistently rehashed) request could be called a Scout capture
+        # even though the production encoder could not have sent those bytes.
+        canonical = json.dumps(
+            request, sort_keys=True, ensure_ascii=False,
+            separators=(",", ":"), allow_nan=False,
+        ).encode("utf-8")
+        if encoded != canonical:
+            raise ValueError(
+                f"captured Scout request {record['id']} is not canonical Rust JSON wire bytes"
+            )
         bodies[record["id"]] = encoded
     return bodies
 
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 1493c0035e9..ab3f7aabfda 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1821,6 +1821,82 @@ def test_captured_question_order_cannot_be_changed_even_when_all_hashes_are_refr
                 self.questions, self.dataset_bytes, reordered,
             )
 
+    def test_noncanonical_wire_bytes_refuse_even_after_rehashing_all_evidence(self):
+        # A syntactically different request can decode to identical Noul
+        # semantics, yet cannot be the original Rust send_json wire payload.
+        # Freeze new capture/holdout/result digests to bypass hash-mismatch
+        # guards and exercise the actual production-wire-shape check.
+        capture, _ = self.captured_scout_fixture()
+        original = qualification.loads_unambiguous_json(capture)
+        for encoding in ("top_level_order", "json_spacing"):
+            with self.subTest(encoding=encoding):
+                records = copy.deepcopy(original)
+                for record in records:
+                    decoded = qualification.loads_unambiguous_json(record["body"])
+                    if encoding == "top_level_order":
+                        changed = json.dumps(
+                            {"state": decoded["state"], "questions": decoded["questions"]},
+                            ensure_ascii=False, separators=(",", ":"),
+                        )
+                    else:
+                        changed = json.dumps(decoded, sort_keys=True, ensure_ascii=False)
+                    self.assertEqual(qualification.loads_unambiguous_json(changed), decoded)
+                    self.assertNotEqual(changed, record["body"])
+                    record["body"] = changed
+                altered = json.dumps(records, ensure_ascii=False).encode("utf-8")
+                _, manifest = self.captured_scout_fixture()
+                manifest["captured_requests_sha256"] = qualification.sha256_bytes(altered)
+                manifest["holdout_requests_sha256"] = score.evidence_digest([
+                    [record["id"], qualification.sha256_bytes(record["body"].encode("utf-8"))]
+                    for record in records
+                ])
+                self.valid(manifest)
+                with self.assertRaisesRegex(ValueError, "canonical Rust JSON wire bytes"):
+                    qualification.captured_request_bodies(
+                        altered, ["holdout-positive", "holdout-negative"],
+                        self.questions, None, manifest["max_request_bytes"],
+                    )
+
+                with tempfile.TemporaryDirectory() as temp:
+                    here = Path(temp)
+                    (here / "dataset.json").write_bytes(self.dataset_bytes)
+                    manifest_bytes = json.dumps(manifest).encode("utf-8")
+                    (here / "manifest.json").write_bytes(manifest_bytes)
+                    (here / "server.json").write_text(json.dumps(self.server_config))
+                    (here / "captured.json").write_bytes(altered)
+                    with patch.object(run_http, "HERE", str(here)):
+                        with patch.object(run_http, "post") as post:
+                            with self.assertRaisesRegex(ValueError, "canonical Rust JSON wire bytes"):
+                                run_http.main([
+                                    "--url", self.endpoint, "--tag", "noncanonical",
+                                    "--server-config", str(here / "server.json"),
+                                    "--qualification-manifest", str(here / "manifest.json"),
+                                    "--captured-requests", str(here / "captured.json"),
+                                    qualification.CAPTURED_SCOUT_CONFIG,
+                                ])
+                            post.assert_not_called()
+                    self.assertFalse((here / "results").exists())
+
+                    result = self.signed_result()
+                    for row, record in zip(result["rows"], records, strict=True):
+                        raw = record["body"].encode("utf-8")
+                        row["request_sha256"] = qualification.sha256_bytes(raw)
+                        row["request_size_bytes"] = len(raw)
+                    result["config"] = "trial_" + qualification.CAPTURED_SCOUT_CONFIG
+                    experiment = result["experiment"]
+                    experiment["config"] = qualification.CAPTURED_SCOUT_CONFIG
+                    experiment["requests_sha256"] = manifest["holdout_requests_sha256"]
+                    experiment["qualification_manifest_sha256"] = qualification.sha256_bytes(
+                        manifest_bytes
+                    )
+                    result["experiment_sha256"] = score.evidence_digest(experiment)
+                    with self.assertRaisesRegex(ValueError, "canonical Rust JSON wire bytes"):
+                        qualification.qualified_report(
+                            result, manifest,
+                            qualification.sha256_bytes(manifest_bytes),
+                            self.questions, self.dataset_bytes, altered,
+                        )
+
 
 if __name__ == "__main__":
     unittest.main()

From 0eb8139bb59e83acb585ee85cea5c0dac9380356 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 19:20:27 -0800
Subject: [PATCH 37/76] fix(evals): require Scout HTTP 200 for qualification
 evidence

Refs #4535

Reject 201/202/206 before recording predictions as successful model
responses. Exercise real loopback non-200 refusal, unchanged partial
and successful 200 resume; keep #4797 and #4776 DRAFT, no merge.
---
 evals/intent_engine/HTTP_RUNNER.md   |  4 ++
 evals/intent_engine/run_http.py      |  4 ++
 evals/intent_engine/test_run_http.py | 59 +++++++++++++++++++++++++++-
 3 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 77861c25204..2dc0d4d6a37 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -30,6 +30,10 @@ the HTTP Authorization header, and never stores the credential or its variable
 name in result evidence. An authenticated remote endpoint must use HTTPS;
 plain HTTP is accepted only for direct loopback addresses. Ambient proxies
 and HTTP redirects are disabled, including for authenticated requests.
+An HTTP response counts as a decision only at status **200**, matching the
+production Scout client. Other successful-looking 2xx statuses (201/202/206)
+refuse before reading or publishing a prediction; an interrupted exact-identity
+run may resume after a genuine 200 without accepting the refused response.
 
 Frozen inputs and HTTP responses must be unambiguous JSON: repeated object keys
 (including identical duplicate values) and nonfinite constants such as `NaN`
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 1300c212882..3903d847a87 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -180,6 +180,10 @@ def post(url, body, timeout=1800, bearer_token=None):
     )
     started = time.perf_counter()
     with opener.open(req, timeout=timeout) as response:
+        # urllib treats every 2xx status as successful. The Rust Scout client
+        # accepts exactly 200, so a 201/202/206 must not become model evidence.
+        if response.status != 200:
+            raise ValueError(f"decision HTTP status {response.status}; Scout requires 200")
         raw = response.read(MAX_HTTP_RESPONSE_BYTES + 1)
         if len(raw) > MAX_HTTP_RESPONSE_BYTES:
             raise ValueError("decision HTTP response exceeds bounded byte limit")
diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index f3a908c0350..6bf4797d9c9 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -87,7 +87,9 @@ def test_ambiguous_http_json_is_rejected_before_response_admission(self):
         ):
             with self.subTest(name=name):
                 with patch("urllib.request.build_opener") as opener:
-                    opener.return_value.open.return_value = io.BytesIO(payload)
+                    response = io.BytesIO(payload)
+                    response.status = 200
+                    opener.return_value.open.return_value = response
                     with self.assertRaisesRegex(ValueError, message):
                         run_http.post("http://localhost:8009/v1/systemone",
                                       {"state": "test", "questions": {}})
@@ -102,6 +104,7 @@ def test_http_response_limit_reads_only_bounded_bytes_and_accepts_boundary(self)
                     # The HTTP body is read from the context manager's
                     # entered stream, not the mock returned by opener.open().
                     response = opener.return_value.open.return_value.__enter__.return_value
+                    response.status = 200
                     response.read.side_effect = io.BytesIO(payload).read
                     if accepted:
                         result, latency = run_http.post(
@@ -122,7 +125,9 @@ def test_oversized_response_preserves_resumable_evidence_without_sample(self):
         sample = self.root / "sample.json"
         payload = b" " * (run_http.MAX_HTTP_RESPONSE_BYTES + 1)
         with patch("urllib.request.build_opener") as opener:
-            opener.return_value.open.return_value = io.BytesIO(payload)
+            response = io.BytesIO(payload)
+            response.status = 200
+            opener.return_value.open.return_value = response
             with self.assertRaisesRegex(ValueError, "bounded byte limit"):
                 self.run_main(("--sample-out", str(sample)))
         self.assertFalse(sample.exists())
@@ -1085,6 +1090,7 @@ def test_calibration_remains_finite_at_extreme_positive_temperatures(self):
 
     def test_transmitted_bytes_match_the_recorded_request_fingerprint(self):
         response = io.BytesIO(json.dumps({"answers": {}}).encode())
+        response.status = 200
         body = {"state": "Unicode λ", "questions": {"a": {}, "b": {}}}
         with patch("urllib.request.build_opener") as build_opener:
             build_opener.return_value.open.return_value = response
@@ -1156,6 +1162,55 @@ def test_bad_auth_configuration_refuses_before_http_or_partial_writes(self):
             post.assert_not_called()
         self.assertFalse(self.partial.exists())
 
+    def test_non_200_success_status_does_not_become_scout_evidence(self):
+        # urllib accepts all 2xx statuses, but the production Scout client
+        # refuses any status other than 200 before parsing a typed response.
+        status = [201]
+        requests = []
+        reply = json.dumps({
+            "answers": {key: {"noul": 0.25} for key in run_http.noul_questions()},
+            "usage": {"input_tokens": 12},
+        }).encode("utf-8")
+
+        class RespondWithStatus(BaseHTTPRequestHandler):
+            def do_POST(self):
+                requests.append(self.rfile.read(int(self.headers["Content-Length"])))
+                self.send_response(status[0])
+                self.send_header("Content-Length", str(len(reply)))
+                self.end_headers()
+                self.wfile.write(reply)
+
+            def log_message(self, *_):
+                pass
+
+        server = HTTPServer(("127.0.0.1", 0), RespondWithStatus)
+        worker = Thread(target=server.serve_forever, daemon=True)
+        worker.start()
+        self.base[1] = f"http://127.0.0.1:{server.server_port}/v1/systemone"
+        sample = self.root / "http-status-sample.json"
+        try:
+            for code in (201, 202, 206):
+                with self.subTest(status=code):
+                    status[0] = code
+                    with self.assertRaisesRegex(ValueError, f"decision HTTP status {code}"):
+                        self.run_main(("--sample-out", str(sample)))
+                    self.assertTrue(self.partial.exists())
+                    self.assertEqual(len(self.partial.read_bytes().splitlines()), 1)
+                    self.assertFalse(self.output.exists())
+                    self.assertFalse(sample.exists())
+            # A legitimate 200 on the same frozen experiment can recover
+            # without accepting the previously refused response as a row.
+            status[0] = 200
+            self.run_main(("--sample-out", str(sample)))
+            self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2)
+            self.assertTrue(sample.exists())
+            self.assertFalse(self.partial.exists())
+            self.assertEqual(len(requests), 5)
+        finally:
+            server.shutdown()
+            worker.join(timeout=2)
+            server.server_close()
+
     def test_bearer_transport_sends_only_to_selected_loopback_server(self):
         received = []
 

From ff68b4a444b92713d411c6e394e6ff7a529dd894 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 19:22:27 -0800
Subject: [PATCH 38/76] fix(evals): bind captured Scout state to runtime UTF-8
 byte budget

Refs #4535

Freeze the production decision client's max_state_bytes independently of
request JSON size and reject invented longer captures before HTTP/scoring.
Cover rehashed Unicode drift, no-network refusal, scorer rejection and
exact-boundary acceptance. Preserve DRAFT/NO MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md        | 11 ++-
 evals/intent_engine/qualification.py      | 23 +++++-
 evals/intent_engine/run_http.py           |  1 +
 evals/intent_engine/test_qualification.py | 95 +++++++++++++++++++++++
 4 files changed, 128 insertions(+), 2 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 2dc0d4d6a37..8988f6b91cb 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -329,7 +329,16 @@ been recomputed. Only the inner request must be canonical; the outer evidence
 file preserves its original hashed bytes. This wire-shape check does **not**
 attest that the bytes actually originated from Scout.
 
-The manifest must set `arm.config` to `S_scout_captured`,
+The manifest must set `arm.config` to `S_scout_captured` and freeze a
+positive integer `max_state_bytes` from the actual production
+`DecisionEngineConfigV1` / effective Scout settings. This limits the UTF-8
+bytes in the decoded `state` field separately from the whole HTTP JSON byte
+ceiling. The Rust client keeps only the newest state tail (on a character
+boundary) if the input is larger; therefore a claimed capture exceeding this
+state cap cannot have been emitted by that configured client. The validator
+refuses it before inference, even if all capture and manifest digests are
+recomputed. This is a **declared** setting until verified against canonical
+settings provenance, not proof of production runtime identity. Set
 `captured_requests_sha256` to the SHA-256 of the entire **original capture
 file**, and `holdout_requests_sha256` to the existing ordered
 `[[case_id, sha256(inner_body_bytes)], ...]` commitment. Other manifest
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 04dd87d795e..da47ebba517 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -32,12 +32,14 @@ def reject_nonfinite(constant):
 CAPTURED_SCOUT_CONFIG = "S_scout_captured"
 
 
-def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes):
+def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes, max_state_bytes):
     """Replay exact, operator-supplied Scout HTTP request bytes without rendering.
 
     Captures are declarations until independently exported from a digest-checked
     owning transcript window. This method does not attest their origin.
     """
+    if type(max_state_bytes) is not int or max_state_bytes <= 0:
+        raise ValueError("captured Scout requests require a positive frozen state byte budget")
     records = loads_unambiguous_json(raw)
     if (not isinstance(records, list) or len(records) != len(case_ids)
             or not all(isinstance(record, dict)
@@ -58,6 +60,13 @@ def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes
                 or request["questions"] != questions
                 or (requested_model is not None and request.get("model") != requested_model)):
             raise ValueError(f"captured Scout request {record['id']} differs from frozen Noul contract")
+        # DecisionEngineClientV1::request_body keeps only the newest
+        # max_state_bytes of UTF-8 state on a character boundary. A longer
+        # captured state could never have been sent by the frozen client.
+        if len(request["state"].encode("utf-8")) > max_state_bytes:
+            raise ValueError(
+                f"captured Scout request {record['id']} exceeds frozen state byte budget"
+            )
         # The production Rust serde_json::Map sorts keys on the Bazel
         # build. The settings Vec order is not the serialized wire order.
         # Capture bytes are pinned independently; reject any question-map
@@ -181,6 +190,16 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     elif captured_sha is not None:
         raise ValueError("only captured Scout arms may declare captured request evidence")
 
+    # Freeze the production DecisionEngineConfigV1 state cap independently
+    # from the whole serialized HTTP request limit. Without this, an invented
+    # capture with a larger model-visible transcript could be called parity.
+    state_limit = manifest.get("max_state_bytes")
+    if arm["config"] == CAPTURED_SCOUT_CONFIG:
+        if type(state_limit) is not int or state_limit <= 0:
+            raise ValueError("captured Scout arm requires frozen positive max_state_bytes")
+    elif state_limit is not None:
+        raise ValueError("only captured Scout arms may declare max_state_bytes")
+
     request_limit = manifest.get("max_request_bytes")
     if (isinstance(request_limit, bool) or not isinstance(request_limit, int)
             or request_limit <= 0):
@@ -301,6 +320,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
         "arm": arm,
         "captured_requests_sha256": captured_sha,
         "max_request_bytes": request_limit,
+        "max_state_bytes": state_limit,
         "max_input_tokens": token_limit,
         "thresholds": thresholds,
         "noul_temperature": client_temperature,
@@ -429,6 +449,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
         captured = captured_request_bodies(
             captured_requests_bytes, validated["heldout_ids"], questions,
             arm["requested_model"], validated["max_request_bytes"],
+            validated["max_state_bytes"],
         )
         renderer = None
     else:
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 3903d847a87..61291ad4029 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -585,6 +585,7 @@ def main(argv=None):
             captured = captured_request_bodies(
                 capture_bytes, qualified["heldout_ids"], questions,
                 args.send_model, qualified["max_request_bytes"],
+                qualified["max_state_bytes"],
             )
         keep = set(qualified["heldout_ids"])
         dataset = [case for case in dataset if case["id"] in keep]
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index ab3f7aabfda..d80d305f1d6 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1609,6 +1609,7 @@ def captured_scout_fixture(self):
         capture_bytes = json.dumps(records, ensure_ascii=False).encode("utf-8")
         manifest = copy.deepcopy(self.manifest)
         manifest["arm"]["config"] = qualification.CAPTURED_SCOUT_CONFIG
+        manifest["max_state_bytes"] = 4096
         manifest["captured_requests_sha256"] = qualification.sha256_bytes(capture_bytes)
         manifest["holdout_requests_sha256"] = score.evidence_digest([
             [record["id"], hashlib.sha256(record["body"].encode("utf-8")).hexdigest()]
@@ -1627,6 +1628,7 @@ def test_captured_scout_accepts_canonical_wire_order_not_settings_vector_order(s
         bodies = qualification.captured_request_bodies(
             capture, ["holdout-positive", "holdout-negative"],
             self.questions, None, manifest["max_request_bytes"],
+        manifest["max_state_bytes"],
         )
         self.assertEqual(bodies["holdout-positive"], records[0]["body"].encode("utf-8"))
         self.assertEqual(bodies["holdout-negative"], records[1]["body"].encode("utf-8"))
@@ -1735,6 +1737,7 @@ def test_capture_drift_and_invalid_contract_refuse_before_network(self):
                         qualification.captured_request_bodies(
                             changed_bytes, ["holdout-positive", "holdout-negative"],
                             self.questions, None, manifest["max_request_bytes"],
+                        manifest["max_state_bytes"],
                         )
             for name, changed in [
                 ("changed text", capture_bytes.replace(b"What changes?", b"What changed?")),
@@ -1779,6 +1782,7 @@ def test_captured_question_order_cannot_be_changed_even_when_all_hashes_are_refr
             qualification.captured_request_bodies(
                 reordered, ["holdout-positive", "holdout-negative"],
                 self.questions, None, manifest["max_request_bytes"],
+            manifest["max_state_bytes"],
             )
 
         # Runner refusal precedes network traffic and evidence-file creation.
@@ -1821,6 +1825,96 @@ def test_captured_question_order_cannot_be_changed_even_when_all_hashes_are_refr
                 self.questions, self.dataset_bytes, reordered,
             )
 
+    def test_captured_state_budget_is_utf8_bytes_and_independently_checked(self):
+        capture, manifest = self.captured_scout_fixture()
+        records = qualification.loads_unambiguous_json(capture)
+        # Two Unicode scalar values are four UTF-8 bytes, not two bytes.
+        # Recompute every capture and request digest, so the refusal is
+        # caused by the runtime state limit, not by stale provenance hashes.
+        for record in records:
+            body = qualification.loads_unambiguous_json(record["body"])
+            body["state"] = "éé"
+            record["body"] = json.dumps(
+                body, sort_keys=True, ensure_ascii=False, separators=(",", ":"),
+            )
+        capture = json.dumps(records, ensure_ascii=False).encode("utf-8")
+        manifest["captured_requests_sha256"] = qualification.sha256_bytes(capture)
+        manifest["holdout_requests_sha256"] = score.evidence_digest([
+            [record["id"], qualification.sha256_bytes(record["body"].encode("utf-8"))]
+            for record in records
+        ])
+        for limit in (None, True, 0, "4096"):
+            with self.subTest(invalid_limit=limit):
+                invalid = copy.deepcopy(manifest)
+                invalid["max_state_bytes"] = limit
+                with self.assertRaisesRegex(ValueError, "max_state_bytes"):
+                    self.valid(invalid)
+        unexpected = copy.deepcopy(self.manifest)
+        unexpected["max_state_bytes"] = 4096
+        with self.assertRaisesRegex(ValueError, "only captured Scout"):
+            self.valid(unexpected)
+
+        manifest["max_state_bytes"] = 3
+        self.valid(manifest)
+        with self.assertRaisesRegex(ValueError, "state byte budget"):
+            qualification.captured_request_bodies(
+                capture, ["holdout-positive", "holdout-negative"],
+                self.questions, None, manifest["max_request_bytes"],
+                manifest["max_state_bytes"],
+            )
+
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            manifest_bytes = json.dumps(manifest).encode("utf-8")
+            (here / "manifest.json").write_bytes(manifest_bytes)
+            (here / "captured.json").write_bytes(capture)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            with patch.object(run_http, "HERE", str(here)):
+                with patch.object(run_http, "post") as inference:
+                    with self.assertRaisesRegex(ValueError, "state byte budget"):
+                        run_http.main([
+                            "--url", self.endpoint, "--tag", "oversized-state",
+                            "--server-config", str(here / "server.json"),
+                            "--qualification-manifest", str(here / "manifest.json"),
+                            "--captured-requests", str(here / "captured.json"),
+                            qualification.CAPTURED_SCOUT_CONFIG,
+                        ])
+                    inference.assert_not_called()
+            self.assertFalse((here / "results").exists())
+
+            result = self.signed_result()
+            for row, record in zip(result["rows"], records, strict=True):
+                encoded = record["body"].encode("utf-8")
+                row["request_sha256"] = qualification.sha256_bytes(encoded)
+                row["request_size_bytes"] = len(encoded)
+            result["config"] = "trial_" + qualification.CAPTURED_SCOUT_CONFIG
+            experiment = result["experiment"]
+            experiment["config"] = qualification.CAPTURED_SCOUT_CONFIG
+            experiment["requests_sha256"] = manifest["holdout_requests_sha256"]
+            experiment["qualification_manifest_sha256"] = qualification.sha256_bytes(manifest_bytes)
+            result["experiment_sha256"] = score.evidence_digest(experiment)
+            with self.assertRaisesRegex(ValueError, "state byte budget"):
+                qualification.qualified_report(
+                    result, manifest, qualification.sha256_bytes(manifest_bytes),
+                    self.questions, self.dataset_bytes, capture,
+                )
+
+        # Boundary acceptance is equally important: the runtime would send
+        # this four-byte state unchanged with a four-byte budget.
+        manifest["max_state_bytes"] = 4
+        self.valid(manifest)
+        accepted = qualification.captured_request_bodies(
+            capture, ["holdout-positive", "holdout-negative"],
+            self.questions, None, manifest["max_request_bytes"],
+            manifest["max_state_bytes"],
+        )
+        self.assertEqual(len(accepted), 2)
+        self.assertEqual(
+            qualification.loads_unambiguous_json(accepted["holdout-positive"])["state"],
+            "éé",
+        )
+
     def test_noncanonical_wire_bytes_refuse_even_after_rehashing_all_evidence(self):
         # A syntactically different request can decode to identical Noul
         # semantics, yet cannot be the original Rust send_json wire payload.
@@ -1855,6 +1949,7 @@ def test_noncanonical_wire_bytes_refuse_even_after_rehashing_all_evidence(self):
                     qualification.captured_request_bodies(
                         altered, ["holdout-positive", "holdout-negative"],
                         self.questions, None, manifest["max_request_bytes"],
+                    manifest["max_state_bytes"],
                     )
 
                 with tempfile.TemporaryDirectory() as temp:

From 42cd1afb4437bdfb1e29e445f81deb96a32827ec Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 19:24:50 -0800
Subject: [PATCH 39/76] test(evals): align captured Scout state-cap fixture
 arguments

Refs #4535

Keep required state-byte-budget arguments aligned with existing test
call indentation. DRAFT only; do not merge.
---
 evals/intent_engine/test_qualification.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index d80d305f1d6..7cade27f2ec 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1628,7 +1628,7 @@ def test_captured_scout_accepts_canonical_wire_order_not_settings_vector_order(s
         bodies = qualification.captured_request_bodies(
             capture, ["holdout-positive", "holdout-negative"],
             self.questions, None, manifest["max_request_bytes"],
-        manifest["max_state_bytes"],
+            manifest["max_state_bytes"],
         )
         self.assertEqual(bodies["holdout-positive"], records[0]["body"].encode("utf-8"))
         self.assertEqual(bodies["holdout-negative"], records[1]["body"].encode("utf-8"))
@@ -1737,7 +1737,7 @@ def test_capture_drift_and_invalid_contract_refuse_before_network(self):
                         qualification.captured_request_bodies(
                             changed_bytes, ["holdout-positive", "holdout-negative"],
                             self.questions, None, manifest["max_request_bytes"],
-                        manifest["max_state_bytes"],
+                            manifest["max_state_bytes"],
                         )
             for name, changed in [
                 ("changed text", capture_bytes.replace(b"What changes?", b"What changed?")),
@@ -1782,7 +1782,7 @@ def test_captured_question_order_cannot_be_changed_even_when_all_hashes_are_refr
             qualification.captured_request_bodies(
                 reordered, ["holdout-positive", "holdout-negative"],
                 self.questions, None, manifest["max_request_bytes"],
-            manifest["max_state_bytes"],
+                manifest["max_state_bytes"],
             )
 
         # Runner refusal precedes network traffic and evidence-file creation.
@@ -1949,7 +1949,7 @@ def test_noncanonical_wire_bytes_refuse_even_after_rehashing_all_evidence(self):
                     qualification.captured_request_bodies(
                         altered, ["holdout-positive", "holdout-negative"],
                         self.questions, None, manifest["max_request_bytes"],
-                    manifest["max_state_bytes"],
+                        manifest["max_state_bytes"],
                     )
 
                 with tempfile.TemporaryDirectory() as temp:

From 29d9a2975f8511a57a150231087929450f91f5c2 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 20:02:36 -0800
Subject: [PATCH 40/76] fix(evals): enforce Scout HTTP endpoint in captured
 qualification

Refs #4535

Reject refrozen captured-replay arms routed away from the production
/v1/systemone endpoint or containing query/fragment selectors. Verify
pre-network refusal with falsifiable Bazel-wired regression coverage.

DRAFT ONLY; never merge.
---
 evals/intent_engine/HTTP_RUNNER.md        | 10 ++++++
 evals/intent_engine/run_http.py           | 18 ++++++++++
 evals/intent_engine/test_qualification.py | 44 +++++++++++++++++++++++
 3 files changed, 72 insertions(+)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 8988f6b91cb..9aa8ce15f37 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -30,6 +30,16 @@ the HTTP Authorization header, and never stores the credential or its variable
 name in result evidence. An authenticated remote endpoint must use HTTPS;
 plain HTTP is accepted only for direct loopback addresses. Ambient proxies
 and HTTP redirects are disabled, including for authenticated requests.
+For the captured Scout arm, the endpoint must terminate in the literal
+`/v1/systemone` path, with no query or fragment. The production
+`DecisionEngineClientV1` appends that path to its configured base URL.
+A refrozen endpoint digest alone cannot establish transport parity if a
+different HTTP route accepts the same request bytes. The evaluator checks
+the URL before creating evidence or making any request; a historical
+(non-Scout) evaluation can still use its own endpoint. Scoring only sees
+the URL's hash, not its original text, so this admission check does **not**
+attest the remote server or prove real Scout provenance.
+
 An HTTP response counts as a decision only at status **200**, matching the
 production Scout client. Other successful-looking 2xx statuses (201/202/206)
 refuse before reading or publishing a prediction; an interrupted exact-identity
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 61291ad4029..419915bdf9c 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -151,6 +151,22 @@ def require_http_endpoint(url):
     return endpoint
 
 
+def require_scout_endpoint(url):
+    """The captured arm must use the same endpoint shape as Scout's client.
+
+    DecisionEngineClientV1 appends /v1/systemone to its base URL; allowing
+    an arbitrary route, query or fragment turns byte-exact request replay
+    into a different inference contract despite a refrozen endpoint digest.
+    """
+    endpoint = require_http_endpoint(url)
+    if (not endpoint.path.endswith("/v1/systemone")
+            or endpoint.query or endpoint.fragment):
+        raise ValueError(
+            "captured Scout inference requires canonical /v1/systemone "
+            "endpoint without query or fragment"
+        )
+
+
 def load_bearer_token(name, url):
     """Read an optional bearer credential without persisting it in evidence."""
     if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name):
@@ -575,6 +591,8 @@ def main(argv=None):
                 or hashlib.sha256(args.url.encode("utf-8")).hexdigest() != arm["endpoint_sha256"]
                 or digest(server_config) != arm["server_config_sha256"]):
             raise ValueError("qualification HTTP arm differs from frozen manifest")
+        if arm["config"] == CAPTURED_SCOUT_CONFIG:
+            require_scout_endpoint(args.url)
         if (arm["config"] == CAPTURED_SCOUT_CONFIG) != bool(args.captured_requests):
             raise ValueError("captured Scout arms require --captured-requests and no historical renderer")
         if args.captured_requests:
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 7cade27f2ec..8b7a80c7dc4 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1633,6 +1633,50 @@ def test_captured_scout_accepts_canonical_wire_order_not_settings_vector_order(s
         self.assertEqual(bodies["holdout-positive"], records[0]["body"].encode("utf-8"))
         self.assertEqual(bodies["holdout-negative"], records[1]["body"].encode("utf-8"))
 
+    def test_captured_scout_requires_production_endpoint_even_after_refreezing(self):
+        # A differently routed HTTP service can return valid-looking Noul
+        # predictions, but Scout's configured client never sends requests
+        # there. Rehash the manifest endpoint to exercise the actual parity
+        # guard instead of an incidental digest mismatch.
+        capture, manifest = self.captured_scout_fixture()
+        self.valid(manifest)
+        self.assertIsNone(run_http.require_scout_endpoint(self.endpoint))
+        self.assertIsNone(run_http.require_scout_endpoint(
+            "https://example.test/adapter/v1/systemone"
+        ))
+        variants = (
+            self.endpoint.replace("/v1/systemone", "/v1/predict"),
+            self.endpoint + "?model=other",
+            self.endpoint + "#different",
+            self.endpoint + "/",
+        )
+        for endpoint in variants:
+            with self.subTest(endpoint=endpoint), tempfile.TemporaryDirectory() as temp:
+                here = Path(temp)
+                refrozen = copy.deepcopy(manifest)
+                refrozen["arm"]["endpoint_sha256"] = qualification.sha256_bytes(
+                    endpoint.encode("utf-8")
+                )
+                (here / "manifest.json").write_text(json.dumps(refrozen))
+                (here / "dataset.json").write_bytes(self.dataset_bytes)
+                (here / "captured.json").write_bytes(capture)
+                (here / "server.json").write_text(json.dumps(self.server_config))
+                with patch.object(run_http, "HERE", str(here)), patch.object(
+                    run_http, "post"
+                ) as post:
+                    with self.assertRaisesRegex(
+                        ValueError, "canonical /v1/systemone endpoint"
+                    ):
+                        run_http.main([
+                            "--url", endpoint, "--tag", "captured-endpoint",
+                            "--server-config", str(here / "server.json"),
+                            "--qualification-manifest", str(here / "manifest.json"),
+                            "--captured-requests", str(here / "captured.json"),
+                            qualification.CAPTURED_SCOUT_CONFIG,
+                        ])
+                    post.assert_not_called()
+                self.assertFalse((here / "results").exists())
+
     def test_captured_scout_requests_replay_exact_bytes_and_score(self):
         # This proves a captured arm is byte-exact, not that the synthetic
         # fixture came from real Scout or that its labels were reviewed.

From f6cde284e6a3a3aeb0cddf631c0424381f9bc0a9 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 20:11:23 -0800
Subject: [PATCH 41/76] test(evals): verify the frozen request deadline during
 truncation

---
 evals/intent_engine/test_qualification.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 8b7a80c7dc4..1fb442cb30c 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -229,7 +229,8 @@ def test_native_kev_truncation_is_measured_not_assumed_from_a_flag(self):
             ]
             calls = []
 
-            def native_answer(url, body):
+            def native_answer(url, body, *, timeout):
+                self.assertEqual(timeout, self.manifest["arm"]["timeout_ms"] / 1000)
                 index = len(calls)
                 calls.append(body)
                 case = self.dataset[index + 2]

From 79fd6aaa31d9a17b3167fddfabf93cffe0adbf93 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 21:20:38 -0800
Subject: [PATCH 42/76] feat(scout): retain decisions without delivered hints

Record content-free window, settings, prediction, fact and selection
receipts for every completed intent pass, including no-candidate paths.
Preserve missing model answers as unknown and test negative provenance.

Refs #4535
---
 .../src/agents/context_scout/intent_pass.rs   | 253 ++++++++++++++++++
 1 file changed, 253 insertions(+)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
index d229ba787d1..7766aa5df06 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
@@ -438,6 +438,20 @@ pub async fn run_context_scout_intent_pass(
         action_fact_score_permille: settings.action_fact_score_permille,
         delivered: &session.delivered_intents,
     });
+    // Preserve every assessed window, including below-threshold and no-signal
+    // decisions that never produce a durable suggestion. This is a content-free
+    // provenance receipt, not a correctness label or a second transcript store.
+    record_intent_decision(
+        &pass,
+        settings,
+        &provider_id,
+        &window,
+        &engine_outcome,
+        &facts,
+        &prompt_inferences,
+        &decision,
+    )
+    .await;
     let selected = match decision {
         IntentDecisionV1::Candidates(candidates) if !candidates.is_empty() => candidates[0],
         IntentDecisionV1::Candidates(_)
@@ -1022,6 +1036,139 @@ async fn record_withheld(
     }
 }
 
+/// Records the outcome of every completed intent assessment, not just hints
+/// that survived selection and delivery. The transcript stays in its owning
+/// session store; the bounded analytics receipt can be used to locate and
+/// independently rehydrate the exact window for qualification.
+async fn record_intent_decision(
+    pass: &ContextScoutIntentPassV1<'_>,
+    settings: &ContextScoutIntentSettingsV1,
+    provider_id: &ProviderId,
+    window: &SessionWindowV1,
+    engine_outcome: &EngineOutcomeV1,
+    facts: &[ContextScoutIntentKindV1],
+    prompt_inferences: &[ContextScoutIntentKindV1],
+    decision: &IntentDecisionV1,
+) {
+    let Some(analytics) = pass.analytics.as_ref() else {
+        return;
+    };
+    let Some(envelope_id) = intent_envelope_id(pass.hook) else {
+        return;
+    };
+    let Ok(settings_digest) = settings.settings_digest() else {
+        return;
+    };
+    let engine_identity = pass.engine.as_ref().map(|engine| engine.identity());
+    let Some(metadata_json) = intent_decision_metadata(
+        window,
+        &settings_digest,
+        &settings.questions,
+        engine_outcome,
+        engine_identity.as_ref(),
+        facts,
+        prompt_inferences,
+        decision,
+    ) else {
+        return;
+    };
+    let event = AnalyticsEventInsert {
+        provider: intent_analytics_provider(provider_id),
+        project_id: analytics.project_key.to_owned(),
+        session_id: Some(pass.lifecycle.session_id.as_str().to_owned()),
+        timestamp: pass.observed_at.0.div_euclid(1_000_000),
+        event_kind: "intent_decision".to_owned(),
+        hook_name: None,
+        tool_name: None,
+        tool_category: None,
+        skill_name: None,
+        hint_category: None,
+        // Same admitted boundary identity as the later delivery/withholding.
+        hint_id: Some(hex::encode(envelope_id)),
+        outcome: None,
+        metadata_json: Some(metadata_json),
+    };
+    if let Err(error) = analytics.analytics.append_analytics_events(&[event]).await {
+        tracing::debug!(%error, "Context Scout intent decision receipt not recorded");
+    }
+}
+
+fn intent_decision_metadata(
+    window: &SessionWindowV1,
+    settings_digest: &ManifestDigest,
+    questions: &[tracedecay_domain::configuration::IntentQuestionSettingsV1],
+    engine_outcome: &EngineOutcomeV1,
+    engine_identity: Option<&super::decision_engine::DecisionEngineIdentityV1>,
+    facts: &[ContextScoutIntentKindV1],
+    prompt_inferences: &[ContextScoutIntentKindV1],
+    decision: &IntentDecisionV1,
+) -> Option<String> {
+    if window.messages.is_empty()
+        || window.first_store_id <= 0
+        || window.last_store_id < window.first_store_id
+    {
+        return None;
+    }
+    let window_digest = ManifestDigest::from_sha256_bytes(&window.window_digest).ok()?;
+    // An unavailable engine is not a negative answer. Preserve null rather
+    // than inventing ten zero probabilities for a missing response.
+    let answers = engine_outcome.answers().map(|reported| {
+        questions
+            .iter()
+            .filter_map(|question| {
+                let probability = reported
+                    .iter()
+                    .find(|(intent, _)| *intent == question.intent)?
+                    .1;
+                probability.is_finite().then(|| ContextScoutIntentAnswerV1 {
+                    intent: question.intent,
+                    probability_permille: probability_permille(probability),
+                    threshold_permille: question.threshold_permille,
+                })
+            })
+            .collect::<Vec<_>>()
+    });
+    let selection = match decision {
+        IntentDecisionV1::Candidates(candidates) => serde_json::json!({
+            "kind": "candidates",
+            "candidates": candidates.iter().map(|candidate| serde_json::json!({
+                "intent": candidate.intent,
+                "source": candidate.source,
+                "relevance": candidate.relevance,
+            })).collect::<Vec<_>>(),
+        }),
+        IntentDecisionV1::BelowThreshold { best } => serde_json::json!({
+            "kind": "below_threshold",
+            "best_intent": best.intent,
+            "best_probability_permille": best.probability_permille,
+        }),
+        IntentDecisionV1::NoSignal => serde_json::json!({"kind": "no_signal"}),
+    };
+    Some(serde_json::json!({
+        "source": "context_scout_intent_pass",
+        "route": "decision_engine",
+        "window": {
+            "first_store_id": window.first_store_id,
+            "last_store_id": window.last_store_id,
+            "message_count": window.messages.len(),
+            "window_digest": window_digest,
+            "newest_timestamp_micros": window.newest_timestamp_micros,
+        },
+        "settings_digest": settings_digest,
+        "model_outcome": engine_outcome.model_outcome(),
+        "engine": engine_identity.map(|identity| serde_json::json!({
+            "base_url_digest": hex::encode(identity.base_url_digest),
+            "configured_model": identity.model,
+            "reported_model": engine_outcome.reported_model(),
+        })),
+        "answers": answers,
+        "facts": facts,
+        "prompt_inferences": prompt_inferences,
+        "decision": selection,
+    })
+    .to_string())
+}
+
 fn receipt_model_is_valid(model: &str) -> bool {
     !model.trim().is_empty()
         && model.len() <= MAX_DECISION_ENGINE_MODEL_BYTES
@@ -1079,6 +1226,112 @@ mod tool_signal_tests {
         }
     }
 
+    #[test]
+    fn completed_intent_decisions_record_negative_windows_without_transcript_text() {
+        use std::collections::BTreeMap;
+
+        use tracedecay_domain::ManifestDigest;
+
+        use super::super::decision_engine::{
+            DecisionAnswerV1, DecisionAnswersV1, DecisionEngineOutcomeV1,
+        };
+        use super::{EngineOutcomeV1, intent_decision_metadata};
+
+        let window = SessionWindowV1 {
+            messages: vec![message(
+                SessionWindowRoleV1::User,
+                "Confidential transcript text must remain in its owning store",
+                None,
+            )],
+            first_store_id: 81,
+            last_store_id: 81,
+            newest_timestamp_micros: Some(1_234_000_000),
+            window_digest: [7; 32],
+            freshness: SessionWindowFreshnessV1::Unknown,
+        };
+        let settings_digest = ManifestDigest::from_sha256_bytes(&[8; 32]).unwrap();
+        let questions = [IntentQuestionSettingsV1 {
+            intent: Kind::Impact,
+            instructions: "Does the request ask about impact?".to_owned(),
+            threshold_permille: 600,
+            margin_permille: None,
+        }];
+        let engine = EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Answered(
+            DecisionAnswersV1 {
+                answers: BTreeMap::from([(
+                    Kind::Impact.as_key().to_owned(),
+                    DecisionAnswerV1::Noul(0.17),
+                )]),
+                model: Some("kev-4b".to_owned()),
+                latency_micros: 1250,
+            },
+        ));
+        let negative = IntentDecisionV1::BelowThreshold {
+            best: IntentBestAnswerV1 {
+                intent: Kind::Impact,
+                probability_permille: 170,
+            },
+        };
+        let evidence = intent_decision_metadata(
+            &window,
+            &settings_digest,
+            &questions,
+            &engine,
+            None,
+            &[],
+            &[Kind::Impact],
+            &negative,
+        )
+        .unwrap();
+        let receipt: serde_json::Value = serde_json::from_str(&evidence).unwrap();
+        assert_eq!(receipt["window"]["first_store_id"], 81);
+        assert_eq!(receipt["window"]["last_store_id"], 81);
+        assert_eq!(receipt["window"]["message_count"], 1);
+        assert!(receipt["window"]["window_digest"].is_string());
+        assert_eq!(receipt["decision"]["kind"], "below_threshold");
+        assert_eq!(receipt["decision"]["best_probability_permille"], 170);
+        assert_eq!(receipt["answers"][0]["probability_permille"], 170);
+        assert_eq!(receipt["answers"][0]["threshold_permille"], 600);
+        assert!(receipt["facts"].as_array().unwrap().is_empty());
+        assert_eq!(receipt["prompt_inferences"].as_array().unwrap().len(), 1);
+        assert!(!evidence.contains("Confidential transcript text"));
+        assert!(!evidence.contains("Does the request ask about impact?"));
+
+        // A transport failure is unknown, never relabeled as a negative model
+        // prediction, even when deterministic policy finds no signal.
+        let unavailable = intent_decision_metadata(
+            &window,
+            &settings_digest,
+            &questions,
+            &EngineOutcomeV1::Unavailable,
+            None,
+            &[],
+            &[],
+            &IntentDecisionV1::NoSignal,
+        )
+        .unwrap();
+        let unknown: serde_json::Value = serde_json::from_str(&unavailable).unwrap();
+        assert_eq!(unknown["decision"]["kind"], "no_signal");
+        assert!(unknown["answers"].is_null());
+        assert!(unknown["engine"].is_null());
+
+        let bad_window = SessionWindowV1 {
+            messages: Vec::new(),
+            ..window
+        };
+        assert!(intent_decision_metadata(
+            &bad_window,
+            &settings_digest,
+            &questions,
+            &engine,
+            None,
+            &[],
+            &[],
+            &negative,
+        )
+        .is_none());
+    }
+
     #[test]
     fn tool_result_claims_cannot_override_a_negative_model_decision() {
         let prompt = message(

From e9498a26a6ae8a3328a27402b5c33075478e9028 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 21:21:49 -0800
Subject: [PATCH 43/76] refactor(scout): group decision receipt evidence

Keep the content-free recording call and metadata builder narrow for
Clippy and make the original window and decision explicit inputs.

Refs #4535
---
 .../src/agents/context_scout/intent_pass.rs   | 83 +++++++++++--------
 1 file changed, 47 insertions(+), 36 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
index 7766aa5df06..12301178249 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
@@ -445,11 +445,13 @@ pub async fn run_context_scout_intent_pass(
         &pass,
         settings,
         &provider_id,
-        &window,
-        &engine_outcome,
-        &facts,
-        &prompt_inferences,
-        &decision,
+        IntentDecisionEvidenceV1 {
+            window: &window,
+            engine_outcome: &engine_outcome,
+            facts: &facts,
+            prompt_inferences: &prompt_inferences,
+            decision: &decision,
+        },
     )
     .await;
     let selected = match decision {
@@ -1040,15 +1042,19 @@ async fn record_withheld(
 /// that survived selection and delivery. The transcript stays in its owning
 /// session store; the bounded analytics receipt can be used to locate and
 /// independently rehydrate the exact window for qualification.
+struct IntentDecisionEvidenceV1<'a> {
+    window: &'a SessionWindowV1,
+    engine_outcome: &'a EngineOutcomeV1,
+    facts: &'a [ContextScoutIntentKindV1],
+    prompt_inferences: &'a [ContextScoutIntentKindV1],
+    decision: &'a IntentDecisionV1,
+}
+
 async fn record_intent_decision(
     pass: &ContextScoutIntentPassV1<'_>,
     settings: &ContextScoutIntentSettingsV1,
     provider_id: &ProviderId,
-    window: &SessionWindowV1,
-    engine_outcome: &EngineOutcomeV1,
-    facts: &[ContextScoutIntentKindV1],
-    prompt_inferences: &[ContextScoutIntentKindV1],
-    decision: &IntentDecisionV1,
+    evidence: IntentDecisionEvidenceV1<'_>,
 ) {
     let Some(analytics) = pass.analytics.as_ref() else {
         return;
@@ -1061,14 +1067,10 @@ async fn record_intent_decision(
     };
     let engine_identity = pass.engine.as_ref().map(|engine| engine.identity());
     let Some(metadata_json) = intent_decision_metadata(
-        window,
+        &evidence,
         &settings_digest,
         &settings.questions,
-        engine_outcome,
         engine_identity.as_ref(),
-        facts,
-        prompt_inferences,
-        decision,
     ) else {
         return;
     };
@@ -1094,15 +1096,18 @@ async fn record_intent_decision(
 }
 
 fn intent_decision_metadata(
-    window: &SessionWindowV1,
+    evidence: &IntentDecisionEvidenceV1<'_>,
     settings_digest: &ManifestDigest,
     questions: &[tracedecay_domain::configuration::IntentQuestionSettingsV1],
-    engine_outcome: &EngineOutcomeV1,
     engine_identity: Option<&super::decision_engine::DecisionEngineIdentityV1>,
-    facts: &[ContextScoutIntentKindV1],
-    prompt_inferences: &[ContextScoutIntentKindV1],
-    decision: &IntentDecisionV1,
 ) -> Option<String> {
+    let IntentDecisionEvidenceV1 {
+        window,
+        engine_outcome,
+        facts,
+        prompt_inferences,
+        decision,
+    } = evidence;
     if window.messages.is_empty()
         || window.first_store_id <= 0
         || window.last_store_id < window.first_store_id
@@ -1235,7 +1240,7 @@ mod tool_signal_tests {
         use super::super::decision_engine::{
             DecisionAnswerV1, DecisionAnswersV1, DecisionEngineOutcomeV1,
         };
-        use super::{EngineOutcomeV1, intent_decision_metadata};
+        use super::{EngineOutcomeV1, IntentDecisionEvidenceV1, intent_decision_metadata};
 
         let window = SessionWindowV1 {
             messages: vec![message(
@@ -1273,14 +1278,16 @@ mod tool_signal_tests {
             },
         };
         let evidence = intent_decision_metadata(
-            &window,
+            &IntentDecisionEvidenceV1 {
+                window: &window,
+                engine_outcome: &engine,
+                facts: &[],
+                prompt_inferences: &[Kind::Impact],
+                decision: &negative,
+            },
             &settings_digest,
             &questions,
-            &engine,
             None,
-            &[],
-            &[Kind::Impact],
-            &negative,
         )
         .unwrap();
         let receipt: serde_json::Value = serde_json::from_str(&evidence).unwrap();
@@ -1300,14 +1307,16 @@ mod tool_signal_tests {
         // A transport failure is unknown, never relabeled as a negative model
         // prediction, even when deterministic policy finds no signal.
         let unavailable = intent_decision_metadata(
-            &window,
+            &IntentDecisionEvidenceV1 {
+                window: &window,
+                engine_outcome: &EngineOutcomeV1::Unavailable,
+                facts: &[],
+                prompt_inferences: &[],
+                decision: &IntentDecisionV1::NoSignal,
+            },
             &settings_digest,
             &questions,
-            &EngineOutcomeV1::Unavailable,
             None,
-            &[],
-            &[],
-            &IntentDecisionV1::NoSignal,
         )
         .unwrap();
         let unknown: serde_json::Value = serde_json::from_str(&unavailable).unwrap();
@@ -1320,14 +1329,16 @@ mod tool_signal_tests {
             ..window
         };
         assert!(intent_decision_metadata(
-            &bad_window,
+            &IntentDecisionEvidenceV1 {
+                window: &bad_window,
+                engine_outcome: &engine,
+                facts: &[],
+                prompt_inferences: &[],
+                decision: &negative,
+            },
             &settings_digest,
             &questions,
-            &engine,
             None,
-            &[],
-            &[],
-            &negative,
         )
         .is_none());
     }

From c83989e8c0dccf7003813d2fc272d1f68196d401 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 21:23:10 -0800
Subject: [PATCH 44/76] docs(evals): identify live Scout negative decision
 receipts

Document the existing analytics authority and the remaining owning-window
rehydration and independent-label qualification boundary.

Refs #4535
---
 evals/intent_engine/HTTP_RUNNER.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 9aa8ce15f37..bb81a1efc8c 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -310,6 +310,20 @@ settings and owning transcript authority before making any application
 replacement decision. The runner itself does not export canonical Scout data.
 
 ## Captured Scout request arm (qualified only)
+The live intent pass also records a content-free `intent_decision` analytics
+event for **every completed assessment**, including below-threshold and
+no-signal results that produce no delivered hint. The event keeps the owning
+session/project/provider and boundary identity, original store-ID bounds and
+window digest, settings digest, typed model outcome, Noul probabilities,
+observed action facts, prompt inferences, and the policy selection separately.
+An unavailable model has **null answers**, not fabricated negative labels.
+The transcript body stays in the owning session store. These events are
+provenance pointers, not independent correctness labels or proof that the
+session text still exists: analytics can also be unavailable, and the missing
+bounded exporter must rehydrate the original owning window and independently
+verify its digest before claiming a canonical Scout capture. No such
+rehydration or real Kev experiment is claimed by this event alone.
+
 
 The historical `A/B/B2` renderers do **not** reproduce production Scout
 requests. A separately captured, byte-exact request file can now be evaluated

From 49993c9f04057f34e58bc20fd1f8769742e5c335 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 21:24:24 -0800
Subject: [PATCH 45/76] fix(scout): keep full prediction and latency in
 decision receipts

Preserve unrounded model probabilities beside policy permille output and
measured request latency, including nulls for unavailable engines.

Refs #4535
---
 .../src/agents/context_scout/intent_pass.rs   | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
index 12301178249..6cfe04aed69 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
@@ -1133,6 +1133,21 @@ fn intent_decision_metadata(
             })
             .collect::<Vec<_>>()
     });
+    // Keep full server probabilities distinct from the policy's rounded
+    // permille receipts. Rounding is appropriate for threshold comparison,
+    // but discards calibration and near-threshold evidence.
+    let raw_model_probabilities = engine_outcome.answers().map(|values| {
+        values
+            .into_iter()
+            .map(|(intent, probability)| (intent.as_key().to_owned(), probability))
+            .collect::<std::collections::BTreeMap<_, _>>()
+    });
+    let request_latency_micros = match engine_outcome {
+        EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Answered(answers)) => {
+            Some(answers.latency_micros)
+        }
+        _ => None,
+    };
     let selection = match decision {
         IntentDecisionV1::Candidates(candidates) => serde_json::json!({
             "kind": "candidates",
@@ -1167,6 +1182,8 @@ fn intent_decision_metadata(
             "reported_model": engine_outcome.reported_model(),
         })),
         "answers": answers,
+        "raw_model_probabilities": raw_model_probabilities,
+        "request_latency_micros": request_latency_micros,
         "facts": facts,
         "prompt_inferences": prompt_inferences,
         "decision": selection,
@@ -1299,6 +1316,8 @@ mod tool_signal_tests {
         assert_eq!(receipt["decision"]["best_probability_permille"], 170);
         assert_eq!(receipt["answers"][0]["probability_permille"], 170);
         assert_eq!(receipt["answers"][0]["threshold_permille"], 600);
+        assert_eq!(receipt["raw_model_probabilities"][Kind::Impact.as_key()], 0.17);
+        assert_eq!(receipt["request_latency_micros"], 1250);
         assert!(receipt["facts"].as_array().unwrap().is_empty());
         assert_eq!(receipt["prompt_inferences"].as_array().unwrap().len(), 1);
         assert!(!evidence.contains("Confidential transcript text"));
@@ -1322,6 +1341,8 @@ mod tool_signal_tests {
         let unknown: serde_json::Value = serde_json::from_str(&unavailable).unwrap();
         assert_eq!(unknown["decision"]["kind"], "no_signal");
         assert!(unknown["answers"].is_null());
+        assert!(unknown["raw_model_probabilities"].is_null());
+        assert!(unknown["request_latency_micros"].is_null());
         assert!(unknown["engine"].is_null());
 
         let bad_window = SessionWindowV1 {

From c4f38a113b85465a1e6f60271d9e13389adece57 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 22:02:29 -0800
Subject: [PATCH 46/76] style(scout): format decision receipt and latency
 checks

---
 .../src/agents/context_scout/intent_pass.rs   | 103 +++++++++---------
 1 file changed, 54 insertions(+), 49 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
index 6cfe04aed69..acfa98226d0 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
@@ -1164,31 +1164,33 @@ fn intent_decision_metadata(
         }),
         IntentDecisionV1::NoSignal => serde_json::json!({"kind": "no_signal"}),
     };
-    Some(serde_json::json!({
-        "source": "context_scout_intent_pass",
-        "route": "decision_engine",
-        "window": {
-            "first_store_id": window.first_store_id,
-            "last_store_id": window.last_store_id,
-            "message_count": window.messages.len(),
-            "window_digest": window_digest,
-            "newest_timestamp_micros": window.newest_timestamp_micros,
-        },
-        "settings_digest": settings_digest,
-        "model_outcome": engine_outcome.model_outcome(),
-        "engine": engine_identity.map(|identity| serde_json::json!({
-            "base_url_digest": hex::encode(identity.base_url_digest),
-            "configured_model": identity.model,
-            "reported_model": engine_outcome.reported_model(),
-        })),
-        "answers": answers,
-        "raw_model_probabilities": raw_model_probabilities,
-        "request_latency_micros": request_latency_micros,
-        "facts": facts,
-        "prompt_inferences": prompt_inferences,
-        "decision": selection,
-    })
-    .to_string())
+    Some(
+        serde_json::json!({
+            "source": "context_scout_intent_pass",
+            "route": "decision_engine",
+            "window": {
+                "first_store_id": window.first_store_id,
+                "last_store_id": window.last_store_id,
+                "message_count": window.messages.len(),
+                "window_digest": window_digest,
+                "newest_timestamp_micros": window.newest_timestamp_micros,
+            },
+            "settings_digest": settings_digest,
+            "model_outcome": engine_outcome.model_outcome(),
+            "engine": engine_identity.map(|identity| serde_json::json!({
+                "base_url_digest": hex::encode(identity.base_url_digest),
+                "configured_model": identity.model,
+                "reported_model": engine_outcome.reported_model(),
+            })),
+            "answers": answers,
+            "raw_model_probabilities": raw_model_probabilities,
+            "request_latency_micros": request_latency_micros,
+            "facts": facts,
+            "prompt_inferences": prompt_inferences,
+            "decision": selection,
+        })
+        .to_string(),
+    )
 }
 
 fn receipt_model_is_valid(model: &str) -> bool {
@@ -1278,16 +1280,14 @@ mod tool_signal_tests {
             threshold_permille: 600,
             margin_permille: None,
         }];
-        let engine = EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Answered(
-            DecisionAnswersV1 {
-                answers: BTreeMap::from([(
-                    Kind::Impact.as_key().to_owned(),
-                    DecisionAnswerV1::Noul(0.17),
-                )]),
-                model: Some("kev-4b".to_owned()),
-                latency_micros: 1250,
-            },
-        ));
+        let engine = EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Answered(DecisionAnswersV1 {
+            answers: BTreeMap::from([(
+                Kind::Impact.as_key().to_owned(),
+                DecisionAnswerV1::Noul(0.17),
+            )]),
+            model: Some("kev-4b".to_owned()),
+            latency_micros: 1250,
+        }));
         let negative = IntentDecisionV1::BelowThreshold {
             best: IntentBestAnswerV1 {
                 intent: Kind::Impact,
@@ -1316,7 +1316,10 @@ mod tool_signal_tests {
         assert_eq!(receipt["decision"]["best_probability_permille"], 170);
         assert_eq!(receipt["answers"][0]["probability_permille"], 170);
         assert_eq!(receipt["answers"][0]["threshold_permille"], 600);
-        assert_eq!(receipt["raw_model_probabilities"][Kind::Impact.as_key()], 0.17);
+        assert_eq!(
+            receipt["raw_model_probabilities"][Kind::Impact.as_key()],
+            0.17
+        );
         assert_eq!(receipt["request_latency_micros"], 1250);
         assert!(receipt["facts"].as_array().unwrap().is_empty());
         assert_eq!(receipt["prompt_inferences"].as_array().unwrap().len(), 1);
@@ -1349,19 +1352,21 @@ mod tool_signal_tests {
             messages: Vec::new(),
             ..window
         };
-        assert!(intent_decision_metadata(
-            &IntentDecisionEvidenceV1 {
-                window: &bad_window,
-                engine_outcome: &engine,
-                facts: &[],
-                prompt_inferences: &[],
-                decision: &negative,
-            },
-            &settings_digest,
-            &questions,
-            None,
-        )
-        .is_none());
+        assert!(
+            intent_decision_metadata(
+                &IntentDecisionEvidenceV1 {
+                    window: &bad_window,
+                    engine_outcome: &engine,
+                    facts: &[],
+                    prompt_inferences: &[],
+                    decision: &negative,
+                },
+                &settings_digest,
+                &questions,
+                None,
+            )
+            .is_none()
+        );
     }
 
     #[test]

From 0020f71fb834fcb63a0d32cd1e1817ab24264646 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 22:08:46 -0800
Subject: [PATCH 47/76] fix(scout): bound production decision response reads

---
 .../agents/context_scout/decision_engine.rs   | 118 +++++++++++++++---
 1 file changed, 103 insertions(+), 15 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index b8ba4c8b0e5..02fa7a566bf 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -21,6 +21,7 @@
 //! fall back to its non-model path.
 
 use std::collections::BTreeMap;
+use std::io::Read as _;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::{Duration, Instant};
@@ -33,6 +34,9 @@ use tracedecay_runtime_core::native_runtime;
 /// Path of the typed-question endpoint under the configured base URL.
 const SYSTEMONE_PATH: &str = "/v1/systemone";
 
+/// Bound decoded HTTP answer bytes before JSON parsing, matching qualified replay.
+const MAX_DECISION_RESPONSE_BYTES: usize = 1024 * 1024;
+
 /// Tolerance when checking a `score` against the rubric's last level index.
 const SCORE_EPSILON: f64 = 1e-6;
 
@@ -286,16 +290,33 @@ impl DecisionEngineClientV1 {
             tracing::debug!(status, "decision engine refused");
             return DecisionEngineOutcomeV1::Refused { status };
         }
-        let value = match response.body_mut().read_json::<Value>() {
+        // The server is independently configured and may return an unlimited
+        // response, including one without Content-Length. Never deserialize
+        // unbounded bytes from a transcript-bearing decision exchange.
+        let mut answer_bytes = Vec::new();
+        if let Err(error) = response
+            .body_mut()
+            .as_reader()
+            .take((MAX_DECISION_RESPONSE_BYTES + 1) as u64)
+            .read_to_end(&mut answer_bytes)
+        {
+            return if is_io_timeout(&error) {
+                tracing::debug!(%error, "decision engine timed out reading its answer");
+                DecisionEngineOutcomeV1::TimedOut
+            } else {
+                tracing::debug!(%error, "decision engine answer unreadable");
+                DecisionEngineOutcomeV1::Malformed
+            };
+        }
+        if answer_bytes.len() > MAX_DECISION_RESPONSE_BYTES {
+            tracing::debug!("decision engine answer exceeds the byte limit");
+            return DecisionEngineOutcomeV1::Malformed;
+        }
+        let value = match serde_json::from_slice::<Value>(&answer_bytes) {
             Ok(value) => value,
             Err(error) => {
-                return if is_timeout(&error) {
-                    tracing::debug!(%error, "decision engine timed out reading its answer");
-                    DecisionEngineOutcomeV1::TimedOut
-                } else {
-                    tracing::debug!(%error, "decision engine answer unreadable");
-                    DecisionEngineOutcomeV1::Malformed
-                };
+                tracing::debug!(%error, "decision engine answer is not JSON");
+                return DecisionEngineOutcomeV1::Malformed;
             }
         };
         let latency_micros = u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX);
@@ -411,16 +432,18 @@ fn transport_failure(error: &ureq::Error) -> DecisionEngineOutcomeV1 {
 fn is_timeout(error: &ureq::Error) -> bool {
     match error {
         ureq::Error::Timeout(_) => true,
-        ureq::Error::Io(io) => {
-            matches!(
-                io.kind(),
-                std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock
-            ) || io.to_string().to_ascii_lowercase().contains("timed out")
-        }
+        ureq::Error::Io(io) => is_io_timeout(io),
         other => other.to_string().to_ascii_lowercase().contains("timeout"),
     }
 }
 
+fn is_io_timeout(error: &std::io::Error) -> bool {
+    matches!(
+        error.kind(),
+        std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock
+    ) || error.to_string().to_ascii_lowercase().contains("timed out")
+}
+
 type ParsedAnswers = (BTreeMap<String, DecisionAnswerV1>, Option<String>);
 
 /// Every requested question must be answered with the requested kind; extra
@@ -660,7 +683,7 @@ pub mod test_server {
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
 mod tests {
-    use std::io::Read as _;
+    use std::io::{BufRead as _, BufReader, Read as _, Write as _};
     use std::time::{Duration, Instant};
 
     use super::test_server::{
@@ -1131,6 +1154,71 @@ mod tests {
         assert_eq!(sent["state"], "9 newest");
     }
 
+    #[test]
+    fn response_byte_limit_accepts_exact_boundary_and_refuses_one_extra_byte() {
+        let valid = single_noul_body("q", 0.5);
+        assert!(valid.len() < MAX_DECISION_RESPONSE_BYTES);
+        let exact = format!(
+            "{valid}{}",
+            " ".repeat(MAX_DECISION_RESPONSE_BYTES - valid.len())
+        );
+        assert_eq!(exact.len(), MAX_DECISION_RESPONSE_BYTES);
+        let accepted = fake_decision_engine(200, &exact, Duration::ZERO);
+        let result = client(&accepted.url).ask_blocking(&request("state", vec![noul("q")]));
+        assert_eq!(answered(result).answers["q"], DecisionAnswerV1::Noul(0.5));
+        assert_eq!(accepted.requests.load(Ordering::SeqCst), 1);
+
+        let oversized = format!("{exact} ");
+        assert_eq!(oversized.len(), MAX_DECISION_RESPONSE_BYTES + 1);
+        let rejected = fake_decision_engine(200, &oversized, Duration::ZERO);
+        assert_eq!(
+            client(&rejected.url).ask_blocking(&request("state", vec![noul("q")])),
+            DecisionEngineOutcomeV1::Malformed
+        );
+        assert_eq!(rejected.requests.load(Ordering::SeqCst), 1);
+    }
+
+    #[test]
+    fn unknown_length_response_is_bounded_without_waiting_for_eof() {
+        let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).unwrap();
+        let url = format!("http://{}", listener.local_addr().unwrap());
+        let sender = std::thread::spawn(move || {
+            let (mut socket, _) = listener.accept().unwrap();
+            let mut reader = BufReader::new(socket.try_clone().unwrap());
+            let mut length = 0;
+            loop {
+                let mut line = String::new();
+                if reader.read_line(&mut line).unwrap_or(0) == 0 || line == "\r\n" {
+                    break;
+                }
+                if let Some(value) = line
+                    .to_ascii_lowercase()
+                    .strip_prefix("content-length:")
+                {
+                    length = value.trim().parse().unwrap_or(0);
+                }
+            }
+            let mut body = vec![0; length];
+            reader.read_exact(&mut body).unwrap();
+            socket
+                .write_all(
+                    b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n",
+                )
+                .unwrap();
+            // Without a declared length or EOF, an unbounded parser would
+            // wait for the server and return TimedOut rather than Malformed.
+            let _ = socket.write_all(&vec![b' '; MAX_DECISION_RESPONSE_BYTES + 1]);
+            std::thread::sleep(Duration::from_millis(1500));
+        });
+        let mut settings = config(&url);
+        settings.timeout = Duration::from_secs(1);
+        let result = DecisionEngineClientV1::new(settings)
+            .unwrap()
+            .ask_blocking(&request("state", vec![noul("q")]));
+        assert_eq!(result, DecisionEngineOutcomeV1::Malformed);
+        sender.join().unwrap();
+    }
+
     #[test]
     fn unusable_answers_are_malformed_or_refused() {
         let cases: Vec<(u16, String, DecisionEngineOutcomeV1)> = vec![

From b0af1649145c11715514ac201d31b255bab1d75a Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 22:22:55 -0800
Subject: [PATCH 48/76] style(scout): format bounded response fixture

---
 .../src/agents/context_scout/decision_engine.rs              | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index 02fa7a566bf..8c81ffd7daa 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -1191,10 +1191,7 @@ mod tests {
                 if reader.read_line(&mut line).unwrap_or(0) == 0 || line == "\r\n" {
                     break;
                 }
-                if let Some(value) = line
-                    .to_ascii_lowercase()
-                    .strip_prefix("content-length:")
-                {
+                if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") {
                     length = value.trim().parse().unwrap_or(0);
                 }
             }

From eabbdc6e604cb8d3348e653b3df7a90cee355eba Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 22:39:01 -0800
Subject: [PATCH 49/76] fix(scout): remove redundant response test import

---
 .../src/agents/context_scout/decision_engine.rs                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index 8c81ffd7daa..1fd4a85807f 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -683,7 +683,7 @@ pub mod test_server {
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
 mod tests {
-    use std::io::{BufRead as _, BufReader, Read as _, Write as _};
+    use std::io::{BufRead as _, BufReader, Write as _};
     use std::time::{Duration, Instant};
 
     use super::test_server::{

From da6e31743ece0478f17cd753bed883c5cda42d38 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Sun, 8 Nov 2026 23:14:43 -0800
Subject: [PATCH 50/76] feat(scout): record prepared HTTP request commitments

Refs #4535. Keep draft; never merge.
---
 .../agents/context_scout/decision_engine.rs   | 77 +++++++++++++++++++
 .../src/agents/context_scout/intent_pass.rs   | 51 ++++++++----
 2 files changed, 115 insertions(+), 13 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index 1fd4a85807f..0484ca4a8bc 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -86,6 +86,14 @@ pub enum DecisionAnswerV1 {
     Score(f64),
 }
 
+/// Byte-exact identity of the request prepared by the configured Rust client.
+/// Carries no transcript content, only the bytes and digest of the JSON body.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct DecisionRequestCommitmentV1 {
+    pub wire_sha256: [u8; 32],
+    pub wire_bytes: usize,
+}
+
 /// Every answer of one successful exchange.
 #[derive(Debug, Clone, PartialEq)]
 pub struct DecisionAnswersV1 {
@@ -159,6 +167,14 @@ pub struct DecisionEngineIdentityV1 {
 pub trait DecisionEnginePortV1: Send + Sync {
     fn ask_blocking(&self, request: &DecisionRequestV1) -> DecisionEngineOutcomeV1;
     fn identity(&self) -> DecisionEngineIdentityV1;
+
+    /// Test doubles cannot claim a production HTTP wire commitment.
+    fn request_commitment(
+        &self,
+        _request: &DecisionRequestV1,
+    ) -> Option<DecisionRequestCommitmentV1> {
+        None
+    }
 }
 
 /// HTTP client for the decision-model API.
@@ -244,6 +260,23 @@ impl DecisionEngineClientV1 {
         self.identity.clone()
     }
 
+    /// Prepare the same serde_json body used by `send_json` without
+    /// retaining the model-visible state or making a second HTTP request.
+    pub fn request_commitment(
+        &self,
+        request: &DecisionRequestV1,
+    ) -> Option<DecisionRequestCommitmentV1> {
+        if !request_is_well_formed(request) {
+            return None;
+        }
+        let body = request_body(request, self.identity.model.as_deref(), self.max_state_bytes);
+        let wire = serde_json::to_vec(&body).ok()?;
+        Some(DecisionRequestCommitmentV1 {
+            wire_sha256: Sha256::digest(&wire).into(),
+            wire_bytes: wire.len(),
+        })
+    }
+
     fn try_acquire(&self) -> Option<Permit> {
         let mut current = self.in_flight.load(Ordering::Acquire);
         loop {
@@ -355,6 +388,13 @@ impl DecisionEnginePortV1 for DecisionEngineClientV1 {
     fn identity(&self) -> DecisionEngineIdentityV1 {
         Self::identity(self)
     }
+
+    fn request_commitment(
+        &self,
+        request: &DecisionRequestV1,
+    ) -> Option<DecisionRequestCommitmentV1> {
+        Self::request_commitment(self, request)
+    }
 }
 
 fn request_is_well_formed(request: &DecisionRequestV1) -> bool {
@@ -809,6 +849,43 @@ mod tests {
         );
     }
 
+    #[test]
+    fn the_prepared_request_commitment_matches_the_actual_utf8_wire() {
+        let server = fake_decision_engine(200, &single_noul_body("q", 0.5), Duration::ZERO);
+        let mut configured = config(&server.url);
+        configured.model = Some("kev-4b".to_owned());
+        configured.max_state_bytes = 9;
+        let client = DecisionEngineClientV1::new(configured.clone()).unwrap();
+        let original = request("old private context éé latest", vec![noul("q")]);
+        let commitment = client.request_commitment(&original).unwrap();
+        let _ = answered(client.ask_blocking(&original));
+        let actual = server.last_body.lock().unwrap().clone();
+        assert_eq!(server.requests.load(Ordering::SeqCst), 1);
+        assert_eq!(commitment.wire_bytes, actual.len());
+        assert_eq!(
+            commitment.wire_sha256,
+            <[u8; 32]>::from(Sha256::digest(actual.as_bytes()))
+        );
+        let sent: Value = serde_json::from_str(&actual).unwrap();
+        assert_eq!(sent["model"], "kev-4b");
+        assert_eq!(sent["state"], tail_on_char_boundary(&original.state, 9));
+        assert!(!actual.contains("old private context"));
+
+        configured.model = Some("different-model".to_owned());
+        let changed_model = DecisionEngineClientV1::new(configured).unwrap();
+        assert_ne!(changed_model.request_commitment(&original), Some(commitment));
+        let mut changed_question = original.clone();
+        changed_question.questions[0].instructions.push_str(" Be precise.");
+        assert_ne!(client.request_commitment(&changed_question), Some(commitment));
+        // The prefix intentionally discarded by the configured byte cap
+        // does not affect the sent request, but changed visible text does.
+        let alternate_prefix = request("different private context éé latest", vec![noul("q")]);
+        assert_eq!(client.request_commitment(&alternate_prefix), Some(commitment));
+        let changed_visible = request("old private context éé latest!", vec![noul("q")]);
+        assert_ne!(client.request_commitment(&changed_visible), Some(commitment));
+        assert!(client.request_commitment(&request("state", vec![])).is_none());
+    }
+
     #[test]
     fn a_noul_answer_is_read_as_a_probability() {
         let server = fake_decision_engine(200, &single_noul_body("recall", 0.73), Duration::ZERO);
diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
index acfa98226d0..4606e7d0deb 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
@@ -50,7 +50,7 @@ use super::address_registry::{
 };
 use super::decision_engine::{
     DecisionAnswerV1, DecisionEngineOutcomeV1, DecisionEnginePortV1, DecisionQuestionKindV1,
-    DecisionQuestionV1, DecisionRequestV1,
+    DecisionQuestionV1, DecisionRequestCommitmentV1, DecisionRequestV1,
 };
 use super::evidence::MAX_DECISION_ENGINE_MODEL_BYTES;
 use super::owner::{ContextScoutIntentSessionStateV1, ProjectContextScoutOwnerV1};
@@ -423,9 +423,9 @@ pub async fn run_context_scout_intent_pass(
     // Keep actual tool activity separate from prompt interpretation. Only an
     // unanswered intent may use the latter as its deterministic fallback.
     let (facts, prompt_inferences) = window_intent_signals(&window);
-    let engine_outcome = match ask_engine(&pass, settings, &window).await {
+    let (engine_outcome, request_commitment) = match ask_engine(&pass, settings, &window).await {
         EngineCallV1::Cancelled => return ContextScoutIntentPassOutcomeV1::Cancelled,
-        EngineCallV1::Finished(outcome) => outcome,
+        EngineCallV1::Finished(outcome, commitment) => (outcome, commitment),
     };
     let model_outcome = engine_outcome.model_outcome();
     let answers = engine_outcome.answers();
@@ -448,6 +448,7 @@ pub async fn run_context_scout_intent_pass(
         IntentDecisionEvidenceV1 {
             window: &window,
             engine_outcome: &engine_outcome,
+            request_commitment,
             facts: &facts,
             prompt_inferences: &prompt_inferences,
             decision: &decision,
@@ -668,7 +669,7 @@ async fn read_boundary_window(
 }
 
 enum EngineCallV1 {
-    Finished(EngineOutcomeV1),
+    Finished(EngineOutcomeV1, Option<DecisionRequestCommitmentV1>),
     Cancelled,
 }
 
@@ -736,10 +737,10 @@ async fn ask_engine(
     window: &SessionWindowV1,
 ) -> EngineCallV1 {
     if settings.questions.is_empty() {
-        return EngineCallV1::Finished(EngineOutcomeV1::NotRequested);
+        return EngineCallV1::Finished(EngineOutcomeV1::NotRequested, None);
     }
     let Some(engine) = pass.engine.clone() else {
-        return EngineCallV1::Finished(EngineOutcomeV1::Unavailable);
+        return EngineCallV1::Finished(EngineOutcomeV1::Unavailable, None);
     };
     let request = DecisionRequestV1 {
         state: render_window_state(
@@ -757,17 +758,21 @@ async fn ask_engine(
             })
             .collect(),
     };
+    let request_commitment = engine.request_commitment(&request);
     let asked = native_runtime::spawn_blocking(move || engine.ask_blocking(&request));
     tokio::select! {
         biased;
         () = pass.cancellation.cancelled() => EngineCallV1::Cancelled,
         answered = native_runtime::timeout_at(pass.deadline.instant(), asked) => {
-            EngineCallV1::Finished(match answered {
-                Ok(Ok(outcome)) => EngineOutcomeV1::Asked(outcome),
-                // A lost blocking worker reads as an unreachable engine.
-                Ok(Err(_)) => EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Unreachable),
-                Err(_) => EngineOutcomeV1::DeadlineExceeded,
-            })
+            EngineCallV1::Finished(
+                match answered {
+                    Ok(Ok(outcome)) => EngineOutcomeV1::Asked(outcome),
+                    // A lost blocking worker reads as an unreachable engine.
+                    Ok(Err(_)) => EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Unreachable),
+                    Err(_) => EngineOutcomeV1::DeadlineExceeded,
+                },
+                request_commitment,
+            )
         }
     }
 }
@@ -1045,6 +1050,7 @@ async fn record_withheld(
 struct IntentDecisionEvidenceV1<'a> {
     window: &'a SessionWindowV1,
     engine_outcome: &'a EngineOutcomeV1,
+    request_commitment: Option<DecisionRequestCommitmentV1>,
     facts: &'a [ContextScoutIntentKindV1],
     prompt_inferences: &'a [ContextScoutIntentKindV1],
     decision: &'a IntentDecisionV1,
@@ -1104,6 +1110,7 @@ fn intent_decision_metadata(
     let IntentDecisionEvidenceV1 {
         window,
         engine_outcome,
+        request_commitment,
         facts,
         prompt_inferences,
         decision,
@@ -1177,6 +1184,12 @@ fn intent_decision_metadata(
             },
             "settings_digest": settings_digest,
             "model_outcome": engine_outcome.model_outcome(),
+            // The byte-exact prepared request is not proof that the remote
+            // server received it, nor that an old window remains rehydratable.
+            "prepared_request": request_commitment.map(|commitment| serde_json::json!({
+                "sha256": hex::encode(commitment.wire_sha256),
+                "bytes": commitment.wire_bytes,
+            })),
             "engine": engine_identity.map(|identity| serde_json::json!({
                 "base_url_digest": hex::encode(identity.base_url_digest),
                 "configured_model": identity.model,
@@ -1259,7 +1272,10 @@ mod tool_signal_tests {
         use super::super::decision_engine::{
             DecisionAnswerV1, DecisionAnswersV1, DecisionEngineOutcomeV1,
         };
-        use super::{EngineOutcomeV1, IntentDecisionEvidenceV1, intent_decision_metadata};
+        use super::{
+            DecisionRequestCommitmentV1, EngineOutcomeV1, IntentDecisionEvidenceV1,
+            intent_decision_metadata,
+        };
 
         let window = SessionWindowV1 {
             messages: vec![message(
@@ -1298,6 +1314,10 @@ mod tool_signal_tests {
             &IntentDecisionEvidenceV1 {
                 window: &window,
                 engine_outcome: &engine,
+                request_commitment: Some(DecisionRequestCommitmentV1 {
+                    wire_sha256: [9; 32],
+                    wire_bytes: 512,
+                }),
                 facts: &[],
                 prompt_inferences: &[Kind::Impact],
                 decision: &negative,
@@ -1321,6 +1341,8 @@ mod tool_signal_tests {
             0.17
         );
         assert_eq!(receipt["request_latency_micros"], 1250);
+        assert_eq!(receipt["prepared_request"]["sha256"], hex::encode([9; 32]));
+        assert_eq!(receipt["prepared_request"]["bytes"], 512);
         assert!(receipt["facts"].as_array().unwrap().is_empty());
         assert_eq!(receipt["prompt_inferences"].as_array().unwrap().len(), 1);
         assert!(!evidence.contains("Confidential transcript text"));
@@ -1332,6 +1354,7 @@ mod tool_signal_tests {
             &IntentDecisionEvidenceV1 {
                 window: &window,
                 engine_outcome: &EngineOutcomeV1::Unavailable,
+                request_commitment: None,
                 facts: &[],
                 prompt_inferences: &[],
                 decision: &IntentDecisionV1::NoSignal,
@@ -1346,6 +1369,7 @@ mod tool_signal_tests {
         assert!(unknown["answers"].is_null());
         assert!(unknown["raw_model_probabilities"].is_null());
         assert!(unknown["request_latency_micros"].is_null());
+        assert!(unknown["prepared_request"].is_null());
         assert!(unknown["engine"].is_null());
 
         let bad_window = SessionWindowV1 {
@@ -1357,6 +1381,7 @@ mod tool_signal_tests {
                 &IntentDecisionEvidenceV1 {
                     window: &bad_window,
                     engine_outcome: &engine,
+                    request_commitment: None,
                     facts: &[],
                     prompt_inferences: &[],
                     decision: &negative,

From 6d1822a088a977301d19fc288a2aff7f1354023a Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 00:23:53 -0800
Subject: [PATCH 51/76] fix(evals): bind qualified raw samples to their
 experiment

Refs #4535. Keep DRAFT; NEVER MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md        | 10 +++
 evals/intent_engine/run_http.py           | 46 +++++++++++
 evals/intent_engine/test_qualification.py | 95 +++++++++++++++++++++++
 3 files changed, 151 insertions(+)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index bb81a1efc8c..e69f566b00a 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -75,6 +75,16 @@ case, endpoint hash, requested model, client calibration, and the
 serving declaration. A completed result retains the same identity and each row's
 request fingerprint. The endpoint itself is not copied into the identity.
 
+For qualified runs, an optional first-response sample is now bound to the
+exact experiment hash, case ID, and request digest (and original captured wire
+bytes when present). A new qualified run refuses an already occupied sample
+path before inference; an interrupted qualified run accepts its existing sample
+only when the identity and original request agree. This prevents a stale sample
+from appearing to describe a different held-out run. Samples remain
+self-recorded diagnostic data, **not remote response attestation**. Historical
+nonqualified sample behavior is unchanged. Old unbound samples are not
+silently migrated: use a new sample path.
+
 An interrupted run resumes only with matching identity. Changed questions,
 question order, renderer output, labels, subset contents, or serving settings
 refuse before any HTTP request or append. Headerless old logs, incomplete rows,
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 419915bdf9c..763f519b0cb 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -52,6 +52,40 @@ def request_bytes(body):
     return json.dumps(body, allow_nan=False).encode("utf-8")
 
 
+def validate_qualified_sample(path, experiment_sha, requests):
+    """Refuse a stale first-response sample during an exact qualified resume.
+
+    Raw samples are debugging evidence, not independent model attestation.
+    Still, a sample from another experiment must never silently accompany a
+    fresh or resumed frozen result. Verify its original request bytes as well
+    as the experiment fingerprint, without normalizing a Scout capture.
+    """
+    if os.path.islink(path) or not os.path.isfile(path):
+        raise ValueError("qualified sample path must be a regular file")
+    with open(path, "rb") as source:
+        sample = loads_unambiguous_json(source.read())
+    if not isinstance(sample, dict):
+        raise ValueError("qualified sample must be a JSON object")
+    case_id = sample.get("case_id")
+    original = requests.get(case_id) if isinstance(case_id, str) else None
+    if (original is None or sample.get("experiment_sha256") != experiment_sha
+            or sample.get("request_sha256") != original[1]):
+        raise ValueError("qualified sample disagrees with frozen experiment identity")
+    body = original[0]
+    if isinstance(body, bytes):
+        matches_request = (
+            sample.get("request_json") == body.decode("utf-8")
+            and sample.get("request") == loads_unambiguous_json(body)
+        )
+    else:
+        matches_request = (
+            "request_json" not in sample and sample.get("request") == body
+        )
+    if not matches_request:
+        raise ValueError("qualified sample request differs from frozen HTTP bytes")
+    noul_probabilities(sample.get("response"), sample["request"]["questions"])
+
+
 def publish_sample(path, serialized):
     """Publish a completely rendered sample without clobbering an existing one.
 
@@ -677,6 +711,12 @@ def main(argv=None):
             identity["timeout_ms"] = args.timeout_ms
         header = {"experiment": identity, "experiment_sha256": digest(identity)}
         partial_path = path + ".partial"
+        if qualification_sha is not None and args.sample_out and os.path.lexists(args.sample_out):
+            # A new experiment cannot inherit a previous tag's first-response
+            # evidence. An exact resume may retain only its own sample.
+            if not os.path.lexists(partial_path):
+                raise ValueError("qualified sample already exists for a new experiment; choose a new --sample-out")
+            validate_qualified_sample(args.sample_out, header["experiment_sha256"], requests)
         try:
             # Default umask 022 (or an explicit umask 000) must not expose
             # result labels, cases or response telemetry to other users.
@@ -770,6 +810,12 @@ def main(argv=None):
                     }
                     if isinstance(body, bytes):
                         sample["request_json"] = body.decode("utf-8")
+                    if qualification_sha is not None:
+                        sample.update({
+                            "experiment_sha256": header["experiment_sha256"],
+                            "case_id": case["id"],
+                            "request_sha256": request_sha,
+                        })
                     serialized_sample = json.dumps(sample, indent=1, allow_nan=False)
                     publish_sample(args.sample_out, serialized_sample)
                 partial.write(serialized_row + "\n")
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 1fb442cb30c..54251e7dee8 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -999,6 +999,101 @@ def answer(url, body, **kwargs):
                 "holdout-positive", "holdout-negative"
             ])
 
+    def test_qualified_sample_is_bound_to_the_frozen_experiment_on_resume(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            sample = here / "first-response.json"
+            args = [
+                "--url", self.endpoint,
+                "--tag", "sample-provenance",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "--sample-out", str(sample),
+                "B_noul_text",
+            ]
+            calls = []
+
+            def answer(url, body, **kwargs):
+                calls.append(body)
+                if len(calls) == 2:
+                    raise OSError("interrupted before second response")
+                return {
+                    "model": "self-reported-test-model",
+                    "answers": {key: {"type": "noul", "noul": 0.2}
+                                for key in body["questions"]},
+                }, 4.0
+
+            partial = here / "results" / "sample-provenance_B_noul_text.json.partial"
+            final = here / "results" / "sample-provenance_B_noul_text.json"
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post", side_effect=answer
+            ):
+                with self.assertRaisesRegex(OSError, "interrupted before second response"):
+                    run_http.main(args)
+            self.assertEqual(len(calls), 2)
+            self.assertTrue(partial.exists())
+            self.assertTrue(sample.exists())
+            self.assertFalse(final.exists())
+            sample_bytes = sample.read_bytes()
+            evidence = json.loads(sample_bytes)
+            header = json.loads(partial.read_text().splitlines()[0])
+            self.assertEqual(evidence["experiment_sha256"], header["experiment_sha256"])
+            self.assertEqual(evidence["case_id"], "holdout-positive")
+            self.assertEqual(
+                evidence["request_sha256"],
+                hashlib.sha256(run_http.request_bytes(calls[0])).hexdigest(),
+            )
+            partial_bytes = partial.read_bytes()
+            for field, replacement in (
+                ("experiment_sha256", "0" * 64),
+                ("request_sha256", "f" * 64),
+            ):
+                with self.subTest(field=field):
+                    forged = copy.deepcopy(evidence)
+                    forged[field] = replacement
+                    sample.write_text(json.dumps(forged))
+                    with patch.object(run_http, "HERE", str(here)), patch.object(
+                        run_http, "post"
+                    ) as post:
+                        with self.assertRaisesRegex(ValueError, "qualified sample disagrees"):
+                            run_http.main(args)
+                        post.assert_not_called()
+                    self.assertEqual(partial.read_bytes(), partial_bytes)
+                    self.assertFalse(final.exists())
+            sample.write_bytes(sample_bytes)
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post", side_effect=lambda url, body, **kw: (
+                    {"model": "self-reported-test-model",
+                     "answers": {key: {"type": "noul", "noul": 0.1}
+                                 for key in body["questions"]}},
+                    3.0,
+                )
+            ) as post:
+                run_http.main(args)
+                self.assertEqual(post.call_count, 1)
+            self.assertTrue(final.exists())
+            self.assertFalse(partial.exists())
+            self.assertEqual(sample.read_bytes(), sample_bytes)
+            self.assertEqual(len(json.loads(final.read_text())["rows"]), 2)
+
+            # A new tag cannot accidentally publish a stale raw response as
+            # evidence of a completely separate experiment.
+            new_args = args.copy()
+            new_args[new_args.index("sample-provenance")] = "other-experiment"
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post"
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "qualified sample already exists"):
+                    run_http.main(new_args)
+                post.assert_not_called()
+            self.assertFalse(
+                (here / "results" / "other-experiment_B_noul_text.json.partial").exists()
+            )
+            self.assertEqual(sample.read_bytes(), sample_bytes)
+
     def test_runner_refuses_group_leakage_without_network_or_partial_output(self):
         with tempfile.TemporaryDirectory() as temp:
             here = Path(temp)

From 05cde28fc9873b52e2d0021e62a558e5b0ac2220 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 00:58:33 -0800
Subject: [PATCH 52/76] fix(evals): bind qualified raw response to resumed
 evidence

Refs #4535. Preserve DRAFT; NEVER MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md        | 14 ++++-
 evals/intent_engine/run_http.py           | 56 +++++++++++++++---
 evals/intent_engine/test_qualification.py | 72 +++++++++++++++++++++++
 3 files changed, 131 insertions(+), 11 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index e69f566b00a..871eaf2b583 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -79,9 +79,17 @@ For qualified runs, an optional first-response sample is now bound to the
 exact experiment hash, case ID, and request digest (and original captured wire
 bytes when present). A new qualified run refuses an already occupied sample
 path before inference; an interrupted qualified run accepts its existing sample
-only when the identity and original request agree. This prevents a stale sample
-from appearing to describe a different held-out run. Samples remain
-self-recorded diagnostic data, **not remote response attestation**. Historical
+only when the identity and original request agree. The sample must be from
+the **first** frozen held-out call; its complete raw response digest must
+match the first completed result row, and the sample commits to that complete
+row. Changed response probabilities, telemetry or row values, missing samples
+after a completed first row, and a sample with no completed first row all refuse
+without further inference or modifying the interrupted log. If the process
+crashes between first-sample publication and its row append, start a new tag;
+the original incomplete evidence is not silently reconstructed. A concurrent
+sample publication is refused instead of being silently adopted. These
+cross-checks prevent accidental stale or corrupted sample attribution, but
+operator-authored digests are **not remote response attestation**. Historical
 nonqualified sample behavior is unchanged. Old unbound samples are not
 silently migrated: use a new sample path.
 
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 763f519b0cb..1d41b75357e 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -67,6 +67,10 @@ def validate_qualified_sample(path, experiment_sha, requests):
     if not isinstance(sample, dict):
         raise ValueError("qualified sample must be a JSON object")
     case_id = sample.get("case_id")
+    # The first-response sample must be from the first frozen HTTP call,
+    # not merely from an arbitrary request in the same experiment.
+    if case_id != next(iter(requests)):
+        raise ValueError("qualified sample is not the first held-out response")
     original = requests.get(case_id) if isinstance(case_id, str) else None
     if (original is None or sample.get("experiment_sha256") != experiment_sha
             or sample.get("request_sha256") != original[1]):
@@ -84,9 +88,13 @@ def validate_qualified_sample(path, experiment_sha, requests):
     if not matches_request:
         raise ValueError("qualified sample request differs from frozen HTTP bytes")
     noul_probabilities(sample.get("response"), sample["request"]["questions"])
+    row_sha = sample.get("row_sha256")
+    if not isinstance(row_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", row_sha):
+        raise ValueError("qualified sample lacks the completed first-row commitment")
+    return digest(sample["response"]), row_sha
 
 
-def publish_sample(path, serialized):
+def publish_sample(path, serialized, refuse_existing=False):
     """Publish a completely rendered sample without clobbering an existing one.
 
     Same-directory staging avoids exposing a partial sample after a write error;
@@ -103,10 +111,11 @@ def publish_sample(path, serialized):
         try:
             os.link(staged, path)
         except FileExistsError:
-            # A dangling symlink occupies the pathname but exists() says
-            # false. Do not treat that unusable capture as already published.
-            if not os.path.isfile(path):
-                raise ValueError(f"sample path exists but is not a regular file: {path}")
+            # Qualified samples must never silently adopt an existing file
+            # that appeared after the preflight. Unqualified historical
+            # samples retain their original no-clobber behavior.
+            if refuse_existing or not os.path.isfile(path):
+                raise ValueError(f"sample path exists or was replaced during publication: {path}")
     finally:
         os.unlink(staged)
 
@@ -430,6 +439,11 @@ def serialize_result(row, questions, temperature):
         if row["probs"] != {key: calibrated(value, temperature) for key, value in raw.items()}:
             raise ValueError("calibrated probabilities disagree with raw probabilities")
     validate_row_truncation(row)
+    if "sample_response_sha256" in row and (
+        not isinstance(row["sample_response_sha256"], str)
+        or not re.fullmatch(r"[0-9a-f]{64}", row["sample_response_sha256"])
+    ):
+        raise ValueError("invalid first-response sample commitment")
     # json.loads accepts NaN/Infinity, including in extra nested fields. Reject
     # them while validating the partial log, not after opening the final file.
     return json.dumps(row, allow_nan=False)
@@ -711,12 +725,15 @@ def main(argv=None):
             identity["timeout_ms"] = args.timeout_ms
         header = {"experiment": identity, "experiment_sha256": digest(identity)}
         partial_path = path + ".partial"
+        admitted_sample = None
         if qualification_sha is not None and args.sample_out and os.path.lexists(args.sample_out):
             # A new experiment cannot inherit a previous tag's first-response
             # evidence. An exact resume may retain only its own sample.
             if not os.path.lexists(partial_path):
                 raise ValueError("qualified sample already exists for a new experiment; choose a new --sample-out")
-            validate_qualified_sample(args.sample_out, header["experiment_sha256"], requests)
+            admitted_sample = validate_qualified_sample(
+                args.sample_out, header["experiment_sha256"], requests,
+            )
         try:
             # Default umask 022 (or an explicit umask 000) must not expose
             # result labels, cases or response telemetry to other users.
@@ -741,6 +758,20 @@ def main(argv=None):
                 done = {}
             else:
                 done = resume_rows(partial, header, cases, requests, questions, args.noul_temperature)
+            if qualification_sha is not None and args.sample_out:
+                first_id = next(iter(cases))
+                if done and admitted_sample is None:
+                    raise ValueError("qualified first-response sample is missing for resumed evidence")
+                if admitted_sample is not None:
+                    if first_id not in done:
+                        # A crash between publishing the sample and appending
+                        # its row cannot be reconstructed as a completed call.
+                        raise ValueError("qualified sample has no completed first-response row; use a new tag")
+                    response_sha, row_sha = admitted_sample
+                    first_row = done[first_id]
+                    if (first_row.get("sample_response_sha256") != response_sha
+                            or digest(first_row) != row_sha):
+                        raise ValueError("qualified sample response or row disagrees with completed evidence")
             previous_model = next(
                 (row["observed_model"] for row in done.values()
                  if row.get("observed_model") is not None), None
@@ -799,11 +830,16 @@ def main(argv=None):
                 if args.noul_temperature is not None:
                     row["probs_raw"] = probs
                     row["probs"] = {key: calibrated(value, args.noul_temperature) for key, value in probs.items()}
+                if qualification_sha is not None and args.sample_out and index == 0:
+                    row["sample_response_sha256"] = digest(response)
                 # Refuse malformed row and raw-sample data before publishing
                 # either. A non-finite response must not leave a sample file
                 # that a later valid retry mistakes for a completed capture.
                 serialized_row = serialize_result(row, questions, args.noul_temperature)
-                if args.sample_out and not os.path.exists(args.sample_out):
+                if args.sample_out and (
+                    (qualification_sha is not None and index == 0)
+                    or (qualification_sha is None and not os.path.exists(args.sample_out))
+                ):
                     sample = {
                         "request": loads_unambiguous_json(body) if isinstance(body, bytes) else body,
                         "response": response,
@@ -815,9 +851,13 @@ def main(argv=None):
                             "experiment_sha256": header["experiment_sha256"],
                             "case_id": case["id"],
                             "request_sha256": request_sha,
+                            "row_sha256": digest(row),
                         })
                     serialized_sample = json.dumps(sample, indent=1, allow_nan=False)
-                    publish_sample(args.sample_out, serialized_sample)
+                    publish_sample(
+                        args.sample_out, serialized_sample,
+                        refuse_existing=qualification_sha is not None,
+                    )
                 partial.write(serialized_row + "\n")
                 partial.flush()
                 done[case["id"]] = row
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 54251e7dee8..7742e752bdc 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1039,6 +1039,9 @@ def answer(url, body, **kwargs):
             self.assertFalse(final.exists())
             sample_bytes = sample.read_bytes()
             evidence = json.loads(sample_bytes)
+            first_row = json.loads(partial.read_text().splitlines()[1])
+            self.assertEqual(first_row["sample_response_sha256"], run_http.digest(evidence["response"]))
+            self.assertEqual(evidence["row_sha256"], run_http.digest(first_row))
             header = json.loads(partial.read_text().splitlines()[0])
             self.assertEqual(evidence["experiment_sha256"], header["experiment_sha256"])
             self.assertEqual(evidence["case_id"], "holdout-positive")
@@ -1063,6 +1066,75 @@ def answer(url, body, **kwargs):
                         post.assert_not_called()
                     self.assertEqual(partial.read_bytes(), partial_bytes)
                     self.assertFalse(final.exists())
+            # Previously a valid Noul response from another call could be
+            # substituted under the correct request hash and accepted.
+            forged = copy.deepcopy(evidence)
+            forged["response"]["answers"]["impact"]["noul"] = 0.9
+            sample.write_text(json.dumps(forged))
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post"
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "sample response or row disagrees"):
+                    run_http.main(args)
+                post.assert_not_called()
+            self.assertEqual(partial.read_bytes(), partial_bytes)
+
+            # An otherwise valid, rewritten result row must not describe a
+            # different original first-response sample.
+            sample.write_bytes(sample_bytes)
+            lines = partial.read_text().splitlines()
+            forged_row = json.loads(lines[1])
+            forged_row["ms"] = 5.0
+            lines[1] = json.dumps(forged_row)
+            partial.write_text("\n".join(lines) + "\n")
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post"
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "sample response or row disagrees"):
+                    run_http.main(args)
+                post.assert_not_called()
+            partial.write_bytes(partial_bytes)
+
+            # A later case from the same frozen arm is not the first response.
+            forged = copy.deepcopy(evidence)
+            second = self.dataset[3]
+            second_request = {"state": run_http.STATES["B_noul_text"](second),
+                              "questions": self.questions}
+            forged["case_id"] = second["id"]
+            forged["request"] = second_request
+            forged["request_sha256"] = hashlib.sha256(
+                run_http.request_bytes(second_request)
+            ).hexdigest()
+            sample.write_text(json.dumps(forged))
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post"
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "not the first held-out"):
+                    run_http.main(args)
+                post.assert_not_called()
+            self.assertEqual(partial.read_bytes(), partial_bytes)
+
+            # Refuse a deleted sample after a completed first row.
+            sample.unlink()
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post"
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "sample is missing"):
+                    run_http.main(args)
+                post.assert_not_called()
+            self.assertEqual(partial.read_bytes(), partial_bytes)
+
+            # A sample published without a completed row is also insufficient
+            # to claim the first held-out call on resume.
+            sample.write_bytes(sample_bytes)
+            partial.write_text(partial_bytes.decode().splitlines()[0] + "\n")
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post"
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "no completed first-response row"):
+                    run_http.main(args)
+                post.assert_not_called()
+            partial.write_bytes(partial_bytes)
             sample.write_bytes(sample_bytes)
             with patch.object(run_http, "HERE", str(here)), patch.object(
                 run_http, "post", side_effect=lambda url, body, **kw: (

From a685c63e32c7e27da241f2f296551e04e6754d1f Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 02:10:13 -0800
Subject: [PATCH 53/76] style(scout): format prepared request commitment checks

---
 .../agents/context_scout/decision_engine.rs   | 36 +++++++++++++++----
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index 0484ca4a8bc..c1fdab444c2 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -269,7 +269,11 @@ impl DecisionEngineClientV1 {
         if !request_is_well_formed(request) {
             return None;
         }
-        let body = request_body(request, self.identity.model.as_deref(), self.max_state_bytes);
+        let body = request_body(
+            request,
+            self.identity.model.as_deref(),
+            self.max_state_bytes,
+        );
         let wire = serde_json::to_vec(&body).ok()?;
         Some(DecisionRequestCommitmentV1 {
             wire_sha256: Sha256::digest(&wire).into(),
@@ -873,17 +877,35 @@ mod tests {
 
         configured.model = Some("different-model".to_owned());
         let changed_model = DecisionEngineClientV1::new(configured).unwrap();
-        assert_ne!(changed_model.request_commitment(&original), Some(commitment));
+        assert_ne!(
+            changed_model.request_commitment(&original),
+            Some(commitment)
+        );
         let mut changed_question = original.clone();
-        changed_question.questions[0].instructions.push_str(" Be precise.");
-        assert_ne!(client.request_commitment(&changed_question), Some(commitment));
+        changed_question.questions[0]
+            .instructions
+            .push_str(" Be precise.");
+        assert_ne!(
+            client.request_commitment(&changed_question),
+            Some(commitment)
+        );
         // The prefix intentionally discarded by the configured byte cap
         // does not affect the sent request, but changed visible text does.
         let alternate_prefix = request("different private context éé latest", vec![noul("q")]);
-        assert_eq!(client.request_commitment(&alternate_prefix), Some(commitment));
+        assert_eq!(
+            client.request_commitment(&alternate_prefix),
+            Some(commitment)
+        );
         let changed_visible = request("old private context éé latest!", vec![noul("q")]);
-        assert_ne!(client.request_commitment(&changed_visible), Some(commitment));
-        assert!(client.request_commitment(&request("state", vec![])).is_none());
+        assert_ne!(
+            client.request_commitment(&changed_visible),
+            Some(commitment)
+        );
+        assert!(
+            client
+                .request_commitment(&request("state", vec![]))
+                .is_none()
+        );
     }
 
     #[test]

From ef48b85ce14ebf8c2ced552e744afea07447d427 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 02:14:06 -0800
Subject: [PATCH 54/76] fix(evals): reject impossible normalized state-token
 counts

Even a legacy HTTP row without raw native counters must not report
more dropped state tokens than originally existed. Reject forged partials
before inference and reject the same impossibility in independent scoring.
Include falsifiable denial, unchanged-partial and valid-resume regressions.

Refs #4535; DRAFT only, NO MERGE.
---
 evals/intent_engine/run_http.py           |  5 +++++
 evals/intent_engine/test_qualification.py | 12 +++++++++++
 evals/intent_engine/test_run_http.py      | 25 +++++++++++++++++++++++
 3 files changed, 42 insertions(+)

diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 1d41b75357e..da1f9499f22 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -372,6 +372,11 @@ def validate_row_truncation(row):
     original = row.get("state_tokens")
     dropped = row.get("dropped")
     used = row.get("state_tokens_used")
+    # Even historical HTTP rows lacking native counters must not claim that
+    # more state tokens were dropped than existed. Otherwise an edited partial
+    # can pass resume validation with a negative inferred used-token count.
+    if type(original) is int and type(dropped) is int and dropped > original:
+        raise ValueError("inconsistent usage: normalized dropped tokens exceed original state")
     if used is None and type(original) is int and type(dropped) is int:
         used = original - dropped
     input_tokens = row.get("input_tokens")
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 7742e752bdc..5469583cdd4 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -406,6 +406,18 @@ def test_impossible_token_counters_cannot_qualify_even_with_rehashed_evidence(se
                 self.questions, self.dataset_bytes,
             )
 
+    def test_qualified_scorer_refuses_impossible_normalized_legacy_counts(self):
+        result = self.signed_result()
+        # Remove the native source count but keep an otherwise self-consistent
+        # frozen experiment. Unknown provenance cannot excuse impossible math.
+        del result["rows"][0]["reported_state_tokens_dropped"]
+        result["rows"][0].update(state_tokens=3, dropped=4)
+        with self.assertRaisesRegex(ValueError, "normalized dropped tokens exceed original state"):
+            qualification.qualified_report(
+                result, self.manifest, self.manifest_sha,
+                self.questions, self.dataset_bytes,
+            )
+
     def test_boolean_truncation_without_counts_is_unknown(self):
         self.assertIsNone(run_http.dropped_state_tokens({"truncated": False}, {}))
         self.assertIsNone(run_http.dropped_state_tokens({"truncated": True}, {}))
diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index 6bf4797d9c9..4db7e6f0656 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -501,6 +501,31 @@ def test_dropped_count_exceeding_original_state_is_refused(self):
         self.assertEqual(self.partial.read_bytes(), before)
         self.assertFalse(self.output.exists())
 
+    def test_legacy_normalized_drops_cannot_exceed_original_on_resume(self):
+        self.interrupt()
+        header, row = self.partial.read_text().splitlines()
+        forged = json.loads(row)
+        # No source counters/flags: the normalized persisted row must still
+        # obey the basic nonnegative used-state invariant.
+        forged.update(state_tokens=3, dropped=4, input_tokens=12)
+        self.partial.write_text(header + "\n" + json.dumps(forged) + "\n")
+        bad_bytes = self.partial.read_bytes()
+        with patch.object(run_http, "post") as post:
+            with self.assertRaisesRegex(ValueError, "normalized dropped tokens exceed original state"):
+                self.run_main()
+            post.assert_not_called()
+        self.assertEqual(self.partial.read_bytes(), bad_bytes)
+        self.assertFalse(self.output.exists())
+
+        # A genuine interrupted prefix remains resumable after refusal.
+        self.partial.write_text(header + "\n" + row + "\n")
+        response, elapsed = self.response("", {"questions": run_http.noul_questions()})
+        with patch.object(run_http, "post", return_value=(response, elapsed)) as post:
+            self.run_main()
+            post.assert_called_once()
+        self.assertTrue(self.output.exists())
+        self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2)
+
     def test_missing_usage_survives_resume_and_scoring(self):
         response, elapsed = self.response("", {"questions": run_http.noul_questions()})
         response["usage"] = None

From d3dff1e7f5da6c7a6afc49bea4e718e20d2db3df Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 03:28:00 -0800
Subject: [PATCH 55/76] fix(evals): recheck first Scout sample before qualified
 publication

Refs #4535. A resumed qualified run checked its first-response sample
before the remaining HTTP calls, allowing replacement of that sample
in flight while still publishing an apparently complete experiment.
Revalidate the sample against the original first result under the
partial writer lock immediately before completed publication. Add a
falsifiable interrupted/resumed model-call substitution regression and
recovery with zero repeated inference. Keep decision-model PR DRAFT.
---
 evals/intent_engine/run_http.py           | 16 +++++
 evals/intent_engine/test_qualification.py | 74 +++++++++++++++++++++++
 2 files changed, 90 insertions(+)

diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index da1f9499f22..992c4271848 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -870,6 +870,22 @@ def main(argv=None):
             # Hold the partial writer lock through exclusive publication.
             # A second evaluator must not publish the same complete log first.
             # Failed publication retains the partial for an exact resume.
+            if qualification_sha is not None and args.sample_out:
+                # Preflight admission alone does not bind the sample for the
+                # duration of a resumed run: the sample path can be replaced
+                # while subsequent held-out HTTP calls are in flight. Recheck
+                # the original first response against the durable first row
+                # immediately before exposing a completed qualification.
+                response_sha, row_sha = validate_qualified_sample(
+                    args.sample_out, header["experiment_sha256"], requests,
+                )
+                first_row = done[next(iter(cases))]
+                if (first_row.get("sample_response_sha256") != response_sha
+                        or digest(first_row) != row_sha):
+                    raise ValueError(
+                        "qualified first-response sample changed during inference; "
+                        "refusing completed result"
+                    )
             publish_final_result(path, {
                 "config": name, "dataset_sha256": dataset_sha,
                 "noul_temperature": args.noul_temperature,
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 5469583cdd4..a09180d747c 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1178,6 +1178,80 @@ def answer(url, body, **kwargs):
             )
             self.assertEqual(sample.read_bytes(), sample_bytes)
 
+    def test_qualified_sample_replaced_during_resume_refuses_completed_result(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            sample = here / "first-response.json"
+            args = [
+                "--url", self.endpoint, "--tag", "sample-race",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "--sample-out", str(sample), "B_noul_text",
+            ]
+            partial = here / "results" / "sample-race_B_noul_text.json.partial"
+            completed = here / "results" / "sample-race_B_noul_text.json"
+            calls = 0
+
+            def interrupt(url, body, **kwargs):
+                nonlocal calls
+                calls += 1
+                if calls == 2:
+                    raise OSError("interrupted after first sample")
+                return {
+                    "model": "self-reported-test-model",
+                    "answers": {key: {"type": "noul", "noul": 0.2}
+                                for key in body["questions"]},
+                }, 4.0
+
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post", side_effect=interrupt,
+            ):
+                with self.assertRaisesRegex(OSError, "interrupted after first sample"):
+                    run_http.main(args)
+            self.assertEqual(calls, 2)
+            original_sample = sample.read_bytes()
+            self.assertEqual(len(partial.read_text().splitlines()), 2)
+            self.assertFalse(completed.exists())
+
+            def replace_after_admission(url, body, **kwargs):
+                replacement = json.loads(original_sample)
+                replacement["response"]["answers"]["impact"]["noul"] = 0.95
+                alternate = here / "replacement.json"
+                alternate.write_text(json.dumps(replacement))
+                alternate.replace(sample)
+                return {
+                    "model": "self-reported-test-model",
+                    "answers": {key: {"type": "noul", "noul": 0.1}
+                                for key in body["questions"]},
+                }, 3.0
+
+            # The first sample passed admission before the second model call.
+            # Replacing it during that call must not certify a completed run.
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post", side_effect=replace_after_admission,
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "sample changed during inference"):
+                    run_http.main(args)
+                self.assertEqual(post.call_count, 1)
+            self.assertFalse(completed.exists())
+            self.assertEqual(len(partial.read_text().splitlines()), 3)
+            self.assertNotEqual(sample.read_bytes(), original_sample)
+
+            # Restore exactly the recorded first response. A complete partial
+            # can then publish without replaying or inventing any model call.
+            sample.write_bytes(original_sample)
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post",
+            ) as post:
+                run_http.main(args)
+                post.assert_not_called()
+            self.assertTrue(completed.exists())
+            self.assertFalse(partial.exists())
+            self.assertEqual(len(json.loads(completed.read_text())["rows"]), 2)
+
     def test_runner_refuses_group_leakage_without_network_or_partial_output(self):
         with tempfile.TemporaryDirectory() as temp:
             here = Path(temp)

From 7043cd70fc78b0401497a835b132328556f6948b Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 04:47:27 -0800
Subject: [PATCH 56/76] fix(evals): refuse swapped qualified first-response
 samples

Refs #4535. Open qualified evidence by regular inode using nonblocking,
no-follow admission; verify the pathname still names that inode after
reading. Exercise before-open symlink and after-open pathname replacement
races with valid, identical sample evidence.

DRAFT / NO MERGE: canonical Scout evidence and Kev qualification pending.
---
 evals/intent_engine/HTTP_RUNNER.md        |  2 +
 evals/intent_engine/run_http.py           | 31 ++++++++-
 evals/intent_engine/test_qualification.py | 80 +++++++++++++++++++++++
 3 files changed, 110 insertions(+), 3 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 871eaf2b583..c34ddabee00 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -88,6 +88,8 @@ without further inference or modifying the interrupted log. If the process
 crashes between first-sample publication and its row append, start a new tag;
 the original incomplete evidence is not silently reconstructed. A concurrent
 sample publication is refused instead of being silently adopted. These
+sample admission uses an opened regular inode, refuses symlink/FIFO substitution,
+and rechecks the pathname after reading to reject a concurrent replacement. These
 cross-checks prevent accidental stale or corrupted sample attribution, but
 operator-authored digests are **not remote response attestation**. Historical
 nonqualified sample behavior is unchanged. Old unbound samples are not
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 992c4271848..c1bab8649a5 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -60,10 +60,35 @@ def validate_qualified_sample(path, experiment_sha, requests):
     fresh or resumed frozen result. Verify its original request bytes as well
     as the experiment fingerprint, without normalizing a Scout capture.
     """
-    if os.path.islink(path) or not os.path.isfile(path):
+    # The path is untrusted resumable evidence. A separate islink/isfile
+    # preflight followed by builtins.open can follow a swapped-in symlink.
+    # Open the inode itself without following links or waiting on a FIFO,
+    # then verify it is still the named, regular sample after reading.
+    if not hasattr(os, "O_NOFOLLOW") and os.path.islink(path):
         raise ValueError("qualified sample path must be a regular file")
-    with open(path, "rb") as source:
-        sample = loads_unambiguous_json(source.read())
+    flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        descriptor = os.open(path, flags)
+    except OSError as error:
+        if error.errno in (errno.ELOOP, errno.ENOTDIR):
+            raise ValueError("qualified sample path must be a regular file") from error
+        raise
+    try:
+        opened = os.fstat(descriptor)
+        if not stat.S_ISREG(opened.st_mode):
+            raise ValueError("qualified sample path must be a regular file")
+        with os.fdopen(descriptor, "rb", closefd=False) as source:
+            sample_bytes = source.read()
+    finally:
+        os.close(descriptor)
+    try:
+        named = os.stat(path, follow_symlinks=False)
+    except FileNotFoundError as error:
+        raise ValueError("qualified sample path disappeared during validation") from error
+    if (not stat.S_ISREG(named.st_mode)
+            or (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino)):
+        raise ValueError("qualified sample path was replaced or redirected")
+    sample = loads_unambiguous_json(sample_bytes)
     if not isinstance(sample, dict):
         raise ValueError("qualified sample must be a JSON object")
     case_id = sample.get("case_id")
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index a09180d747c..5a5c9f1232b 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -2,6 +2,7 @@
 import copy
 import hashlib
 import json
+import os
 import shutil
 import subprocess
 import sys
@@ -1178,6 +1179,85 @@ def answer(url, body, **kwargs):
             )
             self.assertEqual(sample.read_bytes(), sample_bytes)
 
+    @unittest.skipUnless(
+        os.name == "posix" and hasattr(os, "O_NOFOLLOW"),
+        "inode-safe sample admission requires POSIX O_NOFOLLOW",
+    )
+    def test_qualified_sample_path_swaps_refuse_even_with_identical_evidence(self):
+        # The other file contains valid, byte-identical first-response evidence.
+        # Merely checking JSON digests would accept it after path substitution.
+        # Force each race at the syscall boundary, not through an idle symlink.
+        with tempfile.TemporaryDirectory() as temp:
+            sample = Path(temp) / "first.json"
+            other = Path(temp) / "other.json"
+            request = {"state": "original Scout window", "questions": self.questions}
+            response = {
+                "model": "reported-model",
+                "answers": {key: {"type": "noul", "noul": 0.2}
+                            for key in self.questions},
+            }
+            experiment_sha = "f" * 64
+            requests = {
+                "holdout-positive": (
+                    request, hashlib.sha256(run_http.request_bytes(request)).hexdigest()
+                ),
+            }
+            evidence = {
+                "case_id": "holdout-positive",
+                "experiment_sha256": experiment_sha,
+                "request_sha256": requests["holdout-positive"][1],
+                "row_sha256": "a" * 64,
+                "request": request,
+                "response": response,
+            }
+            original_bytes = json.dumps(evidence).encode("utf-8")
+            other.write_bytes(original_bytes)
+            real_open, real_fstat = os.open, os.fstat
+
+            def redirect():
+                sample.unlink()
+                sample.symlink_to(other.name)
+
+            for stage in ("before_open", "after_open"):
+                with self.subTest(stage=stage):
+                    sample.write_bytes(original_bytes)
+                    self.assertEqual(
+                        run_http.validate_qualified_sample(
+                            str(sample), experiment_sha, requests,
+                        ),
+                        (run_http.digest(response), evidence["row_sha256"]),
+                    )
+                    swaps = []
+
+                    def swap_before_open(path, flags, *args, **kwargs):
+                        if path == str(sample):
+                            swaps.append(stage)
+                            redirect()
+                        return real_open(path, flags, *args, **kwargs)
+
+                    def swap_after_open(descriptor):
+                        opened = real_fstat(descriptor)
+                        swaps.append(stage)
+                        redirect()
+                        return opened
+
+                    replacement = (
+                        patch.object(run_http.os, "open", side_effect=swap_before_open)
+                        if stage == "before_open" else
+                        patch.object(run_http.os, "fstat", side_effect=swap_after_open)
+                    )
+                    with replacement:
+                        with self.assertRaisesRegex(
+                            ValueError, "qualified sample path .*regular file|replaced or redirected",
+                        ):
+                            run_http.validate_qualified_sample(
+                                str(sample), experiment_sha, requests,
+                            )
+                    self.assertEqual(swaps, [stage], "the sample inode race must actually occur")
+                    self.assertTrue(sample.is_symlink())
+                    self.assertEqual(other.read_bytes(), original_bytes)
+                    sample.unlink()
+
     def test_qualified_sample_replaced_during_resume_refuses_completed_result(self):
         with tempfile.TemporaryDirectory() as temp:
             here = Path(temp)

From 2deaf0120906e1a486925393fd568e940c343e3f Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 04:48:09 -0800
Subject: [PATCH 57/76] docs(evals): clarify no-follow sample admission

Refs #4535. Document the inode/path guard without implying remote
attestation. DRAFT / NEVER MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index c34ddabee00..31268c46632 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -87,10 +87,11 @@ after a completed first row, and a sample with no completed first row all refuse
 without further inference or modifying the interrupted log. If the process
 crashes between first-sample publication and its row append, start a new tag;
 the original incomplete evidence is not silently reconstructed. A concurrent
-sample publication is refused instead of being silently adopted. These
-sample admission uses an opened regular inode, refuses symlink/FIFO substitution,
-and rechecks the pathname after reading to reject a concurrent replacement. These
-cross-checks prevent accidental stale or corrupted sample attribution, but
+sample publication is refused instead of being silently adopted. Qualified
+sample admission opens a nonblocking, no-follow regular-file inode and checks
+that the pathname still identifies it after reading. Symlink, FIFO and concurrent
+pathname substitution refuse qualification. These cross-checks prevent accidental
+stale or corrupted sample attribution, but
 operator-authored digests are **not remote response attestation**. Historical
 nonqualified sample behavior is unchanged. Old unbound samples are not
 silently migrated: use a new sample path.

From 2179bfe05d9148685871c22c38ed9e80789768b7 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 05:40:18 -0800
Subject: [PATCH 58/76] fix(evals): cross-check qualified raw samples in
 independent scoring

Refs #4535. Keep Kev decision-model qualification DRAFT; NO MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md        |  9 ++-
 evals/intent_engine/qualification.py      | 26 ++++++++-
 evals/intent_engine/score.py              |  7 ++-
 evals/intent_engine/test_qualification.py | 70 +++++++++++++++++++++++
 4 files changed, 106 insertions(+), 6 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 31268c46632..8b1b153e4a9 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -94,7 +94,14 @@ pathname substitution refuse qualification. These cross-checks prevent accidenta
 stale or corrupted sample attribution, but
 operator-authored digests are **not remote response attestation**. Historical
 nonqualified sample behavior is unchanged. Old unbound samples are not
-silently migrated: use a new sample path.
+silently migrated: use a new sample path. When a qualified result contains the
+first-row `sample_response_sha256` commitment, independent scoring requires
+`--sample-file /path/to/first-response.json`. The scorer reopens the original
+regular-file inode, reconstructs the frozen request and verifies experiment,
+response and complete first-row commitments before publishing a report. It
+rejects a sample with no matching first-row commitment, a missing required
+sample, or a forged commitment on a later row. Runs without a raw sample
+report `first_response_sample_checked: false`, not raw-response attestation.
 
 An interrupted run resumes only with matching identity. Changed questions,
 question order, renderer output, labels, subset contents, or serving settings
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index da47ebba517..5fc7623e84e 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -356,12 +356,12 @@ def holdout_decision_metrics(rows, thresholds):
     }
 
 
-def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, captured_requests_bytes=None):
+def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, captured_requests_bytes=None, sample_path=None):
     """Reproduce qualified predictions from the frozen original dataset."""
     from score import score_config
     from run_http import (
         STATES, has_numeric_truncation_receipt, request_bytes, validate_dataset,
-        validate_row_truncation,
+        validate_row_truncation, validate_qualified_sample,
     )
 
     if not isinstance(dataset_bytes, bytes):
@@ -460,6 +460,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
             raise ValueError("qualification has an unsupported request renderer")
     case_map = {case["id"]: case for case in cases}
     request_sizes = []
+    reconstructed_requests = {}
     for row in rows:
         if captured is not None:
             rendered = captured[row["id"]]
@@ -480,6 +481,26 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
                 or recorded_size != size):
             raise ValueError(f"qualification request byte count disagrees for {row['id']}")
         request_sizes.append(size)
+        reconstructed_requests[row["id"]] = (
+            rendered if captured is not None else body,
+            sha256_bytes(rendered),
+        )
+
+    # The runner checks raw first-response evidence on resume. Repeat that
+    # check independently before publishing a qualified scoring report.
+    if any("sample_response_sha256" in row for row in rows[1:]):
+        raise ValueError("qualification has a sample commitment outside the first held-out row")
+    first_row = rows[0]
+    has_sample_commitment = "sample_response_sha256" in first_row
+    if has_sample_commitment != (sample_path is not None):
+        raise ValueError("qualification first-response sample and row commitment must be provided together")
+    if has_sample_commitment:
+        response_sha, row_sha = validate_qualified_sample(
+            sample_path, result["experiment_sha256"], reconstructed_requests,
+        )
+        if (first_row["sample_response_sha256"] != response_sha
+                or sha256_json(first_row) != row_sha):
+            raise ValueError("qualification first-response sample response or row disagrees with completed evidence")
 
     if ("noul_temperature" not in experiment
             or experiment["noul_temperature"] != validated["noul_temperature"]):
@@ -547,6 +568,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
         "experiment_sha256": result["experiment_sha256"],
         "qualification_manifest_sha256": manifest_sha,
         "reported_model": reported_model,
+        "first_response_sample_checked": has_sample_commitment,
         "cases": len(rows),
         "thresholds": validated["thresholds"],
         "tolerances": tolerances,
diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py
index 2bf0ca12201..b4984c0aa51 100644
--- a/evals/intent_engine/score.py
+++ b/evals/intent_engine/score.py
@@ -308,13 +308,13 @@ def ch(rs):
     here = os.path.dirname(os.path.abspath(__file__))
     if len(sys.argv) > 1:
         usage = ("usage: score.py [--qualified MANIFEST DATASET RESULT "
-                 "[--questions-file FILE] [--captured-requests FILE]]")
+                 "[--questions-file FILE] [--captured-requests FILE] [--sample-file FILE]]")
         if sys.argv[1] != "--qualified" or len(sys.argv) < 5 or len(sys.argv[5:]) % 2:
             raise ValueError(usage)
         options = {}
         for index in range(5, len(sys.argv), 2):
             flag = sys.argv[index]
-            if flag not in ("--questions-file", "--captured-requests") or flag in options:
+            if flag not in ("--questions-file", "--captured-requests", "--sample-file") or flag in options:
                 raise ValueError(usage)
             options[flag] = sys.argv[index + 1]
         from qualification import load_manifest, load_questions, loads_unambiguous_json, qualified_report
@@ -330,7 +330,8 @@ def ch(rs):
             with open(options["--captured-requests"], "rb") as source:
                 captured_requests_bytes = source.read()
         report = qualified_report(
-            result, manifest, manifest_sha, questions, dataset_bytes, captured_requests_bytes,
+            result, manifest, manifest_sha, questions, dataset_bytes,
+            captured_requests_bytes, options.get("--sample-file"),
         )
         publish_summary(os.path.join(here, "qualification_summary.json"), report)
         print(json.dumps(report, indent=2, allow_nan=False))
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 5a5c9f1232b..df87767e629 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -585,6 +585,76 @@ def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self):
             qualification.qualified_report(result, stricter, qualification.sha256_json(stricter), self.questions, self.dataset_bytes)
 
 
+    def test_independent_qualified_scoring_rechecks_first_response_sample(self):
+        result = self.signed_result()
+        first = result["rows"][0]
+        case = self.dataset[2]
+        request = {
+            "state": run_http.STATES["B_noul_text"](case),
+            "questions": self.questions,
+        }
+        response = {
+            "model": first["observed_model"],
+            "answers": {
+                key: {"type": "noul", "noul": value}
+                for key, value in first["probs"].items()
+            },
+        }
+        first["sample_response_sha256"] = run_http.digest(response)
+        sample = {
+            "request": request,
+            "response": response,
+            "experiment_sha256": result["experiment_sha256"],
+            "case_id": first["id"],
+            "request_sha256": first["request_sha256"],
+            "row_sha256": run_http.digest(first),
+        }
+        with tempfile.TemporaryDirectory() as temp:
+            path = Path(temp) / "raw-first-response.json"
+            path.write_text(json.dumps(sample), encoding="utf-8")
+
+            def score_sample(value=result):
+                return qualification.qualified_report(
+                    value, self.manifest, self.manifest_sha, self.questions,
+                    self.dataset_bytes, sample_path=str(path),
+                )
+
+            report = score_sample()
+            self.assertTrue(report["qualified_against_frozen_tolerances"])
+            self.assertTrue(report["first_response_sample_checked"])
+            with self.assertRaisesRegex(ValueError, "must be provided together"):
+                qualification.qualified_report(
+                    result, self.manifest, self.manifest_sha,
+                    self.questions, self.dataset_bytes,
+                )
+
+            wrong = copy.deepcopy(sample)
+            wrong["response"]["answers"]["impact"]["noul"] = 0.31
+            path.write_text(json.dumps(wrong), encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "response or row disagrees"):
+                score_sample()
+
+            wrong = copy.deepcopy(sample)
+            wrong["request"]["state"] += " Foreign transcript"
+            path.write_text(json.dumps(wrong), encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "request differs"):
+                score_sample()
+
+            path.write_text(json.dumps(sample), encoding="utf-8")
+            altered_result = copy.deepcopy(result)
+            altered_result["rows"][0]["ms"] = 9
+            with self.assertRaisesRegex(ValueError, "response or row disagrees"):
+                score_sample(altered_result)
+
+            later_sample = copy.deepcopy(result)
+            later_sample["rows"][1]["sample_response_sha256"] = "f" * 64
+            with self.assertRaisesRegex(ValueError, "outside the first"):
+                score_sample(later_sample)
+
+            without_commitment = self.signed_result()
+            with self.assertRaisesRegex(ValueError, "must be provided together"):
+                score_sample(without_commitment)
+
     def test_qualified_scorer_rejects_edited_choice_and_extra_none_evidence(self):
         # These edits previously passed the generic historical scorer. An
         # independently scored Noul-only result must reject them even when

From bde1eddf92851c54bd6689bdebf0c37d1663549b Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 05:41:21 -0800
Subject: [PATCH 59/76] test(evals): exercise qualified raw-sample CLI refusal
 and recovery

Refs #4535. Keep decision-model changes DRAFT; NEVER MERGE.
---
 evals/intent_engine/test_qualification_cli.py | 58 +++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/evals/intent_engine/test_qualification_cli.py b/evals/intent_engine/test_qualification_cli.py
index b964d03f8f6..8f5d985dadc 100644
--- a/evals/intent_engine/test_qualification_cli.py
+++ b/evals/intent_engine/test_qualification_cli.py
@@ -95,6 +95,64 @@ def test_captured_arm_cli_requires_original_request_bytes(self):
         self.assertIn("original frozen Scout request capture", rejected.stderr)
         self.assertEqual((self.root / "qualification_summary.json").read_bytes(), frozen_report)
 
+    def test_raw_first_response_sample_is_checked_by_independent_cli(self):
+        result = self.fixture.signed_result()
+        first = result["rows"][0]
+        request = {
+            "state": fixtures.run_http.STATES["B_noul_text"](self.fixture.dataset[2]),
+            "questions": self.fixture.questions,
+        }
+        response = {
+            "model": first["observed_model"],
+            "answers": {
+                key: {"type": "noul", "noul": value}
+                for key, value in first["probs"].items()
+            },
+        }
+        first["sample_response_sha256"] = fixtures.run_http.digest(response)
+        sample = {
+            "request": request,
+            "response": response,
+            "experiment_sha256": result["experiment_sha256"],
+            "case_id": first["id"],
+            "request_sha256": first["request_sha256"],
+            "row_sha256": fixtures.run_http.digest(first),
+        }
+        (self.root / "result.json").write_text(json.dumps(result), encoding="utf-8")
+        sample_file = self.root / "sample.json"
+        sample_file.write_text(json.dumps(sample), encoding="utf-8")
+        argv = [
+            sys.executable, str(self.root / "score.py"), "--qualified",
+            str(self.root / "manifest.json"), str(self.root / "dataset.json"),
+            str(self.root / "result.json"),
+        ]
+        missing = subprocess.run(
+            argv, cwd=self.root, capture_output=True, text=True, timeout=30,
+        )
+        self.assertNotEqual(missing.returncode, 0)
+        self.assertIn("must be provided together", missing.stderr)
+        self.assertFalse((self.root / "qualification_summary.json").exists())
+
+        qualified = argv + ["--sample-file", str(sample_file)]
+        passing = subprocess.run(
+            qualified, cwd=self.root, capture_output=True, text=True, timeout=30,
+        )
+        self.assertEqual(passing.returncode, 0, passing.stderr)
+        report = self.published_report(passing)
+        self.assertTrue(report["first_response_sample_checked"])
+        previous = (self.root / "qualification_summary.json").read_bytes()
+
+        altered = json.loads(sample_file.read_text(encoding="utf-8"))
+        altered["response"]["answers"]["impact"]["noul"] = 0.4
+        sample_file.write_text(json.dumps(altered), encoding="utf-8")
+        rejected = subprocess.run(
+            qualified, cwd=self.root, capture_output=True, text=True, timeout=30,
+        )
+        self.assertNotEqual(rejected.returncode, 0)
+        self.assertIn("response or row disagrees", rejected.stderr)
+        self.assertEqual(rejected.stdout, "")
+        self.assertEqual((self.root / "qualification_summary.json").read_bytes(), previous)
+
     def test_duplicate_result_keys_cannot_publish_a_passing_report(self):
         valid = self.fixture.signed_result()
         raw = json.dumps(valid, allow_nan=False)

From 719072634537403e67b29e641041bbcfc16e27cd Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 06:55:06 -0800
Subject: [PATCH 60/76] fix(evals): bound raw Scout qualification sample
 admission

Refs #4535. Refuse oversized samples before reading and detect growth after fstat. Add positive and falsifiable sparse-file/racing-writer regression. Keep DRAFT; do not merge.
---
 evals/intent_engine/run_http.py           | 13 +++++-
 evals/intent_engine/test_qualification.py | 57 +++++++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index c1bab8649a5..9a6ca806ce7 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -64,6 +64,11 @@ def validate_qualified_sample(path, experiment_sha, requests):
     # preflight followed by builtins.open can follow a swapped-in symlink.
     # Open the inode itself without following links or waiting on a FIFO,
     # then verify it is still the named, regular sample after reading.
+    # A sample contains two representations of the frozen first request and
+    # a JSON-escaped copy of the at-most-1-MiB response. Allow generous
+    # serialization expansion, not unlimited attacker-controlled file bytes.
+    first_request = next(iter(requests.values()))[0]
+    sample_limit = 16 * (len(request_bytes(first_request)) + MAX_HTTP_RESPONSE_BYTES) + 4096
     if not hasattr(os, "O_NOFOLLOW") and os.path.islink(path):
         raise ValueError("qualified sample path must be a regular file")
     flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0)
@@ -77,8 +82,14 @@ def validate_qualified_sample(path, experiment_sha, requests):
         opened = os.fstat(descriptor)
         if not stat.S_ISREG(opened.st_mode):
             raise ValueError("qualified sample path must be a regular file")
+        if opened.st_size > sample_limit:
+            raise ValueError("qualified sample exceeds bounded evidence byte limit")
         with os.fdopen(descriptor, "rb", closefd=False) as source:
-            sample_bytes = source.read()
+            sample_bytes = source.read(sample_limit + 1)
+        # The inode can grow between fstat and read. A bounded read must also
+        # refuse growth rather than treating a truncated prefix as evidence.
+        if len(sample_bytes) > sample_limit:
+            raise ValueError("qualified sample exceeds bounded evidence byte limit")
     finally:
         os.close(descriptor)
     try:
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index df87767e629..c8be0d255f3 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1249,6 +1249,63 @@ def answer(url, body, **kwargs):
             )
             self.assertEqual(sample.read_bytes(), sample_bytes)
 
+    def test_qualified_sample_refuses_oversize_and_growth_before_parsing(self):
+        with tempfile.TemporaryDirectory() as temp:
+            sample = Path(temp) / "first.json"
+            request = {"state": "bounded Scout window", "questions": self.questions}
+            response = {
+                "model": "reported-model",
+                "answers": {key: {"type": "noul", "noul": 0.2}
+                            for key in self.questions},
+            }
+            request_sha = hashlib.sha256(run_http.request_bytes(request)).hexdigest()
+            requests = {"holdout-positive": (request, request_sha)}
+            evidence = {
+                "case_id": "holdout-positive",
+                "experiment_sha256": "f" * 64,
+                "request_sha256": request_sha,
+                "row_sha256": "a" * 64,
+                "request": request,
+                "response": response,
+            }
+            valid_bytes = json.dumps(evidence).encode("utf-8")
+            sample.write_bytes(valid_bytes)
+            self.assertEqual(
+                run_http.validate_qualified_sample(str(sample), "f" * 64, requests),
+                (run_http.digest(response), "a" * 64),
+            )
+            sample_limit = 16 * (
+                len(run_http.request_bytes(request)) + run_http.MAX_HTTP_RESPONSE_BYTES
+            ) + 4096
+
+            # An oversized sparse file must be refused from fstat alone,
+            # without materializing its huge attacker-controlled contents.
+            with sample.open("r+b") as output:
+                output.truncate(sample_limit + 1)
+            with patch.object(run_http.os, "fdopen") as fdopen:
+                with self.assertRaisesRegex(ValueError, "exceeds bounded evidence"):
+                    run_http.validate_qualified_sample(str(sample), "f" * 64, requests)
+                fdopen.assert_not_called()
+
+            # A writer may grow the same inode *after* the initial size
+            # check. The bounded read must refuse that race too.
+            sample.write_bytes(valid_bytes)
+            actual_fstat = os.fstat
+            changes = []
+
+            def grow_after_fstat(descriptor):
+                original_stat = actual_fstat(descriptor)
+                changes.append(original_stat.st_size)
+                with sample.open("r+b") as output:
+                    output.truncate(sample_limit + 1)
+                return original_stat
+
+            with patch.object(run_http.os, "fstat", side_effect=grow_after_fstat):
+                with self.assertRaisesRegex(ValueError, "exceeds bounded evidence"):
+                    run_http.validate_qualified_sample(str(sample), "f" * 64, requests)
+            self.assertEqual(changes, [len(valid_bytes)])
+            self.assertEqual(sample.stat().st_size, sample_limit + 1)
+
     @unittest.skipUnless(
         os.name == "posix" and hasattr(os, "O_NOFOLLOW"),
         "inode-safe sample admission requires POSIX O_NOFOLLOW",

From 2ccfd40e1bf116d0e9bca3df297aa02f7a989c3c Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 07:46:58 -0800
Subject: [PATCH 61/76] fix(evals): bound resumed HTTP evidence reads
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Refs #4535

Refuse oversized sparse partial logs and individual JSON lines before
inference; prove fail-closed refusal and legitimate interrupted recovery.
DRAFT — NEVER MERGE.
---
 evals/intent_engine/HTTP_RUNNER.md        |  8 +++
 evals/intent_engine/run_http.py           | 19 ++++-
 evals/intent_engine/test_qualification.py | 87 +++++++++++++++++++++++
 3 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 8b1b153e4a9..f0d5712a33b 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -103,6 +103,14 @@ rejects a sample with no matching first-row commitment, a missing required
 sample, or a forged commitment on a later row. Runs without a raw sample
 report `first_response_sample_checked: false`, not raw-response attestation.
 
+Interrupted .partial logs are untrusted evidence. Resume checks a
+case-count-derived total size ceiling before parsing, and bounds reads of
+each identity header and result row. The bounds include the original 1-MiB
+HTTP response cap, frozen questions and case IDs, with space for JSON
+escaping. Oversized sparse files and overlong or unterminated rows refuse
+before inference, without overwriting recoverable legitimate prefixes.
+This is a memory-admission check, not independent source attestation.
+
 An interrupted run resumes only with matching identity. Changed questions,
 question order, renderer output, labels, subset contents, or serving settings
 refuse before any HTTP request or append. Headerless old logs, incomplete rows,
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 9a6ca806ce7..4a8eb9933b6 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -549,14 +549,29 @@ def open_partial_for_resume(path):
 
 def resume_rows(partial, header, cases, requests, questions, temperature):
     """Validate the whole existing log before appending or issuing HTTP work."""
-    first = partial.readline()
+    # .partial is untrusted input. Limit total evidence and individual JSON
+    # lines using the original response cap and frozen case/question identities.
+    # The writer uses ASCII-escaped JSON; these character caps also bound bytes.
+    header_limit = len(json.dumps(header, allow_nan=False)) + 4096
+    row_limit = 16 * (
+        MAX_HTTP_RESPONSE_BYTES
+        + len(json.dumps(questions, allow_nan=False))
+        + max(len(case_id) for case_id in cases)
+    ) + 4096
+    if os.fstat(partial.fileno()).st_size > header_limit + 1 + len(cases) * (row_limit + 1):
+        raise ValueError("partial output exceeds bounded evidence byte limit")
+    first = partial.readline(header_limit + 1)
+    if len(first) > header_limit:
+        raise ValueError("partial header exceeds bounded evidence byte limit")
     if not first.endswith("\n") or loads_unambiguous_json(first) != header:
         raise ValueError("partial output belongs to another or unrecorded experiment; use a new tag")
     done = {}
     ordered_ids = tuple(cases)
     qualified = "qualification_manifest_sha256" in header["experiment"]
     previous_model = None
-    for line_number, line in enumerate(partial, 2):
+    for line_number, line in enumerate(iter(lambda: partial.readline(row_limit + 1), ""), 2):
+        if len(line) > row_limit:
+            raise ValueError(f"partial row at line {line_number} exceeds bounded evidence byte limit")
         if not line.endswith("\n"):
             raise ValueError(f"partial output has an incomplete row at line {line_number}")
         row = loads_unambiguous_json(line)
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index c8be0d255f3..40aeb3f93d5 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1249,6 +1249,93 @@ def answer(url, body, **kwargs):
             )
             self.assertEqual(sample.read_bytes(), sample_bytes)
 
+    def test_qualified_resume_bounds_sparse_partial_and_individual_rows(self):
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_bytes(self.manifest_bytes)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            args = [
+                "--url", self.endpoint, "--tag", "bounded-resume",
+                "--server-config", str(here / "server.json"),
+                "--qualification-manifest", str(here / "manifest.json"),
+                "B_noul_text",
+            ]
+            partial = here / "results" / "bounded-resume_B_noul_text.json.partial"
+            completed = here / "results" / "bounded-resume_B_noul_text.json"
+            calls = 0
+
+            def interrupt(url, body, **kwargs):
+                nonlocal calls
+                calls += 1
+                if calls == 2:
+                    raise OSError("interrupted after first held-out result")
+                return {
+                    "model": "self-reported-test-model",
+                    "answers": {key: {"type": "noul", "noul": 0.2}
+                                for key in body["questions"]},
+                }, 4.0
+
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post", side_effect=interrupt,
+            ):
+                with self.assertRaisesRegex(OSError, "interrupted after first held-out"):
+                    run_http.main(args)
+            self.assertEqual(calls, 2)
+            original = partial.read_bytes()
+            self.assertEqual(len(original.splitlines()), 2)
+            self.assertFalse(completed.exists())
+
+            header = json.loads(original.splitlines()[0])
+            header_limit = len(json.dumps(header, allow_nan=False)) + 4096
+            case_ids = ["holdout-positive", "holdout-negative"]
+            row_limit = 16 * (
+                run_http.MAX_HTTP_RESPONSE_BYTES
+                + len(json.dumps(self.questions, allow_nan=False))
+                + max(map(len, case_ids))
+            ) + 4096
+            total_limit = header_limit + 1 + len(case_ids) * (row_limit + 1)
+
+            # A sparse oversized file fails at fstat rather than being read.
+            with partial.open("r+b") as output:
+                output.truncate(total_limit + 1)
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post",
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "partial output exceeds bounded evidence"):
+                    run_http.main(args)
+                post.assert_not_called()
+            self.assertEqual(partial.stat().st_size, total_limit + 1)
+            self.assertFalse(completed.exists())
+
+            # A single unterminated row exceeds its cap even below total cap.
+            partial.write_bytes(original)
+            with partial.open("r+b") as output:
+                output.truncate(len(original) + row_limit + 1)
+            self.assertLessEqual(partial.stat().st_size, total_limit)
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post",
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "partial row at line 3 exceeds bounded"):
+                    run_http.main(args)
+                post.assert_not_called()
+            self.assertFalse(completed.exists())
+
+            # Restoring the real prefix resumes exactly one remaining case.
+            partial.write_bytes(original)
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post", side_effect=lambda url, body, **kwargs: (
+                    {"model": "self-reported-test-model",
+                     "answers": {key: {"type": "noul", "noul": 0.1}
+                                 for key in body["questions"]}},
+                    3.0,
+                ),
+            ) as post:
+                run_http.main(args)
+                self.assertEqual(post.call_count, 1)
+            self.assertFalse(partial.exists())
+            self.assertEqual(len(json.loads(completed.read_text())["rows"]), 2)
+
     def test_qualified_sample_refuses_oversize_and_growth_before_parsing(self):
         with tempfile.TemporaryDirectory() as temp:
             sample = Path(temp) / "first.json"

From 66b9f1066d389993256b3bfab8c01c4082228f70 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 08:21:12 -0800
Subject: [PATCH 62/76] fix(scout): retain original window and state bounds in
 receipts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Refs #4535

Record exact session selection limits and model render byte ceiling with
negative and positive Scout intent decision provenance. These values are
required to reconstruct a historical window after mutable settings change;
the settings digest alone cannot recover the old bounds. Keep transcript
text in its owning store. DRAFT — NEVER MERGE.
---
 .../src/agents/context_scout/intent_pass.rs   | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
index 4606e7d0deb..d4885d54b75 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
@@ -1076,6 +1076,8 @@ async fn record_intent_decision(
         &evidence,
         &settings_digest,
         &settings.questions,
+        &settings.window,
+        settings.engine.max_state_bytes,
         engine_identity.as_ref(),
     ) else {
         return;
@@ -1105,6 +1107,8 @@ fn intent_decision_metadata(
     evidence: &IntentDecisionEvidenceV1<'_>,
     settings_digest: &ManifestDigest,
     questions: &[tracedecay_domain::configuration::IntentQuestionSettingsV1],
+    window_settings: &tracedecay_domain::configuration::IntentWindowSettingsV1,
+    engine_max_state_bytes: u32,
     engine_identity: Option<&super::decision_engine::DecisionEngineIdentityV1>,
 ) -> Option<String> {
     let IntentDecisionEvidenceV1 {
@@ -1179,10 +1183,19 @@ fn intent_decision_metadata(
                 "first_store_id": window.first_store_id,
                 "last_store_id": window.last_store_id,
                 "message_count": window.messages.len(),
+                // Store the effective bounds, not only a digest of mutable
+                // settings: an exporter must reproduce the original selection.
+                "selection_limits": {
+                    "max_user_turns": window_settings.max_user_turns,
+                    "max_messages": window_settings.max_messages,
+                    "max_bytes": window_settings.max_bytes,
+                    "max_age_secs": window_settings.max_age_secs,
+                },
                 "window_digest": window_digest,
                 "newest_timestamp_micros": window.newest_timestamp_micros,
             },
             "settings_digest": settings_digest,
+            "render_state_limit_bytes": engine_max_state_bytes,
             "model_outcome": engine_outcome.model_outcome(),
             // The byte-exact prepared request is not proof that the remote
             // server received it, nor that an old window remains rehydratable.
@@ -1232,6 +1245,7 @@ mod tool_signal_tests {
 
     use tracedecay_domain::configuration::{
         ContextScoutIntentKindV1 as Kind, IntentDecisionSourceV1, IntentQuestionSettingsV1,
+        IntentWindowSettingsV1,
     };
     use tracedecay_policy::intent_decision::{
         IntentBestAnswerV1, IntentCandidateV1, IntentDecisionInputV1, IntentDecisionV1,
@@ -1290,6 +1304,13 @@ mod tool_signal_tests {
             freshness: SessionWindowFreshnessV1::Unknown,
         };
         let settings_digest = ManifestDigest::from_sha256_bytes(&[8; 32]).unwrap();
+        let window_settings = IntentWindowSettingsV1 {
+            max_user_turns: 2,
+            max_messages: 8,
+            max_bytes: 4096,
+            max_age_secs: 60,
+            freshen_budget_ms: 50,
+        };
         let questions = [IntentQuestionSettingsV1 {
             intent: Kind::Impact,
             instructions: "Does the request ask about impact?".to_owned(),
@@ -1324,6 +1345,8 @@ mod tool_signal_tests {
             },
             &settings_digest,
             &questions,
+            &window_settings,
+            2048,
             None,
         )
         .unwrap();
@@ -1332,6 +1355,13 @@ mod tool_signal_tests {
         assert_eq!(receipt["window"]["last_store_id"], 81);
         assert_eq!(receipt["window"]["message_count"], 1);
         assert!(receipt["window"]["window_digest"].is_string());
+        assert_eq!(receipt["window"]["selection_limits"], serde_json::json!({
+            "max_user_turns": 2,
+            "max_messages": 8,
+            "max_bytes": 4096,
+            "max_age_secs": 60,
+        }));
+        assert_eq!(receipt["render_state_limit_bytes"], 2048);
         assert_eq!(receipt["decision"]["kind"], "below_threshold");
         assert_eq!(receipt["decision"]["best_probability_permille"], 170);
         assert_eq!(receipt["answers"][0]["probability_permille"], 170);
@@ -1361,6 +1391,8 @@ mod tool_signal_tests {
             },
             &settings_digest,
             &questions,
+            &window_settings,
+            2048,
             None,
         )
         .unwrap();
@@ -1388,6 +1420,8 @@ mod tool_signal_tests {
                 },
                 &settings_digest,
                 &questions,
+                &window_settings,
+                2048,
                 None,
             )
             .is_none()

From 9be534775d7e7a9d6008385dd5204b69c43146e6 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 09:29:11 -0800
Subject: [PATCH 63/76] style(scout): format receipt bounds assertion

Refs #4535

Apply rustfmt's required layout to the selection-limits receipt
assertion so the repository formatting gate can pass. Keep the
Kev qualification branch DRAFT under its NO MERGE hold.
---
 .../src/agents/context_scout/intent_pass.rs       | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
index d4885d54b75..2d39c4e7e61 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
@@ -1355,12 +1355,15 @@ mod tool_signal_tests {
         assert_eq!(receipt["window"]["last_store_id"], 81);
         assert_eq!(receipt["window"]["message_count"], 1);
         assert!(receipt["window"]["window_digest"].is_string());
-        assert_eq!(receipt["window"]["selection_limits"], serde_json::json!({
-            "max_user_turns": 2,
-            "max_messages": 8,
-            "max_bytes": 4096,
-            "max_age_secs": 60,
-        }));
+        assert_eq!(
+            receipt["window"]["selection_limits"],
+            serde_json::json!({
+                "max_user_turns": 2,
+                "max_messages": 8,
+                "max_bytes": 4096,
+                "max_age_secs": 60,
+            })
+        );
         assert_eq!(receipt["render_state_limit_bytes"], 2048);
         assert_eq!(receipt["decision"]["kind"], "below_threshold");
         assert_eq!(receipt["decision"]["best_probability_permille"], 170);

From b85961c1c66b387e828e305eab58b8ac99986320 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 09:33:52 -0800
Subject: [PATCH 64/76] fix(scout): preserve precise window selection time in
 receipts

Refs #4535

Keep the microsecond-resolution observed time used by window admission
alongside its limits and digest. Analytics timestamps round to seconds,
so age-boundary replay otherwise cannot reproduce the original window.

Exercise two adjacent instants with identical content/settings while
retaining the no-transcript-text receipt contract.

DRAFT qualification only; NO MERGE.
---
 .../src/agents/context_scout/intent_pass.rs   | 46 ++++++++++++++-----
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
index 2d39c4e7e61..91b4414511f 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
@@ -1078,6 +1078,7 @@ async fn record_intent_decision(
         &settings.questions,
         &settings.window,
         settings.engine.max_state_bytes,
+        pass.observed_at.0,
         engine_identity.as_ref(),
     ) else {
         return;
@@ -1109,6 +1110,7 @@ fn intent_decision_metadata(
     questions: &[tracedecay_domain::configuration::IntentQuestionSettingsV1],
     window_settings: &tracedecay_domain::configuration::IntentWindowSettingsV1,
     engine_max_state_bytes: u32,
+    selection_observed_at_micros: i64,
     engine_identity: Option<&super::decision_engine::DecisionEngineIdentityV1>,
 ) -> Option<String> {
     let IntentDecisionEvidenceV1 {
@@ -1185,6 +1187,10 @@ fn intent_decision_metadata(
                 "message_count": window.messages.len(),
                 // Store the effective bounds, not only a digest of mutable
                 // settings: an exporter must reproduce the original selection.
+                // The age cutoff uses pass.observed_at in microseconds, not
+                // the rounded seconds recorded by analytics. Replaying with
+                // a different clock changes which turns pass the age limit.
+                "selection_observed_at_micros": selection_observed_at_micros,
                 "selection_limits": {
                     "max_user_turns": window_settings.max_user_turns,
                     "max_messages": window_settings.max_messages,
@@ -1331,22 +1337,34 @@ mod tool_signal_tests {
                 probability_permille: 170,
             },
         };
+        let decision_evidence = IntentDecisionEvidenceV1 {
+            window: &window,
+            engine_outcome: &engine,
+            request_commitment: Some(DecisionRequestCommitmentV1 {
+                wire_sha256: [9; 32],
+                wire_bytes: 512,
+            }),
+            facts: &[],
+            prompt_inferences: &[Kind::Impact],
+            decision: &negative,
+        };
         let evidence = intent_decision_metadata(
-            &IntentDecisionEvidenceV1 {
-                window: &window,
-                engine_outcome: &engine,
-                request_commitment: Some(DecisionRequestCommitmentV1 {
-                    wire_sha256: [9; 32],
-                    wire_bytes: 512,
-                }),
-                facts: &[],
-                prompt_inferences: &[Kind::Impact],
-                decision: &negative,
-            },
+            &decision_evidence,
+            &settings_digest,
+            &questions,
+            &window_settings,
+            2048,
+            1_234_567_891,
+            None,
+        )
+        .unwrap();
+        let later = intent_decision_metadata(
+            &decision_evidence,
             &settings_digest,
             &questions,
             &window_settings,
             2048,
+            1_234_567_892,
             None,
         )
         .unwrap();
@@ -1355,6 +1373,10 @@ mod tool_signal_tests {
         assert_eq!(receipt["window"]["last_store_id"], 81);
         assert_eq!(receipt["window"]["message_count"], 1);
         assert!(receipt["window"]["window_digest"].is_string());
+        assert_eq!(receipt["window"]["selection_observed_at_micros"], 1_234_567_891);
+        let later_receipt: serde_json::Value = serde_json::from_str(&later).unwrap();
+        assert_eq!(later_receipt["window"]["selection_observed_at_micros"], 1_234_567_892);
+        assert_ne!(evidence, later);
         assert_eq!(
             receipt["window"]["selection_limits"],
             serde_json::json!({
@@ -1396,6 +1418,7 @@ mod tool_signal_tests {
             &questions,
             &window_settings,
             2048,
+            1_234_567_891,
             None,
         )
         .unwrap();
@@ -1425,6 +1448,7 @@ mod tool_signal_tests {
                 &questions,
                 &window_settings,
                 2048,
+                1_234_567_891,
                 None,
             )
             .is_none()

From b92b46a47a50ad9f0704ead714bc2fa1e24505fb Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 10:47:07 -0800
Subject: [PATCH 65/76] fix(evals): measure single-pick heldout recall and
 repair rustfmt
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Refs #4535

Keep Scout model-only top1 success distinct from any-hint recall. Do
not qualify a single-delivery decision on a buried correct answer.
Align the latest receipt regression with hosted rustfmt evidence.

DRAFT ONLY — DO NOT MERGE.
---
 .../src/agents/context_scout/intent_pass.rs   | 10 ++++--
 evals/intent_engine/HTTP_RUNNER.md            |  8 ++++-
 evals/intent_engine/qualification.py          | 29 ++++++++++++---
 evals/intent_engine/test_qualification.py     | 36 +++++++++++++++++++
 4 files changed, 75 insertions(+), 8 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
index 91b4414511f..c07c2ecfbf3 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs
@@ -1373,9 +1373,15 @@ mod tool_signal_tests {
         assert_eq!(receipt["window"]["last_store_id"], 81);
         assert_eq!(receipt["window"]["message_count"], 1);
         assert!(receipt["window"]["window_digest"].is_string());
-        assert_eq!(receipt["window"]["selection_observed_at_micros"], 1_234_567_891);
+        assert_eq!(
+            receipt["window"]["selection_observed_at_micros"],
+            1_234_567_891
+        );
         let later_receipt: serde_json::Value = serde_json::from_str(&later).unwrap();
-        assert_eq!(later_receipt["window"]["selection_observed_at_micros"], 1_234_567_892);
+        assert_eq!(
+            later_receipt["window"]["selection_observed_at_micros"],
+            1_234_567_892
+        );
         assert_ne!(evidence, later);
         assert_eq!(
             receipt["window"]["selection_limits"],
diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index f0d5712a33b..a2e7cf5d72b 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -257,7 +257,13 @@ question and `error_tolerances` with all four numbers:
 `max_none_false_hint_rate` (0–1), `max_unwanted_hint_rate` (0–1),
 `min_positive_case_recall` (0–1), and `max_p95_ms` (nonnegative milliseconds).
 The unwanted-hint budget counts an incorrect extra suggestion on a positive
-case, even if another correct suggestion made its recall appear successful.
+case, even if another correct suggestion passed its threshold. The frozen
+positive-case recall check uses the **single highest-probability qualifying
+model-only pick** (ties follow frozen question order), not any-hit recall.
+The report retains any-hit recall and all-firing metrics separately. This is
+only a necessary Noul selection check: the model-only comparison does **not**
+replay Scout's deterministic action facts, per-question margins, duplicate
+suppression or delivery state and cannot establish runtime parity.
 These tolerances must be frozen before examining the held-out results.
 
 The `annotations` array must cover every case in the original dataset, in
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 5fc7623e84e..22930da5006 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -331,12 +331,14 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
 
 
 def holdout_decision_metrics(rows, thresholds):
-    """Count unwanted suggestions even when another hint on the same case is correct.
+    """Measure all threshold hits separately from the single model-only pick.
 
-    False hints on negative-only cases are not the full false-positive budget:
-    a positive case can still receive an incorrect additional suggestion.
+    Scout delivers only the first candidate, not every above-threshold answer.
+    This diagnostic omits Scout's action facts, margins and delivery suppression;
+    it is a necessary model-only selection check, not production runtime parity.
     """
     hinted = unwanted = missed_positive = positive = 0
+    selected_correct_positive = selected_unwanted = 0
     for row in rows:
         labels = set(row["labels"])
         fired = {key for key in thresholds if row["probs"][key] >= thresholds[key]}
@@ -344,6 +346,16 @@ def holdout_decision_metrics(rows, thresholds):
         unwanted += bool(fired - labels)
         positive += bool(labels)
         missed_positive += bool(labels and not (fired & labels))
+        # Keep the frozen question order for equal probabilities. The existing
+        # top1 benchmark picks by distance above threshold, which can disagree
+        # with Scout's relevance-first candidate ordering.
+        selected = max(
+            (key for key in thresholds if key in fired),
+            key=lambda key: row["probs"][key],
+            default=None,
+        )
+        selected_correct_positive += bool(labels and selected in labels)
+        selected_unwanted += bool(selected is not None and selected not in labels)
     return {
         "hinted_cases": hinted,
         "abstained_cases": len(rows) - hinted,
@@ -353,9 +365,14 @@ def holdout_decision_metrics(rows, thresholds):
         "unwanted_hint_rate": unwanted / len(rows),
         "missed_positive_cases": missed_positive,
         "missed_positive_rate": missed_positive / positive if positive else None,
+        "model_only_selected_correct_positive_cases": selected_correct_positive,
+        "model_only_selected_positive_case_recall": (
+            selected_correct_positive / positive if positive else None
+        ),
+        "model_only_selected_unwanted_hint_cases": selected_unwanted,
+        "model_only_selected_unwanted_hint_rate": selected_unwanted / len(rows),
     }
 
-
 def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, captured_requests_bytes=None, sample_path=None):
     """Reproduce qualified predictions from the frozen original dataset."""
     from score import score_config
@@ -540,7 +557,9 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
     rates = {
         "none_false_hint_rate": fixed["none_false_hint_rate"],
         "unwanted_hint_rate": decisions["unwanted_hint_rate"],
-        "positive_case_recall": fixed["pos_cases_with_a_correct_hint"],
+        # An any-hit recall can pass even when the highest-ranked suggestion
+        # is wrong. Never qualify a single-delivery application on that number.
+        "positive_case_recall": decisions["model_only_selected_positive_case_recall"],
         "p95_ms": latency["p95_ms"],
     }
     if any(value is None for value in rates.values()):
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 40aeb3f93d5..32c33fce86d 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -214,6 +214,42 @@ def test_unwanted_hints_on_positive_cases_are_not_hidden_by_recall(self):
         self.assertFalse(report["qualified_against_frozen_tolerances"])
 
 
+    def test_single_selected_wrong_hint_blocks_qualification_despite_any_hit_recall(self):
+        # The original all-hits recall is 1.0: impact is over threshold.
+        # Scout-like single selection would deliver the higher wrong category.
+        # Even a precommitted permissive unwanted-hint budget cannot allow the
+        # frozen positive-case recall check to accept that wrong first pick.
+        result = self.signed_result()
+        result["rows"][0]["probs"]["call_graph"] = 0.99
+        frozen = copy.deepcopy(self.manifest)
+        frozen["error_tolerances"]["max_unwanted_hint_rate"] = 1.0
+        manifest_sha = qualification.sha256_json(frozen)
+        result["experiment"]["qualification_manifest_sha256"] = manifest_sha
+        result["experiment_sha256"] = score.evidence_digest(result["experiment"])
+        report = qualification.qualified_report(
+            result, frozen, manifest_sha, self.questions, self.dataset_bytes
+        )
+        self.assertEqual(report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 1)
+        self.assertEqual(report["holdout_decisions"]["unwanted_hint_rate"], 0.5)
+        self.assertEqual(report["holdout_decisions"]["model_only_selected_positive_case_recall"], 0)
+        self.assertEqual(report["holdout_decisions"]["model_only_selected_unwanted_hint_rate"], 0.5)
+        self.assertTrue(report["checks"]["unwanted_hint_rate"])
+        self.assertFalse(report["checks"]["positive_case_recall"])
+        self.assertFalse(report["qualified_against_frozen_tolerances"])
+
+    def test_single_selection_ties_follow_frozen_question_order(self):
+        positive = {"labels": ["impact"], "probs": {"impact": 0.9, "call_graph": 0.9}}
+        correct_first = qualification.holdout_decision_metrics(
+            [positive], {"impact": 0.6, "call_graph": 0.6}
+        )
+        wrong_first = qualification.holdout_decision_metrics(
+            [positive], {"call_graph": 0.6, "impact": 0.6}
+        )
+        self.assertEqual(correct_first["model_only_selected_positive_case_recall"], 1)
+        self.assertEqual(wrong_first["model_only_selected_positive_case_recall"], 0)
+        self.assertEqual(correct_first["unwanted_hint_rate"], 1)
+        self.assertEqual(wrong_first["unwanted_hint_rate"], 1)
+
     def test_native_kev_truncation_is_measured_not_assumed_from_a_flag(self):
         # Native Kev emits state_tokens/state_tokens_used and a top-level
         # truncated flag, not necessarily state_tokens_dropped.

From 07a7ff4054f521e049e23f48929defe82b2e35da Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 11:01:05 -0800
Subject: [PATCH 66/76] fix(evals): align frozen selection gates with Scout
 rounding

---
 evals/intent_engine/HTTP_RUNNER.md        |  9 +++-
 evals/intent_engine/qualification.py      | 31 ++++++++----
 evals/intent_engine/test_qualification.py | 57 +++++++++++++++++++++++
 3 files changed, 87 insertions(+), 10 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index a2e7cf5d72b..49dc6da87df 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -260,7 +260,14 @@ The unwanted-hint budget counts an incorrect extra suggestion on a positive
 case, even if another correct suggestion passed its threshold. The frozen
 positive-case recall check uses the **single highest-probability qualifying
 model-only pick** (ties follow frozen question order), not any-hit recall.
-The report retains any-hit recall and all-firing metrics separately. This is
+The report retains any-hit recall and all-firing metrics separately. The
+qualification gates for negative-case false hints and selected positive-case
+recall emulate Scout's integer-permille threshold rounding and u16 relevance
+rounding (ties remain in frozen question order), rather than treating the
+historical benchmark's floating-point threshold comparison as production.
+Thus a probability of 0.5996 meets Scout's 600-permille threshold, even though
+it is below the historical floating-point threshold of 0.6. The historical
+fixed-threshold metrics remain diagnostic, not acceptance evidence. This is
 only a necessary Noul selection check: the model-only comparison does **not**
 replay Scout's deterministic action facts, per-question margins, duplicate
 suppression or delivery state and cannot establish runtime parity.
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 22930da5006..69d75745c57 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -330,28 +330,39 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     }
 
 
+def runtime_probability_score(probability, scale):
+    """Match Rust's nonnegative f64::round at its fixed-point decision scale."""
+    return math.floor(probability * scale + 0.5)
+
+
 def holdout_decision_metrics(rows, thresholds):
     """Measure all threshold hits separately from the single model-only pick.
 
     Scout delivers only the first candidate, not every above-threshold answer.
-    This diagnostic omits Scout's action facts, margins and delivery suppression;
-    it is a necessary model-only selection check, not production runtime parity.
+    Emulate its permille threshold and u16 relevance rounding, not the historical
+    floating-point benchmark comparison. Action facts, margins and delivery
+    suppression are still omitted; this cannot establish production parity.
     """
     hinted = unwanted = missed_positive = positive = 0
-    selected_correct_positive = selected_unwanted = 0
+    selected_correct_positive = selected_unwanted = none_negative = none_false_hint = 0
     for row in rows:
         labels = set(row["labels"])
-        fired = {key for key in thresholds if row["probs"][key] >= thresholds[key]}
+        fired = {
+            key for key in thresholds
+            if runtime_probability_score(row["probs"][key], 1_000)
+            >= runtime_probability_score(thresholds[key], 1_000)
+        }
+        none_negative += not labels
+        none_false_hint += bool(not labels and fired)
         hinted += bool(fired)
         unwanted += bool(fired - labels)
         positive += bool(labels)
         missed_positive += bool(labels and not (fired & labels))
-        # Keep the frozen question order for equal probabilities. The existing
-        # top1 benchmark picks by distance above threshold, which can disagree
-        # with Scout's relevance-first candidate ordering.
+        # Scout sorts on u16 relevance, not the raw f64. Distinct close
+        # probabilities can quantize to a tie; retain frozen question order.
         selected = max(
             (key for key in thresholds if key in fired),
-            key=lambda key: row["probs"][key],
+            key=lambda key: runtime_probability_score(row["probs"][key], 65_535),
             default=None,
         )
         selected_correct_positive += bool(labels and selected in labels)
@@ -363,6 +374,8 @@ def holdout_decision_metrics(rows, thresholds):
         "abstention_rate": (len(rows) - hinted) / len(rows),
         "unwanted_hint_cases": unwanted,
         "unwanted_hint_rate": unwanted / len(rows),
+        "none_false_hint_cases": none_false_hint,
+        "none_false_hint_rate": none_false_hint / none_negative if none_negative else None,
         "missed_positive_cases": missed_positive,
         "missed_positive_rate": missed_positive / positive if positive else None,
         "model_only_selected_correct_positive_cases": selected_correct_positive,
@@ -555,7 +568,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
         and all(value <= validated["max_input_tokens"] for value in measured_tokens)
     )
     rates = {
-        "none_false_hint_rate": fixed["none_false_hint_rate"],
+        "none_false_hint_rate": decisions["none_false_hint_rate"],
         "unwanted_hint_rate": decisions["unwanted_hint_rate"],
         # An any-hit recall can pass even when the highest-ranked suggestion
         # is wrong. Never qualify a single-delivery application on that number.
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 32c33fce86d..3a0fbcc6397 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -250,6 +250,63 @@ def test_single_selection_ties_follow_frozen_question_order(self):
         self.assertEqual(correct_first["unwanted_hint_rate"], 1)
         self.assertEqual(wrong_first["unwanted_hint_rate"], 1)
 
+    def test_rust_permille_rounding_rejects_false_negative_case_pass(self):
+        result = self.signed_result()
+        # 0.5996 is below the historical float threshold of 0.6, but both
+        # become 600 permille in Scout's actual decision policy.
+        result["rows"][1]["probs"]["call_graph"] = 0.5996
+        report = qualification.qualified_report(
+            result, self.manifest, self.manifest_sha, self.questions,
+            self.dataset_bytes,
+        )
+        self.assertEqual(report["fixed_threshold_metrics"]["none_false_hint_rate"], 0)
+        self.assertEqual(report["holdout_decisions"]["none_false_hint_cases"], 1)
+        self.assertEqual(report["holdout_decisions"]["none_false_hint_rate"], 1)
+        self.assertFalse(report["checks"]["none_false_hint_rate"])
+        self.assertFalse(report["qualified_against_frozen_tolerances"])
+
+    def test_rust_permille_rounding_admits_real_positive_and_refuses_below(self):
+        result = self.signed_result()
+        result["rows"][0]["probs"]["impact"] = 0.5996
+        report = qualification.qualified_report(
+            result, self.manifest, self.manifest_sha, self.questions,
+            self.dataset_bytes,
+        )
+        self.assertEqual(
+            report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 0
+        )
+        self.assertEqual(report["holdout_decisions"]["model_only_selected_positive_case_recall"], 1)
+        self.assertTrue(report["checks"]["positive_case_recall"])
+        self.assertTrue(report["qualified_against_frozen_tolerances"])
+
+        result["rows"][0]["probs"]["impact"] = 0.5994
+        refused = qualification.qualified_report(
+            result, self.manifest, self.manifest_sha, self.questions,
+            self.dataset_bytes,
+        )
+        self.assertEqual(refused["holdout_decisions"]["model_only_selected_positive_case_recall"], 0)
+        self.assertFalse(refused["checks"]["positive_case_recall"])
+
+    def test_scout_u16_relevance_tie_retains_question_order(self):
+        # The later incorrect answer is higher as an f64, but indistinguishable
+        # after Rust rounds relevance to u16.
+        row = {
+            "labels": ["impact"],
+            "probs": {"impact": 0.9, "call_graph": 0.9000000001},
+        }
+        self.assertEqual(
+            qualification.runtime_probability_score(0.9, 65_535),
+            qualification.runtime_probability_score(0.9000000001, 65_535),
+        )
+        correct = qualification.holdout_decision_metrics(
+            [row], {"impact": 0.6, "call_graph": 0.6},
+        )
+        wrong = qualification.holdout_decision_metrics(
+            [row], {"call_graph": 0.6, "impact": 0.6},
+        )
+        self.assertEqual(correct["model_only_selected_positive_case_recall"], 1)
+        self.assertEqual(wrong["model_only_selected_positive_case_recall"], 0)
+
     def test_native_kev_truncation_is_measured_not_assumed_from_a_flag(self):
         # Native Kev emits state_tokens/state_tokens_used and a top-level
         # truncated flag, not necessarily state_tokens_dropped.

From e5dc30b8f8bdeffe815a57df48fffbd14b11b0ab Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 12:37:46 -0800
Subject: [PATCH 67/76] fix(evals): select tied hints in frozen question order
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Refs #4535

Score Scout u16 relevance ties in the frozen question sequence, never
in independently ordered threshold map keys. Reject incomplete question
orders; prove a threshold-map permutation cannot turn a wrong first hint
into passing held-out recall.

DRAFT — NO MERGE; genuine Kev and runtime parity remain unproven.
---
 evals/intent_engine/HTTP_RUNNER.md        |  3 +-
 evals/intent_engine/qualification.py      | 14 ++++++---
 evals/intent_engine/test_qualification.py | 38 ++++++++++++++++++++---
 3 files changed, 45 insertions(+), 10 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 49dc6da87df..f80108e15c1 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -263,7 +263,8 @@ model-only pick** (ties follow frozen question order), not any-hit recall.
 The report retains any-hit recall and all-firing metrics separately. The
 qualification gates for negative-case false hints and selected positive-case
 recall emulate Scout's integer-permille threshold rounding and u16 relevance
-rounding (ties remain in frozen question order), rather than treating the
+rounding (ties follow the frozen question vocabulary order, **not** the
+threshold JSON object's key order), rather than treating the
 historical benchmark's floating-point threshold comparison as production.
 Thus a probability of 0.5996 meets Scout's 600-permille threshold, even though
 it is below the historical floating-point threshold of 0.6. The historical
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 69d75745c57..9b7c8940028 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -335,14 +335,18 @@ def runtime_probability_score(probability, scale):
     return math.floor(probability * scale + 0.5)
 
 
-def holdout_decision_metrics(rows, thresholds):
+def holdout_decision_metrics(rows, thresholds, question_order):
     """Measure all threshold hits separately from the single model-only pick.
 
     Scout delivers only the first candidate, not every above-threshold answer.
     Emulate its permille threshold and u16 relevance rounding, not the historical
-    floating-point benchmark comparison. Action facts, margins and delivery
-    suppression are still omitted; this cannot establish production parity.
+    floating-point benchmark comparison. Stable relevance ties must use the
+    frozen question sequence, never the independently serialized threshold map
+    order. Action facts, margins and delivery suppression are still omitted;
+    this cannot establish production parity.
     """
+    if len(question_order) != len(thresholds) or set(question_order) != set(thresholds):
+        raise ValueError("model-only selection requires the complete frozen question order")
     hinted = unwanted = missed_positive = positive = 0
     selected_correct_positive = selected_unwanted = none_negative = none_false_hint = 0
     for row in rows:
@@ -361,7 +365,7 @@ def holdout_decision_metrics(rows, thresholds):
         # Scout sorts on u16 relevance, not the raw f64. Distinct close
         # probabilities can quantize to a tie; retain frozen question order.
         selected = max(
-            (key for key in thresholds if key in fired),
+            (key for key in question_order if key in fired),
             key=lambda key: runtime_probability_score(row["probs"][key], 65_535),
             default=None,
         )
@@ -553,7 +557,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
     scores = score_config(result, fixed_t=validated["thresholds"], cats=tuple(questions))
     fixed = scores["fixed"]["all"]
     latency = scores["latency"]
-    decisions = holdout_decision_metrics(rows, validated["thresholds"])
+    decisions = holdout_decision_metrics(rows, validated["thresholds"], tuple(questions))
     tolerances = validated["error_tolerances"]
     # A JSON byte ceiling does not bound tokenizer-specific context usage.
     # An unknown or zero server-reported count is not proof of fitting inside
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 3a0fbcc6397..cde3c026aa5 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -240,16 +240,46 @@ def test_single_selected_wrong_hint_blocks_qualification_despite_any_hit_recall(
     def test_single_selection_ties_follow_frozen_question_order(self):
         positive = {"labels": ["impact"], "probs": {"impact": 0.9, "call_graph": 0.9}}
         correct_first = qualification.holdout_decision_metrics(
-            [positive], {"impact": 0.6, "call_graph": 0.6}
+            [positive], {"impact": 0.6, "call_graph": 0.6}, ("impact", "call_graph"),
         )
         wrong_first = qualification.holdout_decision_metrics(
-            [positive], {"call_graph": 0.6, "impact": 0.6}
+            [positive], {"call_graph": 0.6, "impact": 0.6}, ("call_graph", "impact"),
         )
         self.assertEqual(correct_first["model_only_selected_positive_case_recall"], 1)
         self.assertEqual(wrong_first["model_only_selected_positive_case_recall"], 0)
         self.assertEqual(correct_first["unwanted_hint_rate"], 1)
         self.assertEqual(wrong_first["unwanted_hint_rate"], 1)
 
+    def test_threshold_manifest_key_order_cannot_change_scout_tie_selection(self):
+        # Production policy retains its configured question sequence on tied
+        # u16 relevance. The threshold object is an independent JSON map;
+        # changing only its key order must never turn a wrong first hint
+        # into a passing held-out positive case.
+        self.assertLess(
+            list(self.questions).index("call_graph"),
+            list(self.questions).index("impact"),
+        )
+        result = self.signed_result()
+        result["rows"][0]["probs"]["call_graph"] = 0.9
+        frozen = copy.deepcopy(self.manifest)
+        frozen["thresholds"] = dict(reversed(list(frozen["thresholds"].items())))
+        self.assertLess(
+            list(frozen["thresholds"]).index("impact"),
+            list(frozen["thresholds"]).index("call_graph"),
+        )
+        frozen["error_tolerances"]["max_unwanted_hint_rate"] = 1.0
+        manifest_sha = qualification.sha256_json(frozen)
+        result["experiment"]["qualification_manifest_sha256"] = manifest_sha
+        result["experiment_sha256"] = score.evidence_digest(result["experiment"])
+        report = qualification.qualified_report(
+            result, frozen, manifest_sha, self.questions, self.dataset_bytes,
+        )
+        self.assertEqual(
+            report["holdout_decisions"]["model_only_selected_positive_case_recall"], 0,
+        )
+        self.assertFalse(report["checks"]["positive_case_recall"])
+        self.assertFalse(report["qualified_against_frozen_tolerances"])
+
     def test_rust_permille_rounding_rejects_false_negative_case_pass(self):
         result = self.signed_result()
         # 0.5996 is below the historical float threshold of 0.6, but both
@@ -299,10 +329,10 @@ def test_scout_u16_relevance_tie_retains_question_order(self):
             qualification.runtime_probability_score(0.9000000001, 65_535),
         )
         correct = qualification.holdout_decision_metrics(
-            [row], {"impact": 0.6, "call_graph": 0.6},
+            [row], {"impact": 0.6, "call_graph": 0.6}, ("impact", "call_graph"),
         )
         wrong = qualification.holdout_decision_metrics(
-            [row], {"call_graph": 0.6, "impact": 0.6},
+            [row], {"call_graph": 0.6, "impact": 0.6}, ("call_graph", "impact"),
         )
         self.assertEqual(correct["model_only_selected_positive_case_recall"], 1)
         self.assertEqual(wrong["model_only_selected_positive_case_recall"], 0)

From c1adb1fac83703389e8202b19a399d4c11e5e4cc Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 13:30:42 -0800
Subject: [PATCH 68/76] fix(evals): freeze Scout margins in captured
 qualification

---
 evals/intent_engine/HTTP_RUNNER.md        | 21 ++++--
 evals/intent_engine/qualification.py      | 54 ++++++++++++--
 evals/intent_engine/test_qualification.py | 90 +++++++++++++++++++++++
 3 files changed, 152 insertions(+), 13 deletions(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index f80108e15c1..e3b2849568b 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -269,9 +269,13 @@ historical benchmark's floating-point threshold comparison as production.
 Thus a probability of 0.5996 meets Scout's 600-permille threshold, even though
 it is below the historical floating-point threshold of 0.6. The historical
 fixed-threshold metrics remain diagnostic, not acceptance evidence. This is
-only a necessary Noul selection check: the model-only comparison does **not**
-replay Scout's deterministic action facts, per-question margins, duplicate
-suppression or delivery state and cannot establish runtime parity.
+only a necessary Noul selection check: captured Scout arms additionally use
+frozen `margin_permille` values and compare against **all** other answered
+Noul questions, including those below their own threshold, matching Rust's
+saturating integer-permille margin calculation. Historical arms have no margin
+policy. The model-only comparison still does **not** replay Scout's deterministic
+action facts, duplicate suppression or delivery state and cannot establish
+runtime parity.
 These tolerances must be frozen before examining the held-out results.
 
 The `annotations` array must cover every case in the original dataset, in
@@ -403,8 +407,15 @@ been recomputed. Only the inner request must be canonical; the outer evidence
 file preserves its original hashed bytes. This wire-shape check does **not**
 attest that the bytes actually originated from Scout.
 
-The manifest must set `arm.config` to `S_scout_captured` and freeze a
-positive integer `max_state_bytes` from the actual production
+The manifest must set `arm.config` to `S_scout_captured` and freeze
+`margin_permille` as a complete key-to-value map in configured question order:
+each value is either `null` (no margin) or an integer from 0 through 1000
+matching the effective production `IntentQuestionSettingsV1.margin_permille`.
+The HTTP request body does **not** contain these local policy settings; they
+must be obtained from the same settings authority and independently verified,
+not inferred from the model predictions. Missing, incomplete, malformed or
+historical-arm margin declarations fail before HTTP inference. The arm also
+freezes a positive integer `max_state_bytes` from the actual production
 `DecisionEngineConfigV1` / effective Scout settings. This limits the UTF-8
 bytes in the decoded `state` field separately from the whole HTTP JSON byte
 ceiling. The Rust client keeps only the newest state tail (on a character
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 9b7c8940028..2d99e647a01 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -200,6 +200,19 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     elif state_limit is not None:
         raise ValueError("only captured Scout arms may declare max_state_bytes")
 
+    # Margins belong to the local intent policy, not the Noul HTTP body.
+    # Without freezing them a captured arm can pass on hints that Scout
+    # would suppress when a close competing answer is present.
+    margins = manifest.get("margin_permille")
+    if arm["config"] == CAPTURED_SCOUT_CONFIG:
+        if not isinstance(margins, dict) or set(margins) != set(questions):
+            raise ValueError("captured Scout arm requires a complete frozen margin_permille map")
+        for key, margin in margins.items():
+            if margin is not None and (type(margin) is not int or not 0 <= margin <= 1_000):
+                raise ValueError(f"captured Scout margin_permille for {key} must be null or 0..1000")
+    elif margins is not None:
+        raise ValueError("only captured Scout arms may declare margin_permille")
+
     request_limit = manifest.get("max_request_bytes")
     if (isinstance(request_limit, bool) or not isinstance(request_limit, int)
             or request_limit <= 0):
@@ -323,6 +336,7 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
         "max_state_bytes": state_limit,
         "max_input_tokens": token_limit,
         "thresholds": thresholds,
+        "margin_permille": margins,
         "noul_temperature": client_temperature,
         "error_tolerances": tolerances,
         "annotations": {a["id"]: a for a in annotations},
@@ -335,27 +349,48 @@ def runtime_probability_score(probability, scale):
     return math.floor(probability * scale + 0.5)
 
 
-def holdout_decision_metrics(rows, thresholds, question_order):
+def holdout_decision_metrics(rows, thresholds, question_order, margin_permille=None):
     """Measure all threshold hits separately from the single model-only pick.
 
     Scout delivers only the first candidate, not every above-threshold answer.
     Emulate its permille threshold and u16 relevance rounding, not the historical
     floating-point benchmark comparison. Stable relevance ties must use the
     frozen question sequence, never the independently serialized threshold map
-    order. Action facts, margins and delivery suppression are still omitted;
-    this cannot establish production parity.
+    order. Captured arms also apply frozen per-question margins against every
+    usable model answer, even when that competitor is below its own threshold.
+    Action facts and delivery suppression are still omitted; this cannot
+    establish production parity.
     """
     if len(question_order) != len(thresholds) or set(question_order) != set(thresholds):
         raise ValueError("model-only selection requires the complete frozen question order")
+    if margin_permille is not None and (
+        not isinstance(margin_permille, dict)
+        or set(margin_permille) != set(question_order)
+        or any(value is not None and (type(value) is not int or not 0 <= value <= 1_000)
+               for value in margin_permille.values())
+    ):
+        raise ValueError("model-only margins must cover frozen questions in permille")
     hinted = unwanted = missed_positive = positive = 0
     selected_correct_positive = selected_unwanted = none_negative = none_false_hint = 0
     for row in rows:
         labels = set(row["labels"])
-        fired = {
-            key for key in thresholds
-            if runtime_probability_score(row["probs"][key], 1_000)
-            >= runtime_probability_score(thresholds[key], 1_000)
+        scores = {
+            key: runtime_probability_score(row["probs"][key], 1_000)
+            for key in question_order
         }
+        fired = set()
+        for key in question_order:
+            if scores[key] < runtime_probability_score(thresholds[key], 1_000):
+                continue
+            margin = margin_permille[key] if margin_permille is not None else None
+            if margin is not None:
+                # Rust's runner-up is any other answered question, not only
+                # a competitor that passed its own threshold. Saturate at zero.
+                runner_up = max((value for other, value in scores.items()
+                                 if other != key), default=0)
+                if max(0, scores[key] - runner_up) < margin:
+                    continue
+            fired.add(key)
         none_negative += not labels
         none_false_hint += bool(not labels and fired)
         hinted += bool(fired)
@@ -557,7 +592,10 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
     scores = score_config(result, fixed_t=validated["thresholds"], cats=tuple(questions))
     fixed = scores["fixed"]["all"]
     latency = scores["latency"]
-    decisions = holdout_decision_metrics(rows, validated["thresholds"], tuple(questions))
+    decisions = holdout_decision_metrics(
+        rows, validated["thresholds"], tuple(questions),
+        validated["margin_permille"],
+    )
     tolerances = validated["error_tolerances"]
     # A JSON byte ceiling does not bound tokenizer-specific context usage.
     # An unknown or zero server-reported count is not proof of fitting inside
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index cde3c026aa5..31a6f87f2b7 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -2281,6 +2281,8 @@ def captured_scout_fixture(self):
         manifest = copy.deepcopy(self.manifest)
         manifest["arm"]["config"] = qualification.CAPTURED_SCOUT_CONFIG
         manifest["max_state_bytes"] = 4096
+        # The HTTP wire does not contain the local Scout selection margins.
+        manifest["margin_permille"] = {key: None for key in self.questions}
         manifest["captured_requests_sha256"] = qualification.sha256_bytes(capture_bytes)
         manifest["holdout_requests_sha256"] = score.evidence_digest([
             [record["id"], hashlib.sha256(record["body"].encode("utf-8")).hexdigest()]
@@ -2288,6 +2290,94 @@ def captured_scout_fixture(self):
         ])
         return capture_bytes, manifest
 
+    def test_captured_scout_margins_are_frozen_before_network(self):
+        capture, manifest = self.captured_scout_fixture()
+        self.valid(manifest)
+        variants = [
+            ("missing", lambda m: m.pop("margin_permille")),
+            ("incomplete", lambda m: m["margin_permille"].pop("impact")),
+            ("negative", lambda m: m["margin_permille"].update(impact=-1)),
+            ("over-limit", lambda m: m["margin_permille"].update(impact=1001)),
+            ("fraction", lambda m: m["margin_permille"].update(impact=0.2)),
+            ("boolean", lambda m: m["margin_permille"].update(impact=True)),
+        ]
+        for name, mutate in variants:
+            with self.subTest(name=name), tempfile.TemporaryDirectory() as temp:
+                changed = copy.deepcopy(manifest)
+                mutate(changed)
+                here = Path(temp)
+                (here / "manifest.json").write_text(json.dumps(changed))
+                (here / "dataset.json").write_bytes(self.dataset_bytes)
+                (here / "captured.json").write_bytes(capture)
+                (here / "server.json").write_text(json.dumps(self.server_config))
+                with patch.object(run_http, "HERE", str(here)), patch.object(
+                    run_http, "post"
+                ) as post:
+                    with self.assertRaisesRegex(ValueError, "margin_permille"):
+                        run_http.main([
+                            "--url", self.endpoint, "--tag", "margin-refusal",
+                            "--server-config", str(here / "server.json"),
+                            "--qualification-manifest", str(here / "manifest.json"),
+                            "--captured-requests", str(here / "captured.json"),
+                            qualification.CAPTURED_SCOUT_CONFIG,
+                        ])
+                    post.assert_not_called()
+                self.assertFalse((here / "results").exists())
+
+        # Historical benchmark arms must not silently reinterpret a policy
+        # margin declaration as a property of their synthetic renderer.
+        historical = copy.deepcopy(self.manifest)
+        historical["margin_permille"] = {key: None for key in self.questions}
+        with self.assertRaisesRegex(ValueError, "only captured Scout"):
+            self.valid(historical)
+
+    def test_model_only_scout_margin_uses_unqualified_runner_up(self):
+        thresholds = {"impact": 0.6, "call_graph": 0.95}
+        questions = ("impact", "call_graph")
+        rows = [{"labels": ["impact"], "probs": {
+            "impact": 0.90, "call_graph": 0.89,
+        }}]
+        # The competing 890-permille answer is BELOW its own 950 threshold.
+        # Rust still uses it as the runner-up for impact's 20-permille margin.
+        baseline = qualification.holdout_decision_metrics(
+            rows, thresholds, questions
+        )
+        self.assertEqual(baseline["model_only_selected_positive_case_recall"], 1)
+        gated = qualification.holdout_decision_metrics(
+            rows, thresholds, questions,
+            {"impact": 20, "call_graph": None},
+        )
+        self.assertEqual(gated["model_only_selected_positive_case_recall"], 0)
+        self.assertEqual(gated["missed_positive_cases"], 1)
+        self.assertEqual(gated["abstained_cases"], 1)
+        # A true lead of at least 10 permille is accepted.
+        admitted = qualification.holdout_decision_metrics(
+            rows, thresholds, questions,
+            {"impact": 10, "call_graph": None},
+        )
+        self.assertEqual(admitted["model_only_selected_positive_case_recall"], 1)
+        with self.assertRaisesRegex(ValueError, "margins must cover"):
+            qualification.holdout_decision_metrics(
+                rows, thresholds, questions, {"impact": 20}
+            )
+
+    def test_margin_rejects_wrong_selected_hint_at_rounded_boundary(self):
+        rows = [{"labels": ["impact"], "probs": {
+            "impact": 0.9004, "call_graph": 0.9000,
+        }}]
+        thresholds = {"impact": 0.6, "call_graph": 0.6}
+        questions = ("impact", "call_graph")
+        baseline = qualification.holdout_decision_metrics(rows, thresholds, questions)
+        self.assertEqual(baseline["model_only_selected_positive_case_recall"], 1)
+        # Both probabilities round to 900-permille. A nonzero margin
+        # suppresses the correct intent; the competing no-margin intent wins.
+        gated = qualification.holdout_decision_metrics(
+            rows, thresholds, questions,
+            {"impact": 1, "call_graph": None},
+        )
+        self.assertEqual(gated["model_only_selected_positive_case_recall"], 0)
+        self.assertEqual(gated["model_only_selected_unwanted_hint_cases"], 1)
+
     def test_captured_scout_accepts_canonical_wire_order_not_settings_vector_order(self):
         capture, manifest = self.captured_scout_fixture()
         self.assertNotEqual(list(self.questions), sorted(self.questions),

From 79c3aae84f42fa3c075019997ac9f8bb667aa293 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 13:57:58 -0800
Subject: [PATCH 69/76] fix(evals): forbid client calibration in captured Scout
 parity

Refs #4535

Production Scout forwards raw Noul probabilities. A separately calibrated
captured arm could pass frozen positive recall despite a raw below-threshold
answer. Refuse this evaluator-only transform before inference or scoring;
retain calibration for historical non-Scout arms.
---
 evals/intent_engine/HTTP_RUNNER.md        |  7 +++
 evals/intent_engine/qualification.py      |  7 +++
 evals/intent_engine/test_qualification.py | 73 ++++++++++++++++++++++-
 3 files changed, 86 insertions(+), 1 deletion(-)

diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index e3b2849568b..7b94915e91b 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -60,6 +60,13 @@ Do not put credentials in this file. The declaration is operator-supplied
 provenance, **not remote attestation**: update it whenever the deployment changes.
 The optional `--noul-temperature` is a separate client-side calibration step;
 it must be finite and strictly positive.
+For captured Scout parity, the frozen `noul_temperature` must be `null` and
+`--noul-temperature` must be omitted. Production Scout forwards the server's
+raw Noul probabilities to the intent policy without client calibration.
+Applying evaluator-only calibration could turn a below-threshold raw answer
+into a passing held-out result that Scout would never deliver. Historical
+non-Scout evaluation arms may still use frozen calibration; server-side
+`server_config.temperature` is a separate declaration.
 
 New raw-response samples, resumable `.partial` logs and completed result
 files are created owner-readable/writable only (mode 0600 on POSIX), even
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index 2d99e647a01..ca37863e6d3 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -244,6 +244,13 @@ def validate_manifest(manifest, dataset_sha, questions, cases=None):
     ):
         raise ValueError("qualification client calibration must be null or positive finite")
 
+    # The production Scout client forwards the server's Noul probability
+    # unchanged. Historical evaluator calibration is not a runtime feature:
+    # applying it here can turn a below-threshold raw answer into a passing
+    # model-only held-out suggestion that Scout would never produce.
+    if arm["config"] == CAPTURED_SCOUT_CONFIG and client_temperature is not None:
+        raise ValueError("captured Scout parity forbids client-side Noul calibration")
+
     tolerances = manifest.get("error_tolerances")
     required = ("max_none_false_hint_rate", "max_unwanted_hint_rate", "min_positive_case_recall", "max_p95_ms")
     if not isinstance(tolerances, dict) or set(tolerances) != set(required):
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 31a6f87f2b7..328e2e62730 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -14,7 +14,7 @@
 import qualification
 import run_http
 import score
-from common import CATS, noul_questions
+from common import CATS, calibrated, noul_questions
 
 
 class QualificationTests(unittest.TestCase):
@@ -2290,6 +2290,77 @@ def captured_scout_fixture(self):
         ])
         return capture_bytes, manifest
 
+    def test_captured_scout_rejects_calibration_false_pass_before_inference_and_scoring(self):
+        # Scout forwards raw Noul probabilities; the optional historical
+        # calibration could make 0.55 clear a 0.60 production threshold.
+        capture, manifest = self.captured_scout_fixture()
+        self.valid(manifest)
+        self.assertLess(0.55, manifest["thresholds"]["impact"])
+        self.assertGreater(calibrated(0.55, 0.25), manifest["thresholds"]["impact"])
+        forged = copy.deepcopy(manifest)
+        forged["noul_temperature"] = 0.25
+        forged_sha = qualification.sha256_json(forged)
+
+        # Build a coherently refrozen *captured* result rather than relying on
+        # an unrelated config mismatch to make the independent scorer fail.
+        result = self.signed_result()
+        records = qualification.loads_unambiguous_json(capture)
+        for row, record in zip(result["rows"], records):
+            raw = {key: (0.55 if key in row["labels"] else 0.1)
+                   for key in self.questions}
+            wire = record["body"].encode("utf-8")
+            row["probs_raw"] = raw
+            row["probs"] = {key: calibrated(p, 0.25) for key, p in raw.items()}
+            row["request_sha256"] = qualification.sha256_bytes(wire)
+            row["request_size_bytes"] = len(wire)
+        result["config"] = "trial_" + qualification.CAPTURED_SCOUT_CONFIG
+        experiment = result["experiment"]
+        experiment["config"] = qualification.CAPTURED_SCOUT_CONFIG
+        experiment["noul_temperature"] = 0.25
+        experiment["qualification_manifest_sha256"] = forged_sha
+        experiment["requests_sha256"] = score.evidence_digest([
+            [row["id"], row["request_sha256"]] for row in result["rows"]
+        ])
+        result["experiment_sha256"] = score.evidence_digest(experiment)
+        self.assertEqual(experiment["requests_sha256"], forged["holdout_requests_sha256"])
+        self.assertEqual(
+            qualification.holdout_decision_metrics(
+                [{**row, "probs": row["probs_raw"]} for row in result["rows"]],
+                forged["thresholds"], tuple(self.questions), forged["margin_permille"],
+            )["model_only_selected_positive_case_recall"], 0,
+        )
+        self.assertEqual(
+            qualification.holdout_decision_metrics(
+                result["rows"], forged["thresholds"], tuple(self.questions),
+                forged["margin_permille"],
+            )["model_only_selected_positive_case_recall"], 1,
+        )
+        with self.assertRaisesRegex(ValueError, "captured Scout parity forbids"):
+            qualification.qualified_report(
+                result, forged, forged_sha, self.questions, self.dataset_bytes, capture,
+            )
+
+        with tempfile.TemporaryDirectory() as temp:
+            here = Path(temp)
+            (here / "dataset.json").write_bytes(self.dataset_bytes)
+            (here / "manifest.json").write_text(json.dumps(forged))
+            (here / "captured.json").write_bytes(capture)
+            (here / "server.json").write_text(json.dumps(self.server_config))
+            with patch.object(run_http, "HERE", str(here)), patch.object(
+                run_http, "post"
+            ) as post:
+                with self.assertRaisesRegex(ValueError, "captured Scout parity forbids"):
+                    run_http.main([
+                        "--url", self.endpoint, "--tag", "calibration-parity",
+                        "--server-config", str(here / "server.json"),
+                        "--qualification-manifest", str(here / "manifest.json"),
+                        "--captured-requests", str(here / "captured.json"),
+                        "--noul-temperature", "0.25",
+                        qualification.CAPTURED_SCOUT_CONFIG,
+                    ])
+                post.assert_not_called()
+            self.assertFalse((here / "results").exists())
+
     def test_captured_scout_margins_are_frozen_before_network(self):
         capture, manifest = self.captured_scout_fixture()
         self.valid(manifest)

From 6ac8e822d1cc58146d5157a575ceab0a7f28e8c1 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 15:21:47 -0800
Subject: [PATCH 70/76] fix(evals): verify raw Scout samples against scored
 evidence

Refs #4535

A first response sample and scored row could mutually rehash their digest
commitments while disagreeing on Noul probabilities, reported model or
usage. Reconstruct these fields from the raw sample during independent
scoring and resume/final publication; refuse mismatches before further
held-out inference. Add a falsifiable rehashed-evidence regression.
---
 evals/intent_engine/qualification.py      |  1 +
 evals/intent_engine/run_http.py           | 50 ++++++++++++++++++--
 evals/intent_engine/test_qualification.py | 57 +++++++++++++++++++++++
 3 files changed, 105 insertions(+), 3 deletions(-)

diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index ca37863e6d3..d52f7109e5d 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -573,6 +573,7 @@ def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, c
     if has_sample_commitment:
         response_sha, row_sha = validate_qualified_sample(
             sample_path, result["experiment_sha256"], reconstructed_requests,
+            expected_row=first_row,
         )
         if (first_row["sample_response_sha256"] != response_sha
                 or sha256_json(first_row) != row_sha):
diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 4a8eb9933b6..299d8688865 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -52,7 +52,7 @@ def request_bytes(body):
     return json.dumps(body, allow_nan=False).encode("utf-8")
 
 
-def validate_qualified_sample(path, experiment_sha, requests):
+def validate_qualified_sample(path, experiment_sha, requests, expected_row=None):
     """Refuse a stale first-response sample during an exact qualified resume.
 
     Raw samples are debugging evidence, not independent model attestation.
@@ -123,7 +123,46 @@ def validate_qualified_sample(path, experiment_sha, requests):
         )
     if not matches_request:
         raise ValueError("qualified sample request differs from frozen HTTP bytes")
-    noul_probabilities(sample.get("response"), sample["request"]["questions"])
+    raw_response = sample.get("response")
+    raw_probabilities = noul_probabilities(raw_response, sample["request"]["questions"])
+    if expected_row is not None:
+        # A response digest in the row and a row digest in the sample can both
+        # be rewritten together. Independently replay the raw response against
+        # the scored first row so such self-consistent edits cannot certify
+        # invented probabilities or favorable usage/cost telemetry.
+        usage = raw_response.get("usage") or {}
+        if not isinstance(usage, dict):
+            raise ValueError("qualified sample has invalid raw usage")
+        server_ms = raw_response.get("latency_ms")
+        if server_ms is None:
+            timing = raw_response.get("timing") or {}
+            if not isinstance(timing, dict):
+                raise ValueError("qualified sample has invalid raw timing")
+            server_ms = timing.get("inference_ms")
+        recorded_raw = expected_row.get("probs_raw", expected_row.get("probs"))
+        reconstructed = {
+            "observed_model": raw_response.get("model"),
+            "input_tokens": usage.get("input_tokens"),
+            "state_tokens": usage.get("state_tokens"),
+            "dropped": dropped_state_tokens(raw_response, usage),
+            "server_ms": server_ms,
+        }
+        optional = (
+            ("state_tokens_used", "state_tokens_used"),
+            ("state_tokens_dropped", "reported_state_tokens_dropped"),
+            ("truncated", "usage_truncated"),
+        )
+        if (
+            recorded_raw != raw_probabilities
+            or any(expected_row.get(key) != value for key, value in reconstructed.items())
+            or any((name in usage) != (stored in expected_row)
+                   or (name in usage and usage[name] != expected_row[stored])
+                   for name, stored in optional)
+            or ("truncated" in raw_response) != ("truncated" in expected_row)
+            or ("truncated" in raw_response
+                and raw_response["truncated"] != expected_row["truncated"])
+        ):
+            raise ValueError("qualified sample raw response disagrees with first scored row")
     row_sha = sample.get("row_sha256")
     if not isinstance(row_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", row_sha):
         raise ValueError("qualified sample lacks the completed first-row commitment")
@@ -825,6 +864,10 @@ def main(argv=None):
                         raise ValueError("qualified sample has no completed first-response row; use a new tag")
                     response_sha, row_sha = admitted_sample
                     first_row = done[first_id]
+                    validate_qualified_sample(
+                        args.sample_out, header["experiment_sha256"], requests,
+                        expected_row=first_row,
+                    )
                     if (first_row.get("sample_response_sha256") != response_sha
                             or digest(first_row) != row_sha):
                         raise ValueError("qualified sample response or row disagrees with completed evidence")
@@ -927,10 +970,11 @@ def main(argv=None):
                 # while subsequent held-out HTTP calls are in flight. Recheck
                 # the original first response against the durable first row
                 # immediately before exposing a completed qualification.
+                first_row = done[next(iter(cases))]
                 response_sha, row_sha = validate_qualified_sample(
                     args.sample_out, header["experiment_sha256"], requests,
+                    expected_row=first_row,
                 )
-                first_row = done[next(iter(cases))]
                 if (first_row.get("sample_response_sha256") != response_sha
                         or digest(first_row) != row_sha):
                     raise ValueError(
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index 328e2e62730..ee485cbcd20 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -722,6 +722,7 @@ def test_independent_qualified_scoring_rechecks_first_response_sample(self):
                 key: {"type": "noul", "noul": value}
                 for key, value in first["probs"].items()
             },
+            "usage": {"input_tokens": 64, "state_tokens_dropped": 0},
         }
         first["sample_response_sha256"] = run_http.digest(response)
         sample = {
@@ -778,6 +779,62 @@ def score_sample(value=result):
             with self.assertRaisesRegex(ValueError, "must be provided together"):
                 score_sample(without_commitment)
 
+    def test_rehashed_first_sample_cannot_disagree_with_scored_prediction_or_usage(self):
+        # A sample's row_sha256 and a row's sample_response_sha256 are
+        # mutually rehashable. The original raw response must independently
+        # reproduce the frozen model observation, probabilities, and telemetry.
+        result = self.signed_result()
+        first = result["rows"][0]
+        request = {
+            "state": run_http.STATES["B_noul_text"](self.dataset[2]),
+            "questions": self.questions,
+        }
+        raw_response = {
+            "model": first["observed_model"],
+            "answers": {
+                key: {"type": "noul", "noul": value}
+                for key, value in first["probs"].items()
+            },
+            "usage": {"input_tokens": 64, "state_tokens_dropped": 0},
+        }
+        with tempfile.TemporaryDirectory() as temp:
+            path = Path(temp) / "first-response.json"
+
+            def score_raw(response):
+                adjusted = copy.deepcopy(result)
+                adjusted_first = adjusted["rows"][0]
+                adjusted_first["sample_response_sha256"] = run_http.digest(response)
+                evidence = {
+                    "request": request,
+                    "response": response,
+                    "experiment_sha256": adjusted["experiment_sha256"],
+                    "case_id": adjusted_first["id"],
+                    "request_sha256": adjusted_first["request_sha256"],
+                    "row_sha256": run_http.digest(adjusted_first),
+                }
+                path.write_text(json.dumps(evidence), encoding="utf-8")
+                return qualification.qualified_report(
+                    adjusted, self.manifest, self.manifest_sha,
+                    self.questions, self.dataset_bytes, sample_path=str(path),
+                )
+
+            self.assertTrue(score_raw(raw_response)["qualified_against_frozen_tolerances"])
+            for corruption in ("probabilities", "model", "input_tokens", "dropped"):
+                with self.subTest(corruption=corruption):
+                    edited = copy.deepcopy(raw_response)
+                    if corruption == "probabilities":
+                        edited["answers"]["impact"]["noul"] = 0.01
+                    elif corruption == "model":
+                        edited["model"] = "different-checkpoint"
+                    elif corruption == "input_tokens":
+                        edited["usage"]["input_tokens"] = 1024
+                    else:
+                        edited["usage"]["state_tokens_dropped"] = 1
+                    # Rehash *both* directions; these records used to pass
+                    # despite contradicting the first response actually scored.
+                    with self.assertRaisesRegex(ValueError, "raw response disagrees"):
+                        score_raw(edited)
+
     def test_qualified_scorer_rejects_edited_choice_and_extra_none_evidence(self):
         # These edits previously passed the generic historical scorer. An
         # independently scored Noul-only result must reject them even when

From 0a7381c4c10cc725037a6bcaf7c36db565db4811 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 15:51:39 -0800
Subject: [PATCH 71/76] fix(scout): require typed Choice picks at the
 probability maximum
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Refs #4535; partial decoder prerequisite only. DRAFT — NO MERGE.
---
 .../agents/context_scout/decision_engine.rs   | 61 +++++++++++++++----
 1 file changed, 49 insertions(+), 12 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index c1fdab444c2..004fcfd3ce1 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -41,7 +41,7 @@ const MAX_DECISION_RESPONSE_BYTES: usize = 1024 * 1024;
 const SCORE_EPSILON: f64 = 1e-6;
 
 /// Decision servers return a probability distribution, not independent scores.
-/// Allow a small rounding error when servers serialize decimal probabilities.
+/// Allow rounding error in the distribution total, not a conflicting chosen option.
 const CHOICE_EPSILON: f64 = 1e-3;
 
 /// What one question asks.
@@ -530,7 +530,7 @@ fn parse_answer(
             }
             let chosen = match answer.get("choice") {
                 None | Some(Value::Null) => None,
-                Some(value) => Some(label_text(value)?),
+                Some(value) => Some(value.as_str()?.to_owned()),
             };
             if chosen
                 .as_ref()
@@ -553,7 +553,7 @@ fn parse_answer(
                 let selected_probability = *probabilities.get(chosen)?;
                 if probabilities
                     .values()
-                    .any(|probability| *probability > selected_probability + CHOICE_EPSILON)
+                    .any(|probability| *probability > selected_probability)
                 {
                     return None;
                 }
@@ -608,15 +608,6 @@ fn unit_interval(value: &Value) -> Option<f64> {
         .filter(|probability| (0.0..=1.0).contains(probability))
 }
 
-fn label_text(value: &Value) -> Option<String> {
-    match value {
-        Value::String(text) => Some(text.clone()),
-        Value::Number(number) => Some(number.to_string()),
-        Value::Bool(flag) => Some(flag.to_string()),
-        _ => None,
-    }
-}
-
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
 pub mod test_server {
@@ -1001,6 +992,52 @@ mod tests {
         }
     }
 
+    #[test]
+    fn choice_pick_requires_string_label_and_maximum_probability() {
+        // A server Boolean or number is not the caller's string option,
+        // even when stringifying it would happen to match an offered label.
+        // Similarly, a declared pick must be an actual maximum; the
+        // distribution-total rounding tolerance cannot excuse a wrong pick.
+        for answer in [
+            json!({"choice": true, "probs": {"true": 0.5, "1": 0.5}}),
+            json!({"choice": 1, "probs": {"true": 0.5, "1": 0.5}}),
+            json!({"choice": "true", "probs": {"true": 0.4997, "1": 0.5003}}),
+        ] {
+            let body = json!({"answers": {"route": answer}}).to_string();
+            let server = fake_decision_engine(200, &body, Duration::ZERO);
+            assert_eq!(
+                client(&server.url).ask_blocking(&request(
+                    "state",
+                    vec![choice("route", &["true", "1"])],
+                )),
+                DecisionEngineOutcomeV1::Malformed,
+                "{body}"
+            );
+        }
+
+        // A genuine string label remains admissible on an exact tie.
+        let body = json!({"answers": {"route": {
+            "choice": "true",
+            "probs": {"true": 0.5, "1": 0.5}
+        }}})
+        .to_string();
+        let server = fake_decision_engine(200, &body, Duration::ZERO);
+        let result = answered(client(&server.url).ask_blocking(&request(
+            "state",
+            vec![choice("route", &["true", "1"])],
+        )));
+        assert_eq!(
+            result.answers["route"],
+            DecisionAnswerV1::Choice {
+                chosen: Some("true".to_owned()),
+                probabilities: BTreeMap::from([
+                    ("1".to_owned(), 0.5),
+                    ("true".to_owned(), 0.5),
+                ]),
+            }
+        );
+    }
+
     #[test]
     fn choice_probability_aliases_must_agree_and_be_well_formed() {
         let complete = json!({"code": 0.7, "memory": 0.3});

From 642015a1d18572afbda0c2b331d3809078c92213 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 17:25:09 -0800
Subject: [PATCH 72/76] fix(scout): reject duplicate keys in decision HTTP
 answers

Refs #4535. Align production response admission with the qualified
HTTP evaluator, with native loopback controls and failing-on-parent
ambiguous Noul, model and Choice response regressions.

DRAFT / NO MERGE.
---
 .../agents/context_scout/decision_engine.rs   | 116 +++++++++++++++++-
 1 file changed, 114 insertions(+), 2 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index 004fcfd3ce1..880feedb280 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -26,6 +26,8 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::{Duration, Instant};
 
+use serde::de::{self, MapAccess, SeqAccess, Visitor};
+use serde::Deserialize;
 use serde_json::{Map, Value, json};
 use sha2::{Digest, Sha256};
 use tracedecay_application::http_agent::http_agent;
@@ -349,7 +351,7 @@ impl DecisionEngineClientV1 {
             tracing::debug!("decision engine answer exceeds the byte limit");
             return DecisionEngineOutcomeV1::Malformed;
         }
-        let value = match serde_json::from_slice::<Value>(&answer_bytes) {
+        let value = match serde_json::from_slice::<UnambiguousValue>(&answer_bytes) {
             Ok(value) => value,
             Err(error) => {
                 tracing::debug!(%error, "decision engine answer is not JSON");
@@ -357,7 +359,7 @@ impl DecisionEngineClientV1 {
             }
         };
         let latency_micros = u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX);
-        match parse_answers(&value, request) {
+        match parse_answers(&value.0, request) {
             Some((answers, model)) => DecisionEngineOutcomeV1::Answered(DecisionAnswersV1 {
                 answers,
                 model,
@@ -488,6 +490,79 @@ fn is_io_timeout(error: &std::io::Error) -> bool {
     ) || error.to_string().to_ascii_lowercase().contains("timed out")
 }
 
+/// Refuse ambiguous response objects before interpreting model decisions.
+/// serde_json::Value alone silently keeps the last occurrence of a JSON key,
+/// while the qualified HTTP evaluator rejects all duplicate keys.
+struct UnambiguousValue(Value);
+
+impl<'de> Deserialize<'de> for UnambiguousValue {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        deserializer.deserialize_any(UnambiguousValueVisitor)
+    }
+}
+
+struct UnambiguousValueVisitor;
+
+impl<'de> Visitor<'de> for UnambiguousValueVisitor {
+    type Value = UnambiguousValue;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str("JSON without duplicate object keys")
+    }
+
+    fn visit_unit<E: de::Error>(self) -> Result<Self::Value, E> {
+        Ok(UnambiguousValue(Value::Null))
+    }
+
+    fn visit_bool<E: de::Error>(self, value: bool) -> Result<Self::Value, E> {
+        Ok(UnambiguousValue(Value::Bool(value)))
+    }
+
+    fn visit_i64<E: de::Error>(self, value: i64) -> Result<Self::Value, E> {
+        Ok(UnambiguousValue(Value::Number(value.into())))
+    }
+
+    fn visit_u64<E: de::Error>(self, value: u64) -> Result<Self::Value, E> {
+        Ok(UnambiguousValue(Value::Number(value.into())))
+    }
+
+    fn visit_f64<E: de::Error>(self, value: f64) -> Result<Self::Value, E> {
+        let number = serde_json::Number::from_f64(value)
+            .ok_or_else(|| E::custom("nonfinite decision JSON number"))?;
+        Ok(UnambiguousValue(Value::Number(number)))
+    }
+
+    fn visit_str<E: de::Error>(self, value: &str) -> Result<Self::Value, E> {
+        Ok(UnambiguousValue(Value::String(value.to_owned())))
+    }
+
+    fn visit_string<E: de::Error>(self, value: String) -> Result<Self::Value, E> {
+        Ok(UnambiguousValue(Value::String(value)))
+    }
+
+    fn visit_seq<A: SeqAccess<'de>>(self, mut sequence: A) -> Result<Self::Value, A::Error> {
+        let mut values = Vec::new();
+        while let Some(value) = sequence.next_element::<UnambiguousValue>()? {
+            values.push(value.0);
+        }
+        Ok(UnambiguousValue(Value::Array(values)))
+    }
+
+    fn visit_map<A: MapAccess<'de>>(self, mut entries: A) -> Result<Self::Value, A::Error> {
+        let mut values = Map::new();
+        while let Some((key, value)) = entries.next_entry::<String, UnambiguousValue>()? {
+            if values.contains_key(&key) {
+                return Err(de::Error::custom(format!("duplicate decision JSON key: {key}")));
+            }
+            values.insert(key, value.0);
+        }
+        Ok(UnambiguousValue(Value::Object(values)))
+    }
+}
+
 type ParsedAnswers = (BTreeMap<String, DecisionAnswerV1>, Option<String>);
 
 /// Every requested question must be answered with the requested kind; extra
@@ -899,6 +974,43 @@ mod tests {
         );
     }
 
+    #[test]
+    fn duplicate_json_keys_cannot_change_the_model_decision() {
+        // Every response below is otherwise a valid typed answer. The ordinary
+        // serde_json::Value decoder accepts the last duplicate key; it must
+        // not let an ambiguous server response qualify as Scout evidence.
+        for (body, questions) in [
+            (
+                r#"{"answers":{"q":{"type":"noul","noul":0.1,"noul":0.9}}}"#,
+                vec![noul("q")],
+            ),
+            (
+                r#"{"model":"untrusted","model":"kev-4b","answers":{"q":{"type":"noul","noul":0.9}}}"#,
+                vec![noul("q")],
+            ),
+            (
+                r#"{"answers":{"route":{"type":"choice","choice":"code","probs":{"code":0.7,"code":0.7,"memory":0.3}}}}"#,
+                vec![choice("route", &["code", "memory"])],
+            ),
+        ] {
+            let server = fake_decision_engine(200, body, Duration::ZERO);
+            assert_eq!(
+                client(&server.url).ask_blocking(&request("state", questions)),
+                DecisionEngineOutcomeV1::Malformed,
+                "{body}"
+            );
+            assert_eq!(server.requests.load(Ordering::SeqCst), 1);
+        }
+        // Non-ambiguous JSON remains a usable answer.
+        let server = fake_decision_engine(
+            200,
+            &single_noul_body("q", 0.9),
+            Duration::ZERO,
+        );
+        let answers = answered(client(&server.url).ask_blocking(&request("state", vec![noul("q")])));
+        assert_eq!(answers.answers["q"], DecisionAnswerV1::Noul(0.9));
+    }
+
     #[test]
     fn a_noul_answer_is_read_as_a_probability() {
         let server = fake_decision_engine(200, &single_noul_body("recall", 0.73), Duration::ZERO);

From 1a3058c561ae4d0e10dc2d3090f6f1bca5fa493f Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 17:26:27 -0800
Subject: [PATCH 73/76] style(scout): align strict decoder with Rust formatting

Refs #4535. DRAFT / NO MERGE.
---
 .../src/agents/context_scout/decision_engine.rs       | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index 880feedb280..654a6d6850a 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -26,8 +26,8 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::{Duration, Instant};
 
-use serde::de::{self, MapAccess, SeqAccess, Visitor};
 use serde::Deserialize;
+use serde::de::{self, MapAccess, SeqAccess, Visitor};
 use serde_json::{Map, Value, json};
 use sha2::{Digest, Sha256};
 use tracedecay_application::http_agent::http_agent;
@@ -1002,12 +1002,9 @@ mod tests {
             assert_eq!(server.requests.load(Ordering::SeqCst), 1);
         }
         // Non-ambiguous JSON remains a usable answer.
-        let server = fake_decision_engine(
-            200,
-            &single_noul_body("q", 0.9),
-            Duration::ZERO,
-        );
-        let answers = answered(client(&server.url).ask_blocking(&request("state", vec![noul("q")])));
+        let server = fake_decision_engine(200, &single_noul_body("q", 0.9), Duration::ZERO);
+        let answers =
+            answered(client(&server.url).ask_blocking(&request("state", vec![noul("q")])));
         assert_eq!(answers.answers["q"], DecisionAnswerV1::Noul(0.9));
     }
 

From ab5acd645281bb3d748f0009770e53e7f43135ab Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 18:01:35 -0800
Subject: [PATCH 74/76] fix(scout): refuse unrequested decision response
 answers

---
 .../agents/context_scout/decision_engine.rs   | 46 ++++++++++++++++++-
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index 654a6d6850a..d7c02ddfc9c 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -565,10 +565,14 @@ impl<'de> Visitor<'de> for UnambiguousValueVisitor {
 
 type ParsedAnswers = (BTreeMap<String, DecisionAnswerV1>, Option<String>);
 
-/// Every requested question must be answered with the requested kind; extra
-/// answers are ignored.
+/// Every requested question must be answered with the requested kind.
+/// Unexpected answers are malformed, matching the qualified HTTP evaluator's
+/// exact-request contract rather than silently accepting a different batch.
 fn parse_answers(value: &Value, request: &DecisionRequestV1) -> Option<ParsedAnswers> {
     let answers = value.get("answers")?.as_object()?;
+    if answers.len() != request.questions.len() {
+        return None;
+    }
     let mut parsed = BTreeMap::new();
     for question in &request.questions {
         let answer = answers.get(&question.key)?.as_object()?;
@@ -1017,6 +1021,44 @@ mod tests {
         assert_eq!(answers.model, None);
     }
 
+    #[test]
+    fn unsolicited_batch_answers_are_malformed_even_when_requested_answers_are_valid() {
+        // The frozen evaluator rejects extra response keys. An unrequested
+        // Noul or Score must not turn an otherwise unrelated batch into a
+        // successful production Scout decision.
+        for body in [
+            answers_body(
+                None,
+                &[("q", noul_answer(0.8)), ("unexpected", noul_answer(0.9))],
+            ),
+            answers_body(
+                None,
+                &[("q", noul_answer(0.8)), ("unexpected", score_answer(0.5))],
+            ),
+        ] {
+            let server = fake_decision_engine(200, &body, Duration::ZERO);
+            assert_eq!(
+                client(&server.url).ask_blocking(&request("state", vec![noul("q")])),
+                DecisionEngineOutcomeV1::Malformed,
+                "{body}"
+            );
+        }
+
+        // Exact two-question batches remain valid, rather than rejecting
+        // batching itself or dropping the second requested answer.
+        let body = answers_body(
+            None,
+            &[("q", noul_answer(0.8)), ("other", noul_answer(0.3))],
+        );
+        let server = fake_decision_engine(200, &body, Duration::ZERO);
+        let answers = answered(
+            client(&server.url).ask_blocking(&request("state", vec![noul("q"), noul("other")])),
+        );
+        assert_eq!(answers.answers.len(), 2);
+        assert_eq!(answers.answers["q"], DecisionAnswerV1::Noul(0.8));
+        assert_eq!(answers.answers["other"], DecisionAnswerV1::Noul(0.3));
+    }
+
     #[test]
     fn a_choice_answer_reads_the_pick_and_every_probability_spelling() {
         for key in ["probs", "probabilities", "options"] {

From 74bb6b8a5573792054964c6777660bcc15501543 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 22:47:06 -0800
Subject: [PATCH 75/76] fix(scout): align qualified evidence with actual HTTP
 bytes

---
 .../agents/context_scout/decision_engine.rs   | 40 ++++++++++++-------
 evals/intent_engine/HTTP_RUNNER.md            | 13 +++---
 evals/intent_engine/qualification.py          | 10 ++---
 evals/intent_engine/test_qualification.py     | 12 +++---
 evals/intent_engine/test_qualification_cli.py |  4 ++
 evals/intent_engine/test_run_http.py          |  2 +-
 6 files changed, 47 insertions(+), 34 deletions(-)

diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
index d7c02ddfc9c..33b163a6f11 100644
--- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
+++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs
@@ -262,12 +262,9 @@ impl DecisionEngineClientV1 {
         self.identity.clone()
     }
 
-    /// Prepare the same serde_json body used by `send_json` without
-    /// retaining the model-visible state or making a second HTTP request.
-    pub fn request_commitment(
-        &self,
-        request: &DecisionRequestV1,
-    ) -> Option<DecisionRequestCommitmentV1> {
+    /// Render the pretty UTF-8 JSON that is both sent over HTTP and committed.
+    /// Sharing the bytes prevents receipt/transport serializer drift.
+    fn request_wire(&self, request: &DecisionRequestV1) -> Option<Vec<u8>> {
         if !request_is_well_formed(request) {
             return None;
         }
@@ -276,7 +273,14 @@ impl DecisionEngineClientV1 {
             self.identity.model.as_deref(),
             self.max_state_bytes,
         );
-        let wire = serde_json::to_vec(&body).ok()?;
+        serde_json::to_vec_pretty(&body).ok()
+    }
+
+    pub fn request_commitment(
+        &self,
+        request: &DecisionRequestV1,
+    ) -> Option<DecisionRequestCommitmentV1> {
+        let wire = self.request_wire(request)?;
         Some(DecisionRequestCommitmentV1 {
             wire_sha256: Sha256::digest(&wire).into(),
             wire_bytes: wire.len(),
@@ -315,12 +319,16 @@ impl DecisionEngineClientV1 {
             return DecisionEngineOutcomeV1::Backpressured;
         };
         let started = Instant::now();
-        let body = request_body(
-            request,
-            self.identity.model.as_deref(),
-            self.max_state_bytes,
-        );
-        let mut response = match self.agent.post(&self.endpoint).send_json(&body) {
+        let Some(wire) = self.request_wire(request) else {
+            tracing::debug!("decision engine request could not be serialized");
+            return DecisionEngineOutcomeV1::Malformed;
+        };
+        let mut response = match self
+            .agent
+            .post(&self.endpoint)
+            .header("content-type", "application/json")
+            .send(wire.as_slice())
+        {
             Ok(response) => response,
             Err(error) => return transport_failure(&error),
         };
@@ -941,6 +949,7 @@ mod tests {
             <[u8; 32]>::from(Sha256::digest(actual.as_bytes()))
         );
         let sent: Value = serde_json::from_str(&actual).unwrap();
+        assert_eq!(actual, serde_json::to_string_pretty(&sent).unwrap());
         assert_eq!(sent["model"], "kev-4b");
         assert_eq!(sent["state"], tail_on_char_boundary(&original.state, 9));
         assert!(!actual.contains("old private context"));
@@ -1362,7 +1371,6 @@ mod tests {
                     choice_answer(Some("memory"), "probs", &[("code", 0.1), ("memory", 0.9)]),
                 ),
                 ("urgency", score_answer(0.5)),
-                ("unrequested", noul_answer(0.1)),
             ],
         );
         let server = fake_decision_engine(200, &body, Duration::ZERO);
@@ -1374,7 +1382,9 @@ mod tests {
                 score("urgency", &["low", "high"]),
             ],
         )));
-        assert_eq!(answers.answers.len(), 3, "unrequested answers are dropped");
+        assert_eq!(answers.answers.len(), 3, "each requested answer is returned");
+        assert_eq!(answers.answers["recall"], DecisionAnswerV1::Noul(0.9));
+        assert_eq!(answers.answers["urgency"], DecisionAnswerV1::Score(0.5));
         assert_eq!(server.requests.load(Ordering::SeqCst), 1);
     }
 
diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md
index 7b94915e91b..7639c96bcb9 100644
--- a/evals/intent_engine/HTTP_RUNNER.md
+++ b/evals/intent_engine/HTTP_RUNNER.md
@@ -405,12 +405,13 @@ For this arm, `questions_sha256` binds the declared settings vocabulary and
 its order, while `captured_requests_sha256` and `holdout_requests_sha256`
 bind the actual ordered **wire bytes**. The capture validator requires the
 canonical sorted question-map wire order, not a match to the settings vector's
-iteration order. It also checks the full compact Rust `serde_json::Value` wire
-encoding (lexically ordered object keys, no optional JSON whitespace and
-UTF-8 text rather than alternative Unicode escapes). Semantically equivalent
-JSON with a different top-level order or pretty-printing is not a legitimate
-capture of this pinned runtime serializer, even if every evidence hash has
-been recomputed. Only the inner request must be canonical; the outer evidence
+iteration order. It also checks the full pretty Rust `serde_json::Value` wire
+encoding (lexically ordered object keys, two-space indentation and UTF-8 text
+rather than alternative Unicode escapes). The client uses one
+`serde_json::to_vec_pretty` byte buffer for both transport and request
+commitments. Semantically equivalent compact JSON or different indentation
+is not a legitimate capture of this pinned runtime serializer, even if every
+evidence hash has been recomputed. Only the inner request must be canonical; the outer evidence
 file preserves its original hashed bytes. This wire-shape check does **not**
 attest that the bytes actually originated from Scout.
 
diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py
index d52f7109e5d..826747eaad3 100644
--- a/evals/intent_engine/qualification.py
+++ b/evals/intent_engine/qualification.py
@@ -73,14 +73,12 @@ def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes
         # order other than the production encoder's canonical sorted order.
         if list(request["questions"]) != sorted(questions):
             raise ValueError(f"captured Scout request {record['id']} question order differs from canonical Scout wire order")
-        # The production client passes a serde_json::Value to ureq::send_json.
-        # Its default Bazel serializer emits compact JSON with lexically sorted
-        # object keys, not just a sorted questions map. Otherwise an invented
-        # (but self-consistently rehashed) request could be called a Scout capture
-        # even though the production encoder could not have sent those bytes.
+        # The Rust client sends serde_json::to_vec_pretty(Value) directly
+        # and hashes those same bytes. Refuse compact, reordered or otherwise
+        # re-encoded captures even when their digests have been recomputed.
         canonical = json.dumps(
             request, sort_keys=True, ensure_ascii=False,
-            separators=(",", ":"), allow_nan=False,
+            indent=2, allow_nan=False,
         ).encode("utf-8")
         if encoded != canonical:
             raise ValueError(
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index ee485cbcd20..e860b6b0212 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -522,7 +522,7 @@ def test_impossible_token_counters_cannot_qualify_even_with_rehashed_evidence(se
         # as a legitimate, zero-truncation or measured-drop result.
         forged = self.signed_result()
         forged["rows"][0].update(
-            state_tokens=3, reported_state_tokens_dropped=4, dropped=4,
+            state_tokens=3, reported_state_tokens_dropped=4, dropped=0,
         )
         with self.assertRaisesRegex(ValueError, "dropped state tokens exceed original"):
             qualification.qualified_report(
@@ -755,7 +755,7 @@ def score_sample(value=result):
             wrong = copy.deepcopy(sample)
             wrong["response"]["answers"]["impact"]["noul"] = 0.31
             path.write_text(json.dumps(wrong), encoding="utf-8")
-            with self.assertRaisesRegex(ValueError, "response or row disagrees"):
+            with self.assertRaisesRegex(ValueError, "raw response disagrees with first scored row"):
                 score_sample()
 
             wrong = copy.deepcopy(sample)
@@ -1337,7 +1337,7 @@ def answer(url, body, **kwargs):
             with patch.object(run_http, "HERE", str(here)), patch.object(
                 run_http, "post"
             ) as post:
-                with self.assertRaisesRegex(ValueError, "sample response or row disagrees"):
+                with self.assertRaisesRegex(ValueError, "raw response disagrees with first scored row"):
                     run_http.main(args)
                 post.assert_not_called()
             self.assertEqual(partial.read_bytes(), partial_bytes)
@@ -1707,7 +1707,7 @@ def replace_after_admission(url, body, **kwargs):
             with patch.object(run_http, "HERE", str(here)), patch.object(
                 run_http, "post", side_effect=replace_after_admission,
             ) as post:
-                with self.assertRaisesRegex(ValueError, "sample changed during inference"):
+                with self.assertRaisesRegex(ValueError, "raw response disagrees with first scored row"):
                     run_http.main(args)
                 self.assertEqual(post.call_count, 1)
             self.assertFalse(completed.exists())
@@ -2332,7 +2332,7 @@ def captured_scout_fixture(self):
             body = {"questions": self.questions, "state": "User: " + case["current_prompt"]}
             records.append({
                 "id": case["id"],
-                "body": json.dumps(body, ensure_ascii=False, separators=(",", ":"), sort_keys=True),
+                "body": json.dumps(body, ensure_ascii=False, indent=2, sort_keys=True),
             })
         capture_bytes = json.dumps(records, ensure_ascii=False).encode("utf-8")
         manifest = copy.deepcopy(self.manifest)
@@ -2768,7 +2768,7 @@ def test_captured_state_budget_is_utf8_bytes_and_independently_checked(self):
             body = qualification.loads_unambiguous_json(record["body"])
             body["state"] = "éé"
             record["body"] = json.dumps(
-                body, sort_keys=True, ensure_ascii=False, separators=(",", ":"),
+                body, sort_keys=True, ensure_ascii=False, indent=2,
             )
         capture = json.dumps(records, ensure_ascii=False).encode("utf-8")
         manifest["captured_requests_sha256"] = qualification.sha256_bytes(capture)
diff --git a/evals/intent_engine/test_qualification_cli.py b/evals/intent_engine/test_qualification_cli.py
index 8f5d985dadc..495dda9f0eb 100644
--- a/evals/intent_engine/test_qualification_cli.py
+++ b/evals/intent_engine/test_qualification_cli.py
@@ -108,6 +108,10 @@ def test_raw_first_response_sample_is_checked_by_independent_cli(self):
                 key: {"type": "noul", "noul": value}
                 for key, value in first["probs"].items()
             },
+            "usage": {
+                "input_tokens": first["input_tokens"],
+                "state_tokens_dropped": first["reported_state_tokens_dropped"],
+            },
         }
         first["sample_response_sha256"] = fixtures.run_http.digest(response)
         sample = {
diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py
index 4db7e6f0656..74479810c2e 100644
--- a/evals/intent_engine/test_run_http.py
+++ b/evals/intent_engine/test_run_http.py
@@ -1063,7 +1063,7 @@ def test_dangling_sample_link_refuses_without_inference_or_partial_output(self):
         self.assertFalse(sample.exists())
         self.assertFalse(self.partial.exists())
         # Also guard the atomic-publish race after the initial preflight.
-        with self.assertRaisesRegex(ValueError, "not a regular file"):
+        with self.assertRaisesRegex(ValueError, "sample path exists or was replaced during publication"):
             run_http.publish_sample(str(sample), '{"valid": true}')
         self.assertEqual(list(self.root.glob(".noul-sample-*")), [])
 

From 65d9828047e8f80664281450dd9bb0a9ba0d5250 Mon Sep 17 00:00:00 2001
From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com>
Date: Mon, 9 Nov 2026 23:34:13 -0800
Subject: [PATCH 76/76] fix(evals): reject in-flight substitution of raw
 qualification sample
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bind the exact original sample bytes across new and resumed held-out
inference, not only parsed fields and scored-row commitments. Write UTF-8
samples without platform-dependent newline conversion. The regression
replaces a sample during the second model request with semantically equal
but re-encoded evidence; require refusal and preserve recovery.

Refs #4535
DRAFT — NEVER MERGE
---
 evals/intent_engine/run_http.py           | 28 +++++++++++++++--------
 evals/intent_engine/test_qualification.py | 12 ++++++----
 2 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py
index 299d8688865..3bdbd26e2ca 100644
--- a/evals/intent_engine/run_http.py
+++ b/evals/intent_engine/run_http.py
@@ -56,9 +56,9 @@ def validate_qualified_sample(path, experiment_sha, requests, expected_row=None)
     """Refuse a stale first-response sample during an exact qualified resume.
 
     Raw samples are debugging evidence, not independent model attestation.
-    Still, a sample from another experiment must never silently accompany a
-    fresh or resumed frozen result. Verify its original request bytes as well
-    as the experiment fingerprint, without normalizing a Scout capture.
+    Verify the exact original bytes through publication: a replacement can
+    preserve every scored value while rewriting the evidence artifact.
+    Never normalize the original Scout request bytes.
     """
     # The path is untrusted resumable evidence. A separate islink/isfile
     # preflight followed by builtins.open can follow a swapped-in symlink.
@@ -166,7 +166,7 @@ def validate_qualified_sample(path, experiment_sha, requests, expected_row=None)
     row_sha = sample.get("row_sha256")
     if not isinstance(row_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", row_sha):
         raise ValueError("qualified sample lacks the completed first-row commitment")
-    return digest(sample["response"]), row_sha
+    return digest(sample["response"]), row_sha, hashlib.sha256(sample_bytes).hexdigest()
 
 
 def publish_sample(path, serialized, refuse_existing=False):
@@ -181,8 +181,8 @@ def publish_sample(path, serialized, refuse_existing=False):
     staged = os.path.join(directory, f".noul-sample-{secrets.token_hex(16)}")
     descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
     try:
-        with os.fdopen(descriptor, "w", encoding="utf-8") as output:
-            output.write(serialized)
+        with os.fdopen(descriptor, "wb") as output:
+            output.write(serialized.encode("utf-8"))
         try:
             os.link(staged, path)
         except FileExistsError:
@@ -821,6 +821,7 @@ def main(argv=None):
         header = {"experiment": identity, "experiment_sha256": digest(identity)}
         partial_path = path + ".partial"
         admitted_sample = None
+        sample_sha256 = None
         if qualification_sha is not None and args.sample_out and os.path.lexists(args.sample_out):
             # A new experiment cannot inherit a previous tag's first-response
             # evidence. An exact resume may retain only its own sample.
@@ -829,6 +830,7 @@ def main(argv=None):
             admitted_sample = validate_qualified_sample(
                 args.sample_out, header["experiment_sha256"], requests,
             )
+            sample_sha256 = admitted_sample[2]
         try:
             # Default umask 022 (or an explicit umask 000) must not expose
             # result labels, cases or response telemetry to other users.
@@ -862,12 +864,14 @@ def main(argv=None):
                         # A crash between publishing the sample and appending
                         # its row cannot be reconstructed as a completed call.
                         raise ValueError("qualified sample has no completed first-response row; use a new tag")
-                    response_sha, row_sha = admitted_sample
+                    response_sha, row_sha, _ = admitted_sample
                     first_row = done[first_id]
-                    validate_qualified_sample(
+                    current_sample = validate_qualified_sample(
                         args.sample_out, header["experiment_sha256"], requests,
                         expected_row=first_row,
                     )
+                    if current_sample[2] != sample_sha256:
+                        raise ValueError("qualified first-response sample changed during resume admission")
                     if (first_row.get("sample_response_sha256") != response_sha
                             or digest(first_row) != row_sha):
                         raise ValueError("qualified sample response or row disagrees with completed evidence")
@@ -957,6 +961,8 @@ def main(argv=None):
                         args.sample_out, serialized_sample,
                         refuse_existing=qualification_sha is not None,
                     )
+                    if qualification_sha is not None:
+                        sample_sha256 = hashlib.sha256(serialized_sample.encode("utf-8")).hexdigest()
                 partial.write(serialized_row + "\n")
                 partial.flush()
                 done[case["id"]] = row
@@ -971,12 +977,14 @@ def main(argv=None):
                 # the original first response against the durable first row
                 # immediately before exposing a completed qualification.
                 first_row = done[next(iter(cases))]
-                response_sha, row_sha = validate_qualified_sample(
+                response_sha, row_sha, final_sample_sha256 = validate_qualified_sample(
                     args.sample_out, header["experiment_sha256"], requests,
                     expected_row=first_row,
                 )
                 if (first_row.get("sample_response_sha256") != response_sha
-                        or digest(first_row) != row_sha):
+                        or digest(first_row) != row_sha
+                        or sample_sha256 is None
+                        or final_sample_sha256 != sample_sha256):
                     raise ValueError(
                         "qualified first-response sample changed during inference; "
                         "refusing completed result"
diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py
index e860b6b0212..d2959a09a4c 100644
--- a/evals/intent_engine/test_qualification.py
+++ b/evals/intent_engine/test_qualification.py
@@ -1539,7 +1539,7 @@ def test_qualified_sample_refuses_oversize_and_growth_before_parsing(self):
             sample.write_bytes(valid_bytes)
             self.assertEqual(
                 run_http.validate_qualified_sample(str(sample), "f" * 64, requests),
-                (run_http.digest(response), "a" * 64),
+                (run_http.digest(response), "a" * 64, hashlib.sha256(valid_bytes).hexdigest()),
             )
             sample_limit = 16 * (
                 len(run_http.request_bytes(request)) + run_http.MAX_HTTP_RESPONSE_BYTES
@@ -1619,7 +1619,7 @@ def redirect():
                         run_http.validate_qualified_sample(
                             str(sample), experiment_sha, requests,
                         ),
-                        (run_http.digest(response), evidence["row_sha256"]),
+                        (run_http.digest(response), evidence["row_sha256"], hashlib.sha256(original_bytes).hexdigest()),
                     )
                     swaps = []
 
@@ -1692,9 +1692,11 @@ def interrupt(url, body, **kwargs):
 
             def replace_after_admission(url, body, **kwargs):
                 replacement = json.loads(original_sample)
-                replacement["response"]["answers"]["impact"]["noul"] = 0.95
+                # Keep all semantic fields and row commitments unchanged, but
+                # swap in a differently encoded sample after admission.
+                # The older row/response-only check would publish this run.
                 alternate = here / "replacement.json"
-                alternate.write_text(json.dumps(replacement))
+                alternate.write_text(json.dumps(replacement, sort_keys=True))
                 alternate.replace(sample)
                 return {
                     "model": "self-reported-test-model",
@@ -1707,7 +1709,7 @@ def replace_after_admission(url, body, **kwargs):
             with patch.object(run_http, "HERE", str(here)), patch.object(
                 run_http, "post", side_effect=replace_after_admission,
             ) as post:
-                with self.assertRaisesRegex(ValueError, "raw response disagrees with first scored row"):
+                with self.assertRaisesRegex(ValueError, "sample changed during inference"):
                     run_http.main(args)
                 self.assertEqual(post.call_count, 1)
             self.assertFalse(completed.exists())