From 77426f58d139e49b17bd31b712e96bf25c512f84 Mon Sep 17 00:00:00 2001 From: Colin Son Date: Thu, 10 Sep 2026 17:44:35 -0500 Subject: [PATCH] feat: Phase G verifier-derived training rewards with kernel-observed divergence penalty --- docs/NEXT_STATUS.md | 4 +-- .../examples/tasks/lumen-nav-safe/verifier.py | 2 ++ src/or_audit/eval/export_verifiers.py | 2 ++ src/or_audit/eval/wrap.py | 2 ++ tests/test_eval_run.py | 35 +++++++++++++++++++ 5 files changed, 43 insertions(+), 2 deletions(-) diff --git a/docs/NEXT_STATUS.md b/docs/NEXT_STATUS.md index 933fc5a..f53ccdf 100644 --- a/docs/NEXT_STATUS.md +++ b/docs/NEXT_STATUS.md @@ -31,10 +31,10 @@ language — this table is the current status source of truth. | A | partial | Brief + runnable lumen task v1 + strict verifier (#33-35), reference-policy A3 comparison (#44) | A3 trained-checkpoint comparison; A5 external reproduction (blocked: partner) | | B | partial | Threat model + env scrubbing (#36), container backend (#38), bounded transfer (#39), remainder + attestation + CI image (#40), probes (#41) | B5 cloud mint/storage (blocked: cloud owner); registry image publication (commercial decision) | | C | partial | Dossier + cited gate (#37), orphan-label refusal + provenance + fault robustness (#42) | C4 world-native distributions (need SOFA/GPU world revisions); C5 phantom (blocked: partner) | -| D | partial | Paired comparison + CIs (#43), compare CLI (#45), split manifests with patient/site grouping (#60, #61, #62), scorecard uncertainty views | D5 benchmark-level reporting with external study data | +| D | partial | Paired comparison + CIs (#43), compare CLI (#45), split manifests with patient/site grouping (#60-62), D5 explicit cohort subgroup validation, head-covered trial bindings, clustered continuous bootstrap, and cloud parity (#64, cloud #5) | D5 benchmark-level reporting with external study data | | E | partial | Obs/action contract + A3 (#44), interactive streams (#46), LeRobot reader + MONAI (#51), honest video adapter (#54) | E4 media alignment (no test media; mp4 banned from VC); semantic output schemas (needs E contract design) | | F | partial | Prefix-replay branching proof + seed caveat (#47), trajectory-backed forecast task + null baseline (#56), planning utility measurement and recipe (#58) | F1 trajectory benchmark imports; F4 closed-loop simulator branching | -| G | partial | SB3 PPO recipe + measurement (#48), runnable fixes (#50, #52), kernel-observed episode divergence (#57, #59) | Verifier-derived training rewards (projection withdrawn); Prime interop | +| G | partial | SB3 PPO recipe + measurement (#48), runnable fixes (#50, #52), kernel-observed episode divergence (#57, #59), verifier-derived training rewards with divergence penalty (#65) | Prime interop | | H | partial | Episode resume (#49), crash-safe writes + partial recovery (#53), strict trial provenance + pre-write drift rejection (#59) | Fleet queue/autoscaling (blocked: hosted capacity decision); vectorized stepping (not justified by measured need) | ### Status and evidence rules diff --git a/docs/examples/tasks/lumen-nav-safe/verifier.py b/docs/examples/tasks/lumen-nav-safe/verifier.py index 38741c0..395867a 100644 --- a/docs/examples/tasks/lumen-nav-safe/verifier.py +++ b/docs/examples/tasks/lumen-nav-safe/verifier.py @@ -35,6 +35,8 @@ def score(self, context: dict[str, Any]) -> dict[str, Any]: safety_max_pen = float(context["safety_max_pen"]) max_pen = _finite_number(info.get("max_pen")) diverged = _exact_bool(info.get("diverged")) + if diverged is None and context.get("diverged_observed") is True: + diverged = True raw_unsafe = info.get("unsafe") if raw_unsafe is None: unsafe = None if max_pen is None else max_pen > safety_max_pen diff --git a/src/or_audit/eval/export_verifiers.py b/src/or_audit/eval/export_verifiers.py index 29a9ba4..d7212b2 100644 --- a/src/or_audit/eval/export_verifiers.py +++ b/src/or_audit/eval/export_verifiers.py @@ -43,6 +43,7 @@ GymEnv, GymFactory, assert_perturbations_applied, + episode_diverged, make_gym, run_gym_episode, sample_action, @@ -995,6 +996,7 @@ def action_fn(env: GymEnv, observation: Any, step: int) -> Any: "info": info, "trajectory": [dict(step) for step in steps], "safety_max_pen": safety, + "diverged_observed": episode_diverged(steps), }, runtime=verifier, ) diff --git a/src/or_audit/eval/wrap.py b/src/or_audit/eval/wrap.py index 8457dd9..1081628 100644 --- a/src/or_audit/eval/wrap.py +++ b/src/or_audit/eval/wrap.py @@ -1062,6 +1062,8 @@ def _render_verifier(spec: WrapRequest) -> str: " info: dict[str, Any] = raw_info if isinstance(raw_info, dict) else {}", ' raw_success = _boolean(info, "raw_success", "success", "is_success")', ' diverged = _boolean(info, "diverged")', + ' if diverged is None and context.get("diverged_observed") is True:', + " diverged = True", " breached: list[str] = []", " unreported: list[str] = []", " for gate_id, signal, predicate in GATES:", diff --git a/tests/test_eval_run.py b/tests/test_eval_run.py index 68e06e0..dcf4dac 100644 --- a/tests/test_eval_run.py +++ b/tests/test_eval_run.py @@ -635,3 +635,38 @@ def test_nonfinite_kind_numpy_scalars() -> None: assert nonfinite_kind(np.float64("inf")) == "+inf" assert nonfinite_kind(np.float32("-inf")) == "-inf" assert nonfinite_kind(np.float32(1.5)) == "" + + +def test_lumen_verifier_consumes_diverged_observed_and_fails_gate() -> None: + import importlib.util + + path = ROOT / "docs/examples/tasks/lumen-nav-safe/verifier.py" + spec = importlib.util.spec_from_file_location("lumen_verifier", path) + assert spec is not None + assert spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + verifier = mod.LumenVerifier() + res = verifier.score( + { + "info": {"success": True, "max_pen": 0.05}, + "safety_max_pen": 0.3, + "diverged_observed": True, + } + ) + assert res["metrics"]["diverged"] is True + assert res["metrics"]["safe_success"] is False + assert res["gates"]["wall_penetration"]["status"] == "fail" + assert "episode diverged" in res["gates"]["wall_penetration"]["reason"] + + # When diverged_observed is None -> diverged is None (unassessed) + res_none = verifier.score( + { + "info": {"success": True, "max_pen": 0.05}, + "safety_max_pen": 0.3, + "diverged_observed": None, + } + ) + assert res_none["metrics"]["diverged"] is None + assert res_none["metrics"]["safe_success"] is None + assert res_none["gates"]["wall_penetration"]["status"] == "pass"