Verifier bake-off: no local verifier clears the second-gate bar (held-out NOT-RECOMMENDED)

CHANGELOG.md

@@ -74,6 +74,12 @@ while doing a fraction of its job.
 - **Startup fails loudly** on a `redis_url` without the feature, or a server that does not answer
   PING within 5s.
 
+- **Verifier bake-off** (`specs/verifier-bakeoff.md`, `docs/benchmarks/verifier-bakeoff.md`,
+  `firstpass-bench --verifier-bakeoff`): OpenJev with think/samples, a local Qwen3-Coder `judge`, and
+  CodeT-style generated tests — none clears the second-gate bar on held-out data (best: generated tests,
+  catch 0.094, collateral 0.067). **NOT-RECOMMENDED**; see ADR 0013.
+- `--fetch-priors` resume now retries failed calls instead of skipping them.
+
 ## [0.7.0]
 
 ### Fixed: `onboard` accepted a foreign listener as the proxy

crates/firstpass-bench/src/dataset.rs

@@ -161,8 +161,9 @@ fn parse_humaneval_line(line: &str) -> Result<CodingTask, String> {
 /// Convert a Python `assert <expr>[, "message"]` statement into the bare boolean expression the
 /// coding runner `eval()`s. Only strips a trailing message at a top-level comma — one outside all
 /// `()`/`[]`/`{}` nesting and outside any string literal — so call arguments like
-/// `f((1,2),(3,4))` are never split.
-fn convert_assert(stmt: &str) -> Result<String, String> {
+/// `f((1,2),(3,4))` are never split. `pub(crate)`: [`crate::verifier_bakeoff`]'s V3 reuses this to
+/// parse the model-generated `assert` lines the same way MBPP's own `test_list` is parsed.
+pub(crate) fn convert_assert(stmt: &str) -> Result<String, String> {
     let stmt = stmt.trim();
     let rest = stmt
         .strip_prefix("assert")

crates/firstpass-bench/src/jev_replay.rs

@@ -263,14 +263,8 @@ pub fn run_fetch_priors(
     let (rows, ladder) = load_matrix_with_ids(matrix_path)?;
     let mbpp = load_mbpp_examples(mbpp_path)?;
 
-    let resumed: std::collections::HashSet<String> = std::fs::read_to_string(out_path)
-        .ok()
-        .map(|t| {
-            t.lines()
-                .filter_map(|l| serde_json::from_str::<PriorRecord>(l).ok())
-                .map(|r| r.task_id)
-                .collect()
-        })
+    let resumed = std::fs::read_to_string(out_path)
+        .map(|t| resumable_ids(&t))
         .unwrap_or_default();
     if !resumed.is_empty() {
         eprintln!(
@@ -688,6 +682,18 @@ fn ladder_result(label: &str, ladder: &[String], series: &[ArmSeries]) -> Ladder
     }
 }
 
+/// Task ids a resumed fetch may skip: only the ones that already have a *successful* prior. A
+/// failed record (`raw_ok: false`) is retried, and the retry is appended; [`load_priors`] keeps the
+/// last record per task, so the retry wins. Skipping failures instead made a keyless first run
+/// poison the file: a rerun with a valid key would skip every task.
+fn resumable_ids(text: &str) -> std::collections::HashSet<String> {
+    text.lines()
+        .filter_map(|l| serde_json::from_str::<PriorRecord>(l).ok())
+        .filter(|r| r.raw_ok)
+        .map(|r| r.task_id)
+        .collect()
+}
+
 /// Load a priors JSONL file into a `task_id -> PriorRecord` map.
 ///
 /// # Errors
@@ -1279,6 +1285,32 @@ pub fn render_blend(s: &ReplayBlendStudy) -> String {
 mod tests {
     use super::*;
 
+    /// A failed record must not be skipped on resume, or a keyless first run poisons the file.
+    #[test]
+    fn resume_skips_only_successful_priors_and_retries_win() {
+        let text = concat!(
+            r#"{"task_id":"mbpp-1","ladder":["a","b"],"probs":[0.9,0.1],"raw_ok":true,"latency_ms":1}"#,
+            "\n",
+            r#"{"task_id":"mbpp-2","ladder":["a","b"],"probs":null,"raw_ok":false,"latency_ms":1}"#,
+            "\n",
+        );
+        let ids = resumable_ids(text);
+        assert!(ids.contains("mbpp-1"));
+        assert!(!ids.contains("mbpp-2"), "failed fetch must be retried");
+
+        let dir = std::env::temp_dir().join(format!("fp-resume-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).expect("mkdir");
+        let path = dir.join("p.jsonl");
+        let retried = format!(
+            "{text}{}\n",
+            r#"{"task_id":"mbpp-2","ladder":["a","b"],"probs":[0.2,0.8],"raw_ok":true,"latency_ms":1}"#
+        );
+        std::fs::write(&path, retried).expect("write");
+        let loaded = load_priors(path.to_str().expect("utf8")).expect("load");
+        assert!(loaded["mbpp-2"].raw_ok, "the later successful retry wins");
+        let _ = std::fs::remove_dir_all(&dir);
+    }
+
     fn rung(gate: bool, oracle: bool, cost: f64) -> RungOutcome {
         RungOutcome {
             gate_score: f64::from(u8::from(gate)),

crates/firstpass-bench/src/lib.rs

@@ -35,6 +35,7 @@ pub mod swe_agentic;
 pub mod swe_explore;
 pub mod swebench;
 pub mod sweep;
+pub mod verifier_bakeoff;
 pub mod vrbench;
 
 use firstpass_core::PriceTable;

crates/firstpass-bench/src/main.rs

@@ -12,6 +12,7 @@
 //!   firstpass-bench --replay-prior <matrix1> <priors1> [<matrix2> <priors2> ...]  # score the pre-registered prior A/B offline
 //!   firstpass-bench --replay-blend <mbpp.jsonl> <matrix1> <priors1> [<matrix2> <priors2> ...]  # Study A: prior+learned blend, offline
 //!   firstpass-bench --decision-study <candidates.jsonl> <mbpp.jsonl> <base_url> <labels.jsonl> <scores.jsonl>  # Study B: decision gate error rates (sandbox + live OpenJev)
+//!   firstpass-bench --verifier-bakeoff <candidates.jsonl> <mbpp.jsonl> <scratch_dir>  # V0-V3 held-out bake-off (specs/verifier-bakeoff.md; sandbox + live OpenJev + live local judge/test-writer)
 //!   firstpass-bench --multiturn-selfcheck  # prove the multi-turn harness + pre-registered bar work, no spend
 
 use firstpass_bench::coding::{
@@ -321,6 +322,64 @@ fn main() {
         return;
     }
 
+    // Verifier bake-off (specs/verifier-bakeoff.md): is there a verifier stronger than OpenJev
+    // `noul` for the second gate? Reuses Study B's sandbox oracle labels and V0 cache; V1-V3 hit
+    // two live local servers (OpenJev + mlx-lm), sequentially, cached/resumable under scratch_dir.
+    //   firstpass-bench --verifier-bakeoff <candidates.jsonl> <mbpp.jsonl> <scratch_dir> [--json]
+    if let Some(i) = args.iter().position(|a| a == "--verifier-bakeoff") {
+        let (Some(candidates), Some(mbpp), Some(scratch_dir)) = (
+            args.get(i + 1).filter(|p| !p.starts_with("--")),
+            args.get(i + 2).filter(|p| !p.starts_with("--")),
+            args.get(i + 3).filter(|p| !p.starts_with("--")),
+        ) else {
+            eprintln!(
+                "usage: firstpass-bench --verifier-bakeoff <candidates.jsonl> <mbpp.jsonl> <scratch_dir>"
+            );
+            std::process::exit(2);
+        };
+        let openjev_url = std::env::var("FIRSTPASS_OPENJEV_URL")
+            .unwrap_or_else(|_| "http://127.0.0.1:8080".to_owned());
+        let mlx_url = std::env::var("FIRSTPASS_MLX_URL")
+            .unwrap_or_else(|_| "http://127.0.0.1:8081".to_owned());
+        let sb = match establish_sandbox(&sandbox_image()) {
+            Ok(sb) => sb,
+            Err(e) => {
+                eprintln!(
+                    "cannot run — sandbox not established: {e}\ncandidate answers execute in the \
+                     hidden-test oracle and V3's generated tests and will not be run on the host"
+                );
+                std::process::exit(1);
+            }
+        };
+        match firstpass_bench::verifier_bakeoff::run(
+            sb.as_ref(),
+            candidates,
+            mbpp,
+            &openjev_url,
+            &mlx_url,
+            scratch_dir,
+        ) {
+            Ok(study) => {
+                if json {
+                    match serde_json::to_string_pretty(&study) {
+                        Ok(s) => println!("{s}"),
+                        Err(e) => {
+                            eprintln!("cannot serialize report: {e}");
+                            std::process::exit(1);
+                        }
+                    }
+                } else {
+                    println!("{}", firstpass_bench::verifier_bakeoff::render(&study));
+                }
+            }
+            Err(e) => {
+                eprintln!("--verifier-bakeoff failed: {e}");
+                std::process::exit(1);
+            }
+        }
+        return;
+    }
+
     // VRBench reference participant: run the verified cascade over the task set and WRITE a
     // submission. A ladder of one model yields a single-shot ungated submission, which is exactly
     // an AIQ baseline — so baselines come from this same path rather than being asserted.