diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e13fa13..9a1da83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -529,7 +529,8 @@ jobs: - name: The committed candidate gate (development seeds) and its n_eff pool match a fresh run # The gate of ADR-0030 on the development seeds 25 to 49 in two scenarios, and the pool of effective sample # sizes (seeds 25 to 99) that its simulated properties draw from. Held-out blocks (from seed 125) are - # never run here: the command refuses them without --held-out, and CI does not pass it. Text, integers and + # never run here: the candidate gate refuses them without --held-out, and CI does not pass it; the B1 gate refuses + # every held-out seed. Text, integers and # flags must match exactly; floats to 1e-6 relative. if: needs.changes.outputs.code == 'true' && matrix.leg == 'candidate-gate' run: | diff --git a/docs/adr/0028-the-b1-gate.md b/docs/adr/0028-the-b1-gate.md index b668289..4773556 100644 --- a/docs/adr/0028-the-b1-gate.md +++ b/docs/adr/0028-the-b1-gate.md @@ -62,3 +62,5 @@ Passing the gate does not by itself change `vision_enabled`; that stays a separa Note (2026-10-22): for new candidates the gate statistic and scenarios are those of [ADR-0030](0030-the-gate-for-the-next-visual-update-candidate.md); the result recorded here stands. Note (2026-10-23): the synthetic visual translation noise this ADR was measured on is corrected by [ADR-0033](0033-one-convention-for-the-synthetic-visual-translation-noise.md); see there for what is regenerated. + +Note (2026-11-07): the held-out set is spent. `navkit gate b1` takes development seeds only and refuses `--held-out` outright (maintainer decision). The recorded result is rendered from `docs/data/b1_gate_heldout.csv` and is not re-run. diff --git a/src/navkit/b1_gate.py b/src/navkit/b1_gate.py index 94152d5..0059367 100644 --- a/src/navkit/b1_gate.py +++ b/src/navkit/b1_gate.py @@ -8,9 +8,10 @@ GNSS-outage window with the same half-open mask as the calibration report, and :func:`navkit.eval.gate.evaluate_b1_gate` applies the conditions. -Seeds 100 to 124 are the held-out set. ADR-0028 allows one evaluation of it. The command refuses them unless -``--held-out`` is given, and so does :func:`run_gate`. ``--held-out`` takes exactly that set. Every other seed must be -a development seed: the later held-out blocks belong to :mod:`navkit.candidate_gate` (ADR-0030). +Seeds 100 to 124 are the held-out set. ADR-0028 allowed one evaluation of it, and that evaluation is done +(ADR-0036), so the set is spent. The command and :func:`run_gate` take development seeds only, and ``--held-out`` is +refused outright. The later held-out blocks belong to :mod:`navkit.candidate_gate` (ADR-0030). The recorded held-out +result is rendered from its CSV by ``scripts/render_b1_heldout.py``, which runs no case. Two further rows use the same function on the development seeds and are labelled DIAGNOSTIC: the single-anchor ``outage_visual`` (the gate has to be able to say no) and the plain clone ``outage_visual_clone``. They are not gate @@ -109,15 +110,17 @@ def _touches_held_out(seeds: list[int]) -> bool: return any(s in HELD_OUT_SEEDS for s in seeds) -def check_seeds(seeds: list[int], held_out: bool) -> None: - """Raise ``ValueError`` unless ``seeds`` are development seeds, or exactly the ADR-0028 set with ``held_out``.""" - span = f"{HELD_OUT_SEEDS.start} to {HELD_OUT_SEEDS.stop - 1}" - if held_out: - if list(seeds) != list(HELD_OUT_SEEDS): - raise ValueError(f"with --held-out the seeds must be exactly {span}, the held-out set of ADR-0028") - return +#: Why ``--held-out`` is refused. The flag stays so that the refusal says this, not "unrecognized arguments". +SPENT = ( + f"the held-out set of ADR-0028 (seeds {HELD_OUT_SEEDS.start} to {HELD_OUT_SEEDS.stop - 1}) was evaluated once and " + "is spent (ADR-0036); a new candidate is evaluated with `navkit gate candidate` (ADR-0030)" +) + + +def check_seeds(seeds: list[int]) -> None: + """Raise ``ValueError`` unless every seed is a development seed. Runs no case.""" if _touches_held_out(seeds): - raise ValueError(f"seeds {span} are the held-out set of ADR-0028; give --held-out to evaluate it (single use)") + raise ValueError(f"seeds {HELD_OUT_SEEDS.start} to {HELD_OUT_SEEDS.stop - 1} are held-out: {SPENT}") seed_policy.require_development(seeds) @@ -180,16 +183,16 @@ def run_arrays(name: str, case: dict[str, Any], defaults: dict[str, Any], seeds: return _stack(runs) -def run_gate(seeds: list[int], held_out: bool = False, diagnostics: bool = True) -> list[dict[str, Any]]: - """Run the candidate, its control and (on development seeds) the two diagnostics, and return the CSV rows.""" - check_seeds(seeds, held_out) +def run_gate(seeds: list[int], diagnostics: bool = True) -> list[dict[str, Any]]: + """Run the candidate, its control and the two diagnostics on development seeds, and return the CSV rows.""" + check_seeds(seeds) if len(seeds) < 2: raise ValueError("the gate needs at least two seeds") doc = _document() cases, defaults = doc["cases"], doc.get("defaults", {}) control = run_arrays(CONTROL, cases[CONTROL], defaults, seeds) rows: list[dict[str, Any]] = [] - for subject in subjects(cases, diagnostics and not held_out): + for subject in subjects(cases, diagnostics): arrays = run_arrays(subject.name, subject.case, defaults, seeds) result = evaluate_b1_gate(arrays["nees"], arrays["error"], arrays["sigma"], arrays["ate"], control["ate"]) rows.extend(result_rows(subject, seeds, result)) @@ -309,16 +312,16 @@ def markdown_table(rows: list[dict[str, Any]]) -> str: class _Parser(argparse.ArgumentParser): - """Refuses seeds :func:`check_seeds` refuses, and says the set is single use when ``--held-out`` is given.""" + """Refuses ``--held-out`` and every seed :func:`check_seeds` refuses.""" def parse_args(self, args: Any = None, namespace: Any = None) -> argparse.Namespace: # type: ignore[override] ns = super().parse_args(args, namespace) + if ns.held_out: + self.error(f"--held-out is refused: {SPENT}") try: - check_seeds(ns.seeds, ns.held_out) + check_seeds(ns.seeds) except ValueError as exc: self.error(str(exc)) - if ns.held_out: - print("warning: the held-out set (seeds 100 to 124) is single use per ADR-0028.", file=sys.stderr) return ns @@ -334,7 +337,7 @@ def build_parser() -> argparse.ArgumentParser: ap.add_argument("--seeds", type=_seeds_arg, required=True, help="A-B or a comma list; the gate uses exactly 25") ap.add_argument("--csv", type=Path, required=True, help="write the rows here") ap.add_argument("--page", type=Path, default=None, help="write the table between the b1-gate-dev markers") - ap.add_argument("--held-out", action="store_true", help="take exactly seeds 100 to 124 (single use, ADR-0028)") + ap.add_argument("--held-out", action="store_true", help="refused: the held-out set of ADR-0028 is spent") return ap @@ -343,7 +346,7 @@ def main(argv: list[str] | None = None) -> int: args = ap.parse_args(argv) if len(args.seeds) != GATE_RUNS: ap.error(f"the gate is decided on exactly {GATE_RUNS} seeds, got {len(args.seeds)}") - rows = run_gate(args.seeds, held_out=args.held_out) + rows = run_gate(args.seeds) args.csv.parent.mkdir(parents=True, exist_ok=True) args.csv.write_text(csv_text(rows), encoding="utf-8") print(f"wrote {args.csv}", file=sys.stderr) diff --git a/src/navkit/cli.py b/src/navkit/cli.py index 2f05fd0..befd189 100644 --- a/src/navkit/cli.py +++ b/src/navkit/cli.py @@ -136,7 +136,7 @@ def _gate(argv: list[str] | None) -> int: if not args or args[0] in ("-h", "--help") or args[0] not in routes: usage = "usage: navkit gate {b1,candidate,n-eff-pool,tune-scale,tune-gm,usefulness-s2} [options]" notes = ( - " b1 the B1 gate of ADR-0028 on the candidate over 25 seeds (seeds 100 to 124 need --held-out)", + " b1 the B1 gate of ADR-0028 over 25 development seeds (its held-out set is spent)", " candidate the gate of ADR-0030 in two scenarios (seeds from 125 are held-out blocks, one at a time)", " n-eff-pool the effective sample sizes that the simulated properties of ADR-0030 draw from", " tune-scale the pre-registered tuning run of the adaptive visual scale (ADR-0031), seeds 0 to 99 only", diff --git a/src/navkit/seed_policy.py b/src/navkit/seed_policy.py index 1ecf151..a3fb074 100644 --- a/src/navkit/seed_policy.py +++ b/src/navkit/seed_policy.py @@ -5,8 +5,8 @@ case on seed numbers calls :func:`require_development` before it starts: the ``navkit sweep`` entry points, ``navkit run --seed`` and ``--scene-seed``, ``navkit euroc run`` and ``navkit euroc compare``. Seeds a config file sets itself are checked too: by ``navkit run`` before the first case, and by :func:`navkit.benchmark.run_case` -whenever no ``seed`` replaces them. The B1 gate and the candidate gate are the two procedures allowed to use held-out -seeds; the B1 gate takes only its own set, 100 to 124, and only with ``--held-out``. +whenever no ``seed`` replaces them. The candidate gate is the one procedure allowed to use held-out +seeds. The B1 gate takes development seeds only: its set, 100 to 124, is spent, and ``--held-out`` is refused. """ from __future__ import annotations diff --git a/tests/test_b1_gate.py b/tests/test_b1_gate.py index ef50e9f..287b782 100644 --- a/tests/test_b1_gate.py +++ b/tests/test_b1_gate.py @@ -278,38 +278,35 @@ def test_the_command_accepts_other_seeds_without_the_flag(text): assert args.held_out is False and len(args.seeds) >= 1 -def test_the_flag_allows_the_held_out_seeds_and_says_it_is_single_use(capsys): - args = b1_gate.build_parser().parse_args(["--seeds", "100-124", "--csv", "x.csv", "--held-out"]) - assert list(args.seeds) == list(range(100, 125)) - assert "single use" in capsys.readouterr().err - - -@pytest.mark.parametrize("text", ["125-149", "0-24", "101-125", "100-123"]) -def test_the_flag_allows_only_the_b1_held_out_set(text, capsys): - """Later blocks belong to the candidate gate (ADR-0030); the flag is not a way around it.""" +@pytest.mark.parametrize("text", ["100-124", "0-24", "125-149"]) +def test_the_held_out_flag_is_refused_outright(text, capsys): + """The set of ADR-0028 was evaluated once and is spent (ADR-0036). A new candidate goes to the candidate gate.""" with pytest.raises(SystemExit) as stop: b1_gate.build_parser().parse_args(["--seeds", text, "--csv", "x.csv", "--held-out"]) assert stop.value.code == 2 - assert "100 to 124" in capsys.readouterr().err + err = capsys.readouterr().err + assert "spent" in err and "navkit gate candidate" in err -def test_the_run_function_refuses_later_blocks_with_or_without_the_flag(monkeypatch): - def never(*a, **k): # if the guard is missing, stop here: no held-out seed is ever run +@pytest.fixture +def no_run(monkeypatch): + def never(*a, **k): # if a guard is missing, stop here: no held-out seed is ever run raise AssertionError("the gate reached a run") monkeypatch.setattr(b1_gate, "_document", never) monkeypatch.setattr(b1_gate, "run_arrays", never) - with pytest.raises(ValueError, match="held-out"): - b1_gate.run_gate(list(range(125, 150))) - with pytest.raises(ValueError, match="100 to 124"): - b1_gate.run_gate(list(range(125, 150)), held_out=True) -def test_the_run_function_refuses_the_held_out_seeds_itself(): - with pytest.raises(ValueError, match="held-out"): - b1_gate.run_gate(list(range(100, 125))) +@pytest.mark.parametrize("seeds", [list(range(100, 125)), list(range(125, 150)), [3, 110]]) +def test_the_run_function_refuses_every_held_out_seed(seeds, no_run): with pytest.raises(ValueError, match="held-out"): - b1_gate.run_gate([3, 110]) + b1_gate.run_gate(seeds) + + +def test_the_run_function_has_no_held_out_mode(): + import inspect + + assert "held_out" not in inspect.signature(b1_gate.run_gate).parameters def test_the_command_fixes_the_gate_at_twenty_five_seeds(capsys): diff --git a/tests/test_seed_policy.py b/tests/test_seed_policy.py index 75a6d32..29ccd3f 100644 --- a/tests/test_seed_policy.py +++ b/tests/test_seed_policy.py @@ -136,12 +136,13 @@ def test_held_out_constants_are_defined_once() -> None: # --- the inventory: a new seed flag cannot ship without a refusal case ------------------------- -#: Modules with a seed argument that are not development-only: the two gate procedures may use held-out seeds. -GATE_MODULES = {"b1_gate.py", "candidate_gate.py"} +#: Modules with a seed argument that are not development-only: the candidate gate may use held-out blocks. +GATE_MODULES = {"candidate_gate.py"} #: Every other module with a ``--...seed...`` argument, and where its refusal is tested. #: ``sweep`` modules are in ``SEED_SWEEPS`` or the generic list above, ``run`` and ``euroc`` have their own tests. GUARDED_MODULES = { "attribution.py", + "b1_gate.py", # tests/test_b1_gate.py: development seeds only, --held-out refused (ADR-0028 set spent) "runner.py", # the shared study runner, for any study that declares Flags(seeds=True) "benchmark.py", "correlated_gnss_study.py",