Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -529,7 +529,8 @@ jobs:
- name: The committed candidate gate (development seeds) and its n_eff pool match a fresh run
# The gate of ADR-0030 on the development seeds 25 to 49 in two scenarios, and the pool of effective sample
# sizes (seeds 25 to 99) that its simulated properties draw from. Held-out blocks (from seed 125) are
# never run here: the command refuses them without --held-out, and CI does not pass it. Text, integers and
# never run here: the candidate gate refuses them without --held-out, and CI does not pass it; the B1 gate refuses
# every held-out seed. Text, integers and
# flags must match exactly; floats to 1e-6 relative.
if: needs.changes.outputs.code == 'true' && matrix.leg == 'candidate-gate'
run: |
Expand Down
2 changes: 2 additions & 0 deletions docs/adr/0028-the-b1-gate.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,5 @@ Passing the gate does not by itself change `vision_enabled`; that stays a separa
Note (2026-10-22): for new candidates the gate statistic and scenarios are those of [ADR-0030](0030-the-gate-for-the-next-visual-update-candidate.md); the result recorded here stands.

Note (2026-10-23): the synthetic visual translation noise this ADR was measured on is corrected by [ADR-0033](0033-one-convention-for-the-synthetic-visual-translation-noise.md); see there for what is regenerated.

Note (2026-11-07): the held-out set is spent. `navkit gate b1` takes development seeds only and refuses `--held-out` outright (maintainer decision). The recorded result is rendered from `docs/data/b1_gate_heldout.csv` and is not re-run.
45 changes: 24 additions & 21 deletions src/navkit/b1_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@
GNSS-outage window with the same half-open mask as the calibration report, and :func:`navkit.eval.gate.evaluate_b1_gate`
applies the conditions.

Seeds 100 to 124 are the held-out set. ADR-0028 allows one evaluation of it. The command refuses them unless
``--held-out`` is given, and so does :func:`run_gate`. ``--held-out`` takes exactly that set. Every other seed must be
a development seed: the later held-out blocks belong to :mod:`navkit.candidate_gate` (ADR-0030).
Seeds 100 to 124 are the held-out set. ADR-0028 allowed one evaluation of it, and that evaluation is done
(ADR-0036), so the set is spent. The command and :func:`run_gate` take development seeds only, and ``--held-out`` is
refused outright. The later held-out blocks belong to :mod:`navkit.candidate_gate` (ADR-0030). The recorded held-out
result is rendered from its CSV by ``scripts/render_b1_heldout.py``, which runs no case.

Two further rows use the same function on the development seeds and are labelled DIAGNOSTIC: the single-anchor
``outage_visual`` (the gate has to be able to say no) and the plain clone ``outage_visual_clone``. They are not gate
Expand Down Expand Up @@ -109,15 +110,17 @@ def _touches_held_out(seeds: list[int]) -> bool:
return any(s in HELD_OUT_SEEDS for s in seeds)


def check_seeds(seeds: list[int], held_out: bool) -> None:
"""Raise ``ValueError`` unless ``seeds`` are development seeds, or exactly the ADR-0028 set with ``held_out``."""
span = f"{HELD_OUT_SEEDS.start} to {HELD_OUT_SEEDS.stop - 1}"
if held_out:
if list(seeds) != list(HELD_OUT_SEEDS):
raise ValueError(f"with --held-out the seeds must be exactly {span}, the held-out set of ADR-0028")
return
#: Why ``--held-out`` is refused. The flag stays so that the refusal says this, not "unrecognized arguments".
SPENT = (
f"the held-out set of ADR-0028 (seeds {HELD_OUT_SEEDS.start} to {HELD_OUT_SEEDS.stop - 1}) was evaluated once and "
"is spent (ADR-0036); a new candidate is evaluated with `navkit gate candidate` (ADR-0030)"
)


def check_seeds(seeds: list[int]) -> None:
"""Raise ``ValueError`` unless every seed is a development seed. Runs no case."""
if _touches_held_out(seeds):
raise ValueError(f"seeds {span} are the held-out set of ADR-0028; give --held-out to evaluate it (single use)")
raise ValueError(f"seeds {HELD_OUT_SEEDS.start} to {HELD_OUT_SEEDS.stop - 1} are held-out: {SPENT}")
seed_policy.require_development(seeds)


Expand Down Expand Up @@ -180,16 +183,16 @@ def run_arrays(name: str, case: dict[str, Any], defaults: dict[str, Any], seeds:
return _stack(runs)


def run_gate(seeds: list[int], held_out: bool = False, diagnostics: bool = True) -> list[dict[str, Any]]:
"""Run the candidate, its control and (on development seeds) the two diagnostics, and return the CSV rows."""
check_seeds(seeds, held_out)
def run_gate(seeds: list[int], diagnostics: bool = True) -> list[dict[str, Any]]:
"""Run the candidate, its control and the two diagnostics on development seeds, and return the CSV rows."""
check_seeds(seeds)
if len(seeds) < 2:
raise ValueError("the gate needs at least two seeds")
doc = _document()
cases, defaults = doc["cases"], doc.get("defaults", {})
control = run_arrays(CONTROL, cases[CONTROL], defaults, seeds)
rows: list[dict[str, Any]] = []
for subject in subjects(cases, diagnostics and not held_out):
for subject in subjects(cases, diagnostics):
arrays = run_arrays(subject.name, subject.case, defaults, seeds)
result = evaluate_b1_gate(arrays["nees"], arrays["error"], arrays["sigma"], arrays["ate"], control["ate"])
rows.extend(result_rows(subject, seeds, result))
Expand Down Expand Up @@ -309,16 +312,16 @@ def markdown_table(rows: list[dict[str, Any]]) -> str:


class _Parser(argparse.ArgumentParser):
"""Refuses seeds :func:`check_seeds` refuses, and says the set is single use when ``--held-out`` is given."""
"""Refuses ``--held-out`` and every seed :func:`check_seeds` refuses."""

def parse_args(self, args: Any = None, namespace: Any = None) -> argparse.Namespace: # type: ignore[override]
ns = super().parse_args(args, namespace)
if ns.held_out:
self.error(f"--held-out is refused: {SPENT}")
try:
check_seeds(ns.seeds, ns.held_out)
check_seeds(ns.seeds)
except ValueError as exc:
self.error(str(exc))
if ns.held_out:
print("warning: the held-out set (seeds 100 to 124) is single use per ADR-0028.", file=sys.stderr)
return ns


Expand All @@ -334,7 +337,7 @@ def build_parser() -> argparse.ArgumentParser:
ap.add_argument("--seeds", type=_seeds_arg, required=True, help="A-B or a comma list; the gate uses exactly 25")
ap.add_argument("--csv", type=Path, required=True, help="write the rows here")
ap.add_argument("--page", type=Path, default=None, help="write the table between the b1-gate-dev markers")
ap.add_argument("--held-out", action="store_true", help="take exactly seeds 100 to 124 (single use, ADR-0028)")
ap.add_argument("--held-out", action="store_true", help="refused: the held-out set of ADR-0028 is spent")
return ap


Expand All @@ -343,7 +346,7 @@ def main(argv: list[str] | None = None) -> int:
args = ap.parse_args(argv)
if len(args.seeds) != GATE_RUNS:
ap.error(f"the gate is decided on exactly {GATE_RUNS} seeds, got {len(args.seeds)}")
rows = run_gate(args.seeds, held_out=args.held_out)
rows = run_gate(args.seeds)
args.csv.parent.mkdir(parents=True, exist_ok=True)
args.csv.write_text(csv_text(rows), encoding="utf-8")
print(f"wrote {args.csv}", file=sys.stderr)
Expand Down
2 changes: 1 addition & 1 deletion src/navkit/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def _gate(argv: list[str] | None) -> int:
if not args or args[0] in ("-h", "--help") or args[0] not in routes:
usage = "usage: navkit gate {b1,candidate,n-eff-pool,tune-scale,tune-gm,usefulness-s2} [options]"
notes = (
" b1 the B1 gate of ADR-0028 on the candidate over 25 seeds (seeds 100 to 124 need --held-out)",
" b1 the B1 gate of ADR-0028 over 25 development seeds (its held-out set is spent)",
" candidate the gate of ADR-0030 in two scenarios (seeds from 125 are held-out blocks, one at a time)",
" n-eff-pool the effective sample sizes that the simulated properties of ADR-0030 draw from",
" tune-scale the pre-registered tuning run of the adaptive visual scale (ADR-0031), seeds 0 to 99 only",
Expand Down
4 changes: 2 additions & 2 deletions src/navkit/seed_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
case on seed numbers calls :func:`require_development` before it starts: the ``navkit sweep`` entry points,
``navkit run --seed`` and ``--scene-seed``, ``navkit euroc run`` and ``navkit euroc compare``. Seeds a config file
sets itself are checked too: by ``navkit run`` before the first case, and by :func:`navkit.benchmark.run_case`
whenever no ``seed`` replaces them. The B1 gate and the candidate gate are the two procedures allowed to use held-out
seeds; the B1 gate takes only its own set, 100 to 124, and only with ``--held-out``.
whenever no ``seed`` replaces them. The candidate gate is the one procedure allowed to use held-out
seeds. The B1 gate takes development seeds only: its set, 100 to 124, is spent, and ``--held-out`` is refused.
"""

from __future__ import annotations
Expand Down
37 changes: 17 additions & 20 deletions tests/test_b1_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,38 +278,35 @@ def test_the_command_accepts_other_seeds_without_the_flag(text):
assert args.held_out is False and len(args.seeds) >= 1


def test_the_flag_allows_the_held_out_seeds_and_says_it_is_single_use(capsys):
args = b1_gate.build_parser().parse_args(["--seeds", "100-124", "--csv", "x.csv", "--held-out"])
assert list(args.seeds) == list(range(100, 125))
assert "single use" in capsys.readouterr().err


@pytest.mark.parametrize("text", ["125-149", "0-24", "101-125", "100-123"])
def test_the_flag_allows_only_the_b1_held_out_set(text, capsys):
"""Later blocks belong to the candidate gate (ADR-0030); the flag is not a way around it."""
@pytest.mark.parametrize("text", ["100-124", "0-24", "125-149"])
def test_the_held_out_flag_is_refused_outright(text, capsys):
"""The set of ADR-0028 was evaluated once and is spent (ADR-0036). A new candidate goes to the candidate gate."""
with pytest.raises(SystemExit) as stop:
b1_gate.build_parser().parse_args(["--seeds", text, "--csv", "x.csv", "--held-out"])
assert stop.value.code == 2
assert "100 to 124" in capsys.readouterr().err
err = capsys.readouterr().err
assert "spent" in err and "navkit gate candidate" in err


def test_the_run_function_refuses_later_blocks_with_or_without_the_flag(monkeypatch):
def never(*a, **k): # if the guard is missing, stop here: no held-out seed is ever run
@pytest.fixture
def no_run(monkeypatch):
def never(*a, **k): # if a guard is missing, stop here: no held-out seed is ever run
raise AssertionError("the gate reached a run")

monkeypatch.setattr(b1_gate, "_document", never)
monkeypatch.setattr(b1_gate, "run_arrays", never)
with pytest.raises(ValueError, match="held-out"):
b1_gate.run_gate(list(range(125, 150)))
with pytest.raises(ValueError, match="100 to 124"):
b1_gate.run_gate(list(range(125, 150)), held_out=True)


def test_the_run_function_refuses_the_held_out_seeds_itself():
with pytest.raises(ValueError, match="held-out"):
b1_gate.run_gate(list(range(100, 125)))
@pytest.mark.parametrize("seeds", [list(range(100, 125)), list(range(125, 150)), [3, 110]])
def test_the_run_function_refuses_every_held_out_seed(seeds, no_run):
with pytest.raises(ValueError, match="held-out"):
b1_gate.run_gate([3, 110])
b1_gate.run_gate(seeds)


def test_the_run_function_has_no_held_out_mode():
import inspect

assert "held_out" not in inspect.signature(b1_gate.run_gate).parameters


def test_the_command_fixes_the_gate_at_twenty_five_seeds(capsys):
Expand Down
5 changes: 3 additions & 2 deletions tests/test_seed_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,12 +136,13 @@ def test_held_out_constants_are_defined_once() -> None:

# --- the inventory: a new seed flag cannot ship without a refusal case -------------------------

#: Modules with a seed argument that are not development-only: the two gate procedures may use held-out seeds.
GATE_MODULES = {"b1_gate.py", "candidate_gate.py"}
#: Modules with a seed argument that are not development-only: the candidate gate may use held-out blocks.
GATE_MODULES = {"candidate_gate.py"}
#: Every other module with a ``--...seed...`` argument, and where its refusal is tested.
#: ``sweep`` modules are in ``SEED_SWEEPS`` or the generic list above, ``run`` and ``euroc`` have their own tests.
GUARDED_MODULES = {
"attribution.py",
"b1_gate.py", # tests/test_b1_gate.py: development seeds only, --held-out refused (ADR-0028 set spent)
"runner.py", # the shared study runner, for any study that declares Flags(seeds=True)
"benchmark.py",
"correlated_gnss_study.py",
Expand Down
Loading