diff --git a/src/navkit/report/runner.py b/src/navkit/report/runner.py index 7665448..cff7203 100644 --- a/src/navkit/report/runner.py +++ b/src/navkit/report/runner.py @@ -14,6 +14,7 @@ from dataclasses import dataclass, field from pathlib import Path +from .. import seed_policy from .page import block_is_current, write_block from .table import Row, Table @@ -133,7 +134,13 @@ def _csv_differences(study: Study, compare: Path, rows: list[Row]) -> list[str]: def run_study(study: Study, argv: list[str] | None = None) -> int: - args = build_parser(study).parse_args(argv) + ap = build_parser(study) + args = ap.parse_args(argv) + if getattr(args, "seeds", None) is not None and study.flags.seeds: + try: + seed_policy.require_development(args.seeds) + except ValueError as exc: + ap.error(str(exc)) csv_path: Path | None = getattr(args, "csv", None) page_path: Path | None = getattr(args, "page", None) compare: Path | None = getattr(args, "compare", None) diff --git a/tests/test_seed_policy.py b/tests/test_seed_policy.py index b713446..a74f465 100644 --- a/tests/test_seed_policy.py +++ b/tests/test_seed_policy.py @@ -130,3 +130,90 @@ def test_held_out_constants_are_defined_once() -> None: root = Path(navkit.__file__).parent hits = [p.name for p in root.glob("*.py") if literal.search(p.read_text())] assert hits == ["seed_policy.py"] + + +# --- the inventory: a new seed flag cannot ship without a refusal case ------------------------- + +#: Modules with a seed argument that are not development-only: the two gate procedures may use held-out seeds. +GATE_MODULES = {"b1_gate.py", "candidate_gate.py"} +#: Every other module with a ``--...seed...`` argument, and where its refusal is tested. +#: ``sweep`` modules are in ``SEED_SWEEPS`` or the generic list above, ``run`` and ``euroc`` have their own tests. +GUARDED_MODULES = { + "attribution.py", + "runner.py", # the shared study runner, for any study that declares Flags(seeds=True) + "benchmark.py", + "correlated_gnss_study.py", + "euroc_compare.py", + "euroc_eval.py", + "fault_classification.py", + "fault_matrix.py", + "fdir_false_alarm.py", + "mismatch_sweep.py", + "outage_sweep.py", + "scene_sweep.py", + "seed_sweep.py", + "separability.py", + "shared_stream_effect.py", + "slow_ramp.py", + "start_draw_study.py", + "vision_ramp.py", +} + + +def _modules_with_a_seed_flag() -> set[str]: + import re + from pathlib import Path + + import navkit + + # An ``add_argument`` call whose first string is a flag with "seed" in it, on the same or the next line. + flag = re.compile(r"add_argument\(\s*[\"']--[a-z-]*seed[a-z-]*[\"']") + root = Path(navkit.__file__).parent + return {p.name for p in root.rglob("*.py") if flag.search(p.read_text())} + + +def test_every_module_with_a_seed_flag_is_listed() -> None: + found = _modules_with_a_seed_flag() + unlisted = found - GATE_MODULES - GUARDED_MODULES + assert not unlisted, f"seed flag without a refusal case in tests/test_seed_policy.py: {sorted(unlisted)}" + stale = (GATE_MODULES | GUARDED_MODULES) - found + assert not stale, f"listed but no longer takes a seed flag: {sorted(stale)}" + + +@pytest.mark.parametrize( + "kind, args", + [ + ("attribution", ["--growth-seeds", "101"]), + ("attribution", ["--fault-seeds", "101"]), + ("mismatch", ["--ensemble-seeds", "101"]), + ], +) +def test_the_secondary_seed_flags_refuse_held_out_counts(kind: str, args: list[str], capsys) -> None: + with pytest.raises(SystemExit) as e: + cli.main(["sweep", kind, *args]) + assert e.value.code == 2 + assert "held-out" in capsys.readouterr().err + + +def test_the_shared_study_runner_refuses_held_out_seeds(tmp_path, capsys) -> None: + from navkit.report.runner import Flags, Study + from navkit.report.runner import run_study as run + + calls: list[object] = [] + + def never(seeds): # the refusal must come before any case runs + calls.append(seeds) + return [] + + study = Study( + prog="x", + description="x", + table=None, # type: ignore[arg-type] + run=never, + flags=Flags(seeds=True), + ) + with pytest.raises(SystemExit) as e: + run(study, ["--seeds", "99,101"]) + assert e.value.code == 2 + assert "held-out" in capsys.readouterr().err + assert calls == []