Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/navkit/report/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from dataclasses import dataclass, field
from pathlib import Path

from .. import seed_policy
from .page import block_is_current, write_block
from .table import Row, Table

Expand Down Expand Up @@ -133,7 +134,13 @@ def _csv_differences(study: Study, compare: Path, rows: list[Row]) -> list[str]:


def run_study(study: Study, argv: list[str] | None = None) -> int:
args = build_parser(study).parse_args(argv)
ap = build_parser(study)
args = ap.parse_args(argv)
if getattr(args, "seeds", None) is not None and study.flags.seeds:
try:
seed_policy.require_development(args.seeds)
except ValueError as exc:
ap.error(str(exc))
csv_path: Path | None = getattr(args, "csv", None)
page_path: Path | None = getattr(args, "page", None)
compare: Path | None = getattr(args, "compare", None)
Expand Down
87 changes: 87 additions & 0 deletions tests/test_seed_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,90 @@
root = Path(navkit.__file__).parent
hits = [p.name for p in root.glob("*.py") if literal.search(p.read_text())]
assert hits == ["seed_policy.py"]


# --- the inventory: a new seed flag cannot ship without a refusal case -------------------------

#: Modules with a seed argument that are not development-only: the two gate procedures may use held-out seeds.
GATE_MODULES = {"b1_gate.py", "candidate_gate.py"}
#: Every other module with a ``--...seed...`` argument, and where its refusal is tested.
#: ``sweep`` modules are in ``SEED_SWEEPS`` or the generic list above, ``run`` and ``euroc`` have their own tests.
GUARDED_MODULES = {
"attribution.py",
"runner.py", # the shared study runner, for any study that declares Flags(seeds=True)
"benchmark.py",
"correlated_gnss_study.py",
"euroc_compare.py",
"euroc_eval.py",
"fault_classification.py",
"fault_matrix.py",
"fdir_false_alarm.py",
"mismatch_sweep.py",
"outage_sweep.py",
"scene_sweep.py",
"seed_sweep.py",
"separability.py",
"shared_stream_effect.py",
"slow_ramp.py",
"start_draw_study.py",
"vision_ramp.py",
}


def _modules_with_a_seed_flag() -> set[str]:
import re
from pathlib import Path

import navkit

# An ``add_argument`` call whose first string is a flag with "seed" in it, on the same or the next line.
flag = re.compile(r"add_argument\(\s*[\"']--[a-z-]*seed[a-z-]*[\"']")
root = Path(navkit.__file__).parent
return {p.name for p in root.rglob("*.py") if flag.search(p.read_text())}


def test_every_module_with_a_seed_flag_is_listed() -> None:
found = _modules_with_a_seed_flag()
unlisted = found - GATE_MODULES - GUARDED_MODULES
assert not unlisted, f"seed flag without a refusal case in tests/test_seed_policy.py: {sorted(unlisted)}"
stale = (GATE_MODULES | GUARDED_MODULES) - found
assert not stale, f"listed but no longer takes a seed flag: {sorted(stale)}"


@pytest.mark.parametrize(
"kind, args",
[
("attribution", ["--growth-seeds", "101"]),
("attribution", ["--fault-seeds", "101"]),
("mismatch", ["--ensemble-seeds", "101"]),
],
)
def test_the_secondary_seed_flags_refuse_held_out_counts(kind: str, args: list[str], capsys) -> None:
with pytest.raises(SystemExit) as e:
cli.main(["sweep", kind, *args])
assert e.value.code == 2
assert "held-out" in capsys.readouterr().err


def test_the_shared_study_runner_refuses_held_out_seeds(tmp_path, capsys) -> None:
from navkit.report.runner import Flags, Study
from navkit.report.runner import run_study as run

calls: list[object] = []

def never(seeds): # the refusal must come before any case runs
calls.append(seeds)
return []

study = Study(
prog="x",
description="x",
table=None, # type: ignore[arg-type]
run=never,
flags=Flags(seeds=True),
)
with pytest.raises(SystemExit) as e:
run(study, ["--seeds", "99,101"])
assert e.value.code == 2
assert "held-out" in capsys.readouterr().err
assert calls == []
Loading