From b1cf19a1792e231686684af088e90959e466a0f9 Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:05:56 +0100 Subject: [PATCH 1/4] fix(security): keep local .env out of the image; derive Sandcastle branches from the issue id .dockerignore did not exclude .sandcastle/.env, and the Dockerfile ends with COPY . ., so a local build copied the sandbox tokens into a layer. The Sandcastle planner's free-text branch went into shell blocks of the review prompt; it is now built from a numeric issue id. Co-Authored-By: Claude Opus 5.5 --- .dockerignore | 6 ++++++ .sandcastle/main.mts | 13 +++++++++++-- tests/test_supply_chain.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index eff89a1..c11ca88 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,3 +12,9 @@ dist .hypothesis opencode.json .opencode +# Local secrets and tooling outside the package. The Dockerfile copies the whole context. +.env +**/.env +node_modules +.sandcastle/logs +.sandcastle/worktrees diff --git a/.sandcastle/main.mts b/.sandcastle/main.mts index 8d87b4f..eb4d2d5 100644 --- a/.sandcastle/main.mts +++ b/.sandcastle/main.mts @@ -29,12 +29,18 @@ import { z } from "zod"; // and validates it against this schema. We use Zod here, but any Standard // Schema validator works just as well, Valibot, ArkType, etc. See // https://standardschema.dev. +// +// The id must be an issue number. The branch is not taken from the planner: +// the review prompt runs it inside shell blocks, and the planner reads issue +// bodies, so a branch name from its text could carry a command. const planSchema = z.object({ issues: z.array( - z.object({ id: z.string(), title: z.string(), branch: z.string() }), + z.object({ id: z.string().regex(/^[1-9][0-9]*$/), title: z.string() }), ), }); +const branchFor = (id: string) => `sandcastle/issue-${id}`; + // --------------------------------------------------------------------------- // Configuration // --------------------------------------------------------------------------- @@ -93,7 +99,10 @@ for (let iteration = 1; iteration <= MAX_ITERATIONS; iteration++) { output: sandcastle.Output.object({ tag: "plan", schema: planSchema }), }); - const issues = plan.output.issues; + const issues = plan.output.issues.map((issue) => ({ + ...issue, + branch: branchFor(issue.id), + })); if (issues.length === 0) { // No unblocked work: either everything is done or everything is blocked. diff --git a/tests/test_supply_chain.py b/tests/test_supply_chain.py index 9aca594..467ec9a 100644 --- a/tests/test_supply_chain.py +++ b/tests/test_supply_chain.py @@ -2,6 +2,7 @@ from __future__ import annotations +import fnmatch import re from pathlib import Path from typing import Any @@ -52,6 +53,39 @@ def test_the_dockerfile_base_image_is_pinned_by_digest() -> None: assert re.search(r"@sha256:[0-9a-f]{64}$", image), f"{image} is not pinned by digest" +def _left_out_of_the_image(rel: str) -> bool: + """Whether ``.dockerignore`` drops ``rel`` from the build context. Plain patterns and ``**/``, no ``!``.""" + lines = (ROOT / ".dockerignore").read_text(encoding="utf-8").splitlines() + patterns = [s.strip() for s in lines if s.strip() and not s.lstrip().startswith("#")] + assert not any(p.startswith("!") for p in patterns), "the helper does not model re-inclusion" + parts = rel.split("/") + prefixes = ["/".join(parts[: i + 1]) for i in range(len(parts))] + for pat in patterns: + tails = [pat, pat[3:]] if pat.startswith("**/") else [pat] + if any(fnmatch.fnmatchcase(p, t) for p in prefixes for t in tails): + return True + return False + + +@pytest.mark.parametrize( + "rel, dropped", + [ + (".env", True), + (".sandcastle/.env", True), # holds the sandbox's GitHub and Claude tokens in a local checkout + (".sandcastle/worktrees/sandcastle-issue-1/src/navkit/cli.py", True), + (".sandcastle/logs/run.log", True), + ("node_modules/zod/package.json", True), + (".sandcastle/.env.example", False), + ("src/navkit/cli.py", False), + ("configs/benchmark.yaml", False), + ], +) +def test_the_image_build_leaves_local_secrets_out(rel: str, dropped: bool) -> None: + """The Dockerfile ends with ``COPY . .``, so whatever the context holds lands in a layer.""" + assert "COPY . ." in (ROOT / "Dockerfile").read_text(encoding="utf-8") + assert _left_out_of_the_image(rel) is dropped + + def test_dependabot_keeps_the_base_image_and_the_actions_current() -> None: cfg = _load(ROOT / ".github" / "dependabot.yml") assert {u["package-ecosystem"] for u in cfg["updates"]} >= {"docker", "github-actions", "pip"} From 9a8e4a2eac4a2cb0f20aee46dc73cad535e0a306 Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:38:15 +0100 Subject: [PATCH 2/4] docs: make licence, provenance and process claims match the evidence - EuRoC and TUM VI: replace "Not verified here" with the rights statement and licence the pages already quote; point the README licence section at ADR-0042 and #278, say some files also hold TUM VI rows and no rights-holder answer is recorded. - FAQ: per-frame rows are committed (vo_errors.csv), not only aggregates. - Site footer: the MIT licence is the code's; EuRoC-derived results are non-commercial. - README: PRs through CI are the practice, not an enforced rule; some commits went to main directly and Sandcastle merges its own PRs. - UrbanNav: the gate section no longer names multipath as the measured cause. - urbannav_imposed docstring: it does write a CSV with --csv. - ADR-0010 amendment: the streams-only comparison sentence covers the three vision cases and the two metrics it was read on. No number, table, CSV or golden is changed. Co-Authored-By: Claude Opus 5.5 --- README.md | 15 +++++++++------ docs/adr/0010-determinism-and-seeding.md | 2 +- docs/euroc.md | 3 ++- docs/faq.md | 4 +++- docs/tumvi.md | 2 +- docs/urbannav.md | 2 +- mkdocs.yml | 2 +- src/navkit/urbannav_imposed.py | 4 ++-- tests/test_urbannav_run.py | 2 +- 9 files changed, 21 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6aaabd4..9b1dd9e 100644 --- a/README.md +++ b/README.md @@ -208,8 +208,9 @@ All of it, with sources, is in [Status and limits](docs/status.md) and The code was written mostly with an AI coding assistant (Claude Code). The maintainer sets the scope and the order of work, and the project's rule is that the maintainer writes and accepts the decision in each ADR. The evidence to trust is the gates, not the authorship: finite-difference checks on the measurement Jacobians, property tests, a golden -snapshot of the benchmark, and CI. Since the first week of the repository, every change lands through a pull request -that must pass CI and that the maintainer merges. +snapshot of the benchmark, and CI. Most changes land through a pull request that passes CI. That is the practice, not +a rule the repository enforces: a few commits went to `main` directly, and the Sandcastle automation merges its own +pull requests. ## Scope and responsible use @@ -256,10 +257,10 @@ tests/ unit, property, golden-snapshot and documentation consistency test Recorded data and its citations: - **EuRoC** ([page](docs/euroc.md)): Burri, M. et al. (2016). The EuRoC micro aerial vehicle datasets. *The International - Journal of Robotics Research*, 35(10), 1157-1163. Licence: set by the dataset's authors (ETH Zurich ASL); read the terms on the - dataset page before reuse. Not verified here. + Journal of Robotics Research*, 35(10), 1157-1163. Rights statement: "In Copyright - Non-Commercial Use + Permitted", set by the dataset's authors (ETH Zurich ASL). See the License section below for the committed result files. - **TUM VI** ([page](docs/tumvi.md)): Schubert, D. et al. (2018). The TUM VI benchmark for evaluating visual-inertial odometry. - *IROS 2018*. Licence: set by the dataset's authors; read the terms on the dataset page before reuse. Not verified here. + *IROS 2018*. Licence: CC BY 4.0, as the dataset page states; the [TUM VI page](docs/tumvi.md) gives the attribution. - **UrbanNav** ([page](docs/urbannav.md)): Hsu, L.-T. et al. (2023). UrbanNav: an open-sourced multisensory dataset for urban positioning. *NAVIGATION*. Licence: MIT, as stated by the authors. @@ -278,4 +279,6 @@ for example `euroc_*.csv`, `vo_*.csv`, `fault_classification*.csv`, `multipath_* `slow_ramp.csv`, `vision_ramp.csv` and `timing.csv`. They are results, not the dataset. The EuRoC rights statement is "In Copyright - Non-Commercial Use Permitted", so use those files for non-commercial purposes only, whatever the MIT licence says about the code. The terms are the dataset authors' to set. Read them on the dataset page before any other -use. +use. Some of those files also hold rows computed from TUM VI. Whether the per-run and per-frame rows may stay +published is an open maintainer decision: [ADR-0042](docs/adr/0042-publishing-rows-derived-from-euroc.md), +[issue #278](https://github.com/Telschow/contested-nav/issues/278). No answer from the rights holder is recorded. diff --git a/docs/adr/0010-determinism-and-seeding.md b/docs/adr/0010-determinism-and-seeding.md index e345182..3abd2cb 100644 --- a/docs/adr/0010-determinism-and-seeding.md +++ b/docs/adr/0010-determinism-and-seeding.md @@ -59,7 +59,7 @@ writes it down so the next stochastic stream follows it too. **What was measured.** The development seeds, 10 of them, were run twice: as they are, and with the vision seed derived from the case seed and a stream id ([issue #259](https://github.com/Telschow/contested-nav/issues/259), `docs/data/shared_stream_effect.csv`, `docs/calibration.md`). The result was within seed noise for the aggregate. The sign of the per-seed change was mixed, and the mean paired difference was within about one standard error of zero for ATE RMSE and mean NEES in all four cases. The single-anchor control stayed overconfident and the clone stayed calibrated. Single seeds moved far (seed 0 of `outage_visual`, 17.6 m to 7.9 m ATE). The comparison also changes the IMU noise draw, which is hashed from both seeds, so it measures both changes together. Ten seeds exclude a large systematic effect, not a small one. -**Follow-up: the two changes apart.** The study now has a third seeding, `streams_only`: the independent vision seed with the IMU noise pinned to its shared draw, for the duration of one run and with nothing in `src/navkit/degrade` changed. It gives the stream relationship alone, with the IMU fixed. The medians under it are generated on `docs/calibration.md` from `docs/data/shared_stream_effect.csv`. For every case the streams-only change is of the same size and sign as the combined change, so redrawing the IMU explains little of it. `outage_control`, which has no vision, comes out identical to the shared run, as it must. The sample is the same ten development seeds, so the earlier limit stands: a large systematic effect is excluded, a small one is not. +**Follow-up: the two changes apart.** The study now has a third seeding, `streams_only`: the independent vision seed with the IMU noise pinned to its shared draw, for the duration of one run and with nothing in `src/navkit/degrade` changed. It gives the stream relationship alone, with the IMU fixed. The medians under it are generated on `docs/calibration.md` from `docs/data/shared_stream_effect.csv`. For the three cases with vision, the streams-only change of the ATE RMSE and mean NEES medians has the same sign as the combined change and the same order of size, so redrawing the IMU explains little of it. Both changes are within seed noise. `outage_control`, which has no vision, comes out identical to the shared run, as it must. The sample is the same ten development seeds, so the earlier limit stands: a large systematic effect is excluded, a small one is not. **Decision.** The streams are not changed. No committed number, table or golden is regenerated, and the B1 result in [ADR-0028](0028-the-b1-gate.md) stands as it was produced. The shared stream is documented instead, in `docs/MODEL.md` and in the calibration page, where the measurement is generated from its CSV. diff --git a/docs/euroc.md b/docs/euroc.md index 605b8b5..7b82143 100644 --- a/docs/euroc.md +++ b/docs/euroc.md @@ -6,7 +6,8 @@ decision behind it is [ADR-0013](adr/0013-recorded-imu-with-simulated-gnss.md); are discussed in [ADR-0014](adr/0014-textbook-imu-process-noise-by-default.md). Citation: Burri, M. et al. (2016). The EuRoC micro aerial vehicle datasets. *The International Journal of Robotics -Research*, 35(10), 1157-1163. Licence: set by the dataset's authors; read the terms on the dataset page before reuse. Not verified here. +Research*, 35(10), 1157-1163. Rights statement: "In Copyright - Non-Commercial Use Permitted", set by the dataset's authors. Whether the committed +result rows may stay published is open ([ADR-0042](adr/0042-publishing-rows-derived-from-euroc.md)). ## Read this before quoting anything diff --git a/docs/faq.md b/docs/faq.md index 03c5e32..f551196 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -70,7 +70,9 @@ navkit euroc compare --markdown # default vs preset over every fet ``` The data is not vendored. Its rights statement is "In Copyright - Non-Commercial Use Permitted", so -the files go under `data/raw/`, which git ignores, and only aggregate results may be committed. A +the files go under `data/raw/`, which git ignores. Results computed from them are committed, including per-frame +front-end errors in `docs/data/vo_errors.csv`; whether those rows may stay is open +([ADR-0042](adr/0042-publishing-rows-derived-from-euroc.md)). A result from this path is labelled `real_imu_simulated_gnss`. It tests the filter against real inertial noise and a known reference. It is not GNSS-denied navigation in the field. The decision behind it is [ADR-0013](adr/0013-recorded-imu-with-simulated-gnss.md). diff --git a/docs/tumvi.md b/docs/tumvi.md index 7243e10..0b3eabf 100644 --- a/docs/tumvi.md +++ b/docs/tumvi.md @@ -5,7 +5,7 @@ ADIS16448). The decision behind it is [ADR-0016](adr/0016-tum-vi-as-a-second-rec the one of [ADR-0013](adr/0013-recorded-imu-with-simulated-gnss.md). Citation: Schubert, D. et al. (2018). The TUM VI benchmark for evaluating visual-inertial odometry. *IROS 2018*. -Licence: set by the dataset's authors; read the terms on the dataset page before reuse. Not verified here. +Licence: CC BY 4.0, as the dataset page states. The attribution is under "Read this before quoting anything" below. ## Read this before quoting anything diff --git a/docs/urbannav.md b/docs/urbannav.md index 2041be8..3ded3eb 100644 --- a/docs/urbannav.md +++ b/docs/urbannav.md @@ -322,7 +322,7 @@ What the section tests: attitude normalised errors over the whole run and by third of the run, the pooled 2-sigma coverage per axis, the sigma the filter claims against the receiver's own, and the fixes' own normalised error against the receiver's reported sigma. -2. **How the gate behaves on real multipath fixes.** The rows are the fixes it rejects, whether the channel is +2. **How the gate behaves on real urban fixes.** Multipath is the likely cause of their errors, but it is not measured here. The rows are the fixes it rejects, whether the channel is declared faulty, and the NIS against the gate's level. **As declared**, one scalar GNSS sigma for every fix: diff --git a/mkdocs.yml b/mkdocs.yml index fee230f..9468b7f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,7 +4,7 @@ site_url: https://telschow.github.io/contested-nav/ repo_url: https://github.com/Telschow/contested-nav repo_name: Telschow/contested-nav edit_uri: edit/main/docs/ -copyright: MIT licensed. Not field validated. +copyright: Code MIT licensed. Results derived from EuRoC are for non-commercial use (see the README). Not field validated. docs_dir: docs site_dir: site diff --git a/src/navkit/urbannav_imposed.py b/src/navkit/urbannav_imposed.py index 257fe2a..b1f5573 100644 --- a/src/navkit/urbannav_imposed.py +++ b/src/navkit/urbannav_imposed.py @@ -18,8 +18,8 @@ question to answer here. Results are rows in the format of :func:`urbannav_run.rows_for`, under a sequence name of their own -(``deepurban1_imposed_s_s``). This module writes no CSV and no page. Rows in the committed -UrbanNav CSVs are not touched. +(``deepurban1_imposed_s_s``). This module writes no page. With ``--csv`` it merges its rows +into that file under those names; rows of other sequences in the file are not touched. """ from __future__ import annotations diff --git a/tests/test_urbannav_run.py b/tests/test_urbannav_run.py index 4ec1fd3..3db2d80 100644 --- a/tests/test_urbannav_run.py +++ b/tests/test_urbannav_run.py @@ -240,7 +240,7 @@ def test_the_deep_urban_section_says_what_it_is_and_what_it_is_not(): "aided run", "not applicable", "right about how wrong it is", - "multipath fixes", + "not measured here", "It is not field performance", "Imposed outages are a separate experiment", ): From dae0d96ddb028fca8775db1c9c214f1cda7b25d6 Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:38:30 +0100 Subject: [PATCH 3/4] docs(sandcastle): the merger prompt no longer claims main rejects its pushes The Main ruleset targets refs/heads/Main, so main enforces only deletion and non-fast-forward, and the sandbox token is an admin token. The prompt's guard stays; only the false reason is gone. Co-Authored-By: Claude Opus 5.5 --- .sandcastle/merge-prompt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.sandcastle/merge-prompt.md b/.sandcastle/merge-prompt.md index f5bf95d..396500a 100644 --- a/.sandcastle/merge-prompt.md +++ b/.sandcastle/merge-prompt.md @@ -27,11 +27,11 @@ If something fails, fix it and rerun only what failed. If the doc tables or metr # PUSH -`main` rejects pushes from this sandbox (403), so deliver through a pull request. Only when every check above passes: +Deliver through a pull request. Do not count on `main` rejecting a push: the branch rules do not stop this token today. Only when every check above passes: 1. `git fetch origin` 2. `git merge origin/main --no-edit`. If this brings in changes, rerun the checks. -3. Check `git branch --show-current` is not `main`, then `git push origin HEAD` (the current branch, never `main`). If the current branch is `main`, stop and say so: a push to `main` skips the pull request, and the 403 is not guaranteed. +3. Check `git branch --show-current` is not `main`, then `git push origin HEAD` (the current branch, never `main`). If the current branch is `main`, stop and say so: a push to `main` skips the pull request and nothing on the server stops it. Never force-push and never rewrite history. If the push is rejected, fetch, merge `origin/main` once more and try once more. If it is rejected again, stop and say so. From 320e1ec9fac9ad1a345ff992c455729691988881 Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:59:14 +0100 Subject: [PATCH 4/4] fix(seed-policy): gate b1 takes development seeds or exactly 100-124; config seeds are checked - `navkit gate b1` accepted seeds from 125, the ADR-0030 blocks of the candidate gate, without any flag, and `--held-out` accepted any seeds. Both the parser and run_gate now go through b1_gate.check_seeds: development seeds, or exactly the ADR-0028 set with --held-out. - `navkit run --config` used the YAML's own GNSS, vision and camera-drop seeds unchecked when --seed was absent. run_case now refuses held-out configured seeds when no seed replaces them, and navkit run checks every selected case before the first one runs. Test-first. Every new refusal test stops at a monkeypatched sentinel before any run, so a red phase cannot draw a held-out seed. The test that asserted 125-149 is accepted without the flag now asserts the refusal. Co-Authored-By: Claude Opus 5.5 --- src/navkit/b1_gate.py | 28 ++++++++++++++----- src/navkit/benchmark.py | 20 +++++++++++++ src/navkit/seed_policy.py | 6 ++-- tests/test_b1_gate.py | 27 ++++++++++++++++-- tests/test_seed_policy.py | 59 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 11 deletions(-) diff --git a/src/navkit/b1_gate.py b/src/navkit/b1_gate.py index df0d7ed..94152d5 100644 --- a/src/navkit/b1_gate.py +++ b/src/navkit/b1_gate.py @@ -9,7 +9,8 @@ applies the conditions. Seeds 100 to 124 are the held-out set. ADR-0028 allows one evaluation of it. The command refuses them unless -``--held-out`` is given, and so does :func:`run_gate`. +``--held-out`` is given, and so does :func:`run_gate`. ``--held-out`` takes exactly that set. Every other seed must be +a development seed: the later held-out blocks belong to :mod:`navkit.candidate_gate` (ADR-0030). Two further rows use the same function on the development seeds and are labelled DIAGNOSTIC: the single-anchor ``outage_visual`` (the gate has to be able to say no) and the plain clone ``outage_visual_clone``. They are not gate @@ -108,6 +109,18 @@ def _touches_held_out(seeds: list[int]) -> bool: return any(s in HELD_OUT_SEEDS for s in seeds) +def check_seeds(seeds: list[int], held_out: bool) -> None: + """Raise ``ValueError`` unless ``seeds`` are development seeds, or exactly the ADR-0028 set with ``held_out``.""" + span = f"{HELD_OUT_SEEDS.start} to {HELD_OUT_SEEDS.stop - 1}" + if held_out: + if list(seeds) != list(HELD_OUT_SEEDS): + raise ValueError(f"with --held-out the seeds must be exactly {span}, the held-out set of ADR-0028") + return + if _touches_held_out(seeds): + raise ValueError(f"seeds {span} are the held-out set of ADR-0028; give --held-out to evaluate it (single use)") + seed_policy.require_development(seeds) + + # --------------------------------------------------------------------------------------------- the runs @@ -169,8 +182,7 @@ def run_arrays(name: str, case: dict[str, Any], defaults: dict[str, Any], seeds: def run_gate(seeds: list[int], held_out: bool = False, diagnostics: bool = True) -> list[dict[str, Any]]: """Run the candidate, its control and (on development seeds) the two diagnostics, and return the CSV rows.""" - if _touches_held_out(seeds) and not held_out: - raise ValueError("seeds 100 to 124 are the held-out set of ADR-0028 and need held_out=True") + check_seeds(seeds, held_out) if len(seeds) < 2: raise ValueError("the gate needs at least two seeds") doc = _document() @@ -297,14 +309,16 @@ def markdown_table(rows: list[dict[str, Any]]) -> str: class _Parser(argparse.ArgumentParser): - """Refuses the held-out seeds unless ``--held-out`` is given, and says the set is single use when it is.""" + """Refuses seeds :func:`check_seeds` refuses, and says the set is single use when ``--held-out`` is given.""" def parse_args(self, args: Any = None, namespace: Any = None) -> argparse.Namespace: # type: ignore[override] ns = super().parse_args(args, namespace) + try: + check_seeds(ns.seeds, ns.held_out) + except ValueError as exc: + self.error(str(exc)) if ns.held_out: print("warning: the held-out set (seeds 100 to 124) is single use per ADR-0028.", file=sys.stderr) - elif _touches_held_out(ns.seeds): - self.error("seeds 100 to 124 are the held-out set of ADR-0028; give --held-out to evaluate it (single use)") return ns @@ -320,7 +334,7 @@ def build_parser() -> argparse.ArgumentParser: ap.add_argument("--seeds", type=_seeds_arg, required=True, help="A-B or a comma list; the gate uses exactly 25") ap.add_argument("--csv", type=Path, required=True, help="write the rows here") ap.add_argument("--page", type=Path, default=None, help="write the table between the b1-gate-dev markers") - ap.add_argument("--held-out", action="store_true", help="allow seeds 100 to 124 (single use, ADR-0028)") + ap.add_argument("--held-out", action="store_true", help="take exactly seeds 100 to 124 (single use, ADR-0028)") return ap diff --git a/src/navkit/benchmark.py b/src/navkit/benchmark.py index a3091e0..e2b4eea 100644 --- a/src/navkit/benchmark.py +++ b/src/navkit/benchmark.py @@ -277,6 +277,14 @@ def _eskf_config(scenario: Scenario, keys: dict[str, Any]) -> EskfConfig: ) +def stream_seeds(scenario: Scenario) -> list[int]: + """The seeds of a scenario's stochastic streams: GNSS, vision and, when it has one, camera drop.""" + seeds = [scenario.gnss.seed, scenario.vision.seed] + if scenario.camera_drop is not None: + seeds.append(scenario.camera_drop.seed) + return seeds + + def run_case( name: str, case: dict[str, Any], @@ -357,6 +365,10 @@ def run_case( scenario.vision.seed = seed if scenario.camera_drop is not None: scenario.camera_drop.seed = seed + else: + # The case's own seeds are used as written, so they must be development seeds. Only an explicit ``seed`` + # replaces them: the sweeps check theirs, and the two gates pass their own sets. + seed_policy.require_development(stream_seeds(scenario)) # Clean streams from the analytic reference, then degraded by the scenario. reference = synthetic_trajectory(syn, name=f"{name}-reference") @@ -629,6 +641,14 @@ def main(argv: list[str] | None = None) -> int: unknown = [k for k in selected if k not in cases] if unknown: raise SystemExit(f"unknown case(s): {', '.join(unknown)}; known: {', '.join(cases)}") + if args.seed is None: + # Refuse before the first case runs, not when run_case reaches the offending one. + for name in selected: + scenario = scenario_from_dict(cases[name].get("scenario", {})) + try: + seed_policy.require_development(stream_seeds(scenario) if scenario is not None else []) + except ValueError as exc: + ap.error(f"case {name!r} in {args.config}: {exc}") results: list[dict[str, Any]] = [] for name in selected: diff --git a/src/navkit/seed_policy.py b/src/navkit/seed_policy.py index ddb1799..1ecf151 100644 --- a/src/navkit/seed_policy.py +++ b/src/navkit/seed_policy.py @@ -3,8 +3,10 @@ Seeds 0 to 99 are development seeds. Seeds 100 to 124 are the held-out set of ADR-0028, spent. From 125 the seeds are held-out blocks of 25 (ADR-0030), run one at a time through :mod:`navkit.candidate_gate`. Every command that runs a case on seed numbers calls :func:`require_development` before it starts: the ``navkit sweep`` entry points, -``navkit run --seed`` and ``--scene-seed``, ``navkit euroc run`` and ``navkit euroc compare``. The B1 gate and the -candidate gate are the two procedures allowed to use held-out seeds. +``navkit run --seed`` and ``--scene-seed``, ``navkit euroc run`` and ``navkit euroc compare``. Seeds a config file +sets itself are checked too: by ``navkit run`` before the first case, and by :func:`navkit.benchmark.run_case` +whenever no ``seed`` replaces them. The B1 gate and the candidate gate are the two procedures allowed to use held-out +seeds; the B1 gate takes only its own set, 100 to 124, and only with ``--held-out``. """ from __future__ import annotations diff --git a/tests/test_b1_gate.py b/tests/test_b1_gate.py index 4cd3cd9..ef50e9f 100644 --- a/tests/test_b1_gate.py +++ b/tests/test_b1_gate.py @@ -262,7 +262,9 @@ def test_bad_input_is_refused(): # ------------------------------------------------------------------------------ the held-out seeds guard -@pytest.mark.parametrize("text", ["100-124", "0-124", "124-124", "100,101", "99-100", "24-100"]) +@pytest.mark.parametrize( + "text", ["100-124", "0-124", "124-124", "100,101", "99-100", "24-100", "125-149", "99-125", "150-174"] +) def test_the_command_refuses_the_held_out_seeds_without_the_flag(text, capsys): with pytest.raises(SystemExit) as stop: b1_gate.build_parser().parse_args(["--seeds", text, "--csv", "x.csv"]) @@ -270,7 +272,7 @@ def test_the_command_refuses_the_held_out_seeds_without_the_flag(text, capsys): assert "held-out" in capsys.readouterr().err -@pytest.mark.parametrize("text", ["0-24", "0-99", "125-149", "24-24", "5,6,7"]) +@pytest.mark.parametrize("text", ["0-24", "0-99", "24-24", "5,6,7"]) def test_the_command_accepts_other_seeds_without_the_flag(text): args = b1_gate.build_parser().parse_args(["--seeds", text, "--csv", "x.csv"]) assert args.held_out is False and len(args.seeds) >= 1 @@ -282,6 +284,27 @@ def test_the_flag_allows_the_held_out_seeds_and_says_it_is_single_use(capsys): assert "single use" in capsys.readouterr().err +@pytest.mark.parametrize("text", ["125-149", "0-24", "101-125", "100-123"]) +def test_the_flag_allows_only_the_b1_held_out_set(text, capsys): + """Later blocks belong to the candidate gate (ADR-0030); the flag is not a way around it.""" + with pytest.raises(SystemExit) as stop: + b1_gate.build_parser().parse_args(["--seeds", text, "--csv", "x.csv", "--held-out"]) + assert stop.value.code == 2 + assert "100 to 124" in capsys.readouterr().err + + +def test_the_run_function_refuses_later_blocks_with_or_without_the_flag(monkeypatch): + def never(*a, **k): # if the guard is missing, stop here: no held-out seed is ever run + raise AssertionError("the gate reached a run") + + monkeypatch.setattr(b1_gate, "_document", never) + monkeypatch.setattr(b1_gate, "run_arrays", never) + with pytest.raises(ValueError, match="held-out"): + b1_gate.run_gate(list(range(125, 150))) + with pytest.raises(ValueError, match="100 to 124"): + b1_gate.run_gate(list(range(125, 150)), held_out=True) + + def test_the_run_function_refuses_the_held_out_seeds_itself(): with pytest.raises(ValueError, match="held-out"): b1_gate.run_gate(list(range(100, 125))) diff --git a/tests/test_seed_policy.py b/tests/test_seed_policy.py index a74f465..75a6d32 100644 --- a/tests/test_seed_policy.py +++ b/tests/test_seed_policy.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + import pytest from navkit import cli, euroc_compare, euroc_eval, mismatch_sweep, outage_sweep, scene_sweep, seed_policy, seed_sweep @@ -217,3 +219,60 @@ def never(seeds): # the refusal must come before any case runs assert e.value.code == 2 assert "held-out" in capsys.readouterr().err assert calls == [] + + +# --- seeds a config file sets itself ------------------------------------------------------------ + + +def _config_with_gnss_seed(tmp_path, seed: int) -> tuple[Path, str, dict]: + import yaml + + from navkit.benchmark import default_config_path + + doc = yaml.safe_load(default_config_path().read_text()) + name = next(n for n, c in doc["cases"].items() if (c.get("scenario") or {}).get("gnss")) + doc["cases"][name]["scenario"]["gnss"]["seed"] = seed + path = tmp_path / "cfg.yaml" + path.write_text(yaml.safe_dump(doc)) + return path, name, doc + + +def test_run_refuses_a_config_whose_own_seed_is_held_out(tmp_path, monkeypatch, capsys) -> None: + from navkit import benchmark + + calls: list[str] = [] + monkeypatch.setattr(benchmark, "run_case", lambda name, *a, **k: calls.append(name)) + path, name, _ = _config_with_gnss_seed(tmp_path, 150) + with pytest.raises(SystemExit) as e: + cli.main(["run", "--config", str(path), "--only", name]) + assert e.value.code == 2 + assert "held-out" in capsys.readouterr().err + assert calls == [] + + +def test_run_case_refuses_a_configured_held_out_seed_before_it_builds_anything(tmp_path, monkeypatch) -> None: + from navkit import benchmark + + def never(*a, **k): # if the guard is missing, stop before any seeded draw + raise AssertionError("run_case reached the streams") + + monkeypatch.setattr(benchmark, "synthetic_trajectory", never) + _, name, doc = _config_with_gnss_seed(tmp_path, 150) + with pytest.raises(ValueError, match="held-out"): + benchmark.run_case(name, doc["cases"][name], doc.get("defaults", {})) + + +def test_an_explicit_seed_replaces_the_configured_one(tmp_path, monkeypatch) -> None: + """The sweeps and the gates pass ``seed=``. The config's own value is then not used, so it is not checked.""" + from navkit import benchmark + + class Reached(Exception): + pass + + def stop(*a, **k): + raise Reached + + monkeypatch.setattr(benchmark, "synthetic_trajectory", stop) + _, name, doc = _config_with_gnss_seed(tmp_path, 150) + with pytest.raises(Reached): + benchmark.run_case(name, doc["cases"][name], doc.get("defaults", {}), seed=5)