From 382e0a61732c3aa973e4242e80e4dc99e8d53fe8 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Nov 2026 17:43:11 +0530 Subject: [PATCH] Remove seed-config-legacy rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A point with no seed_set is already rejected by point-disclosure-complete. The legacy rule added a warning when the seeds were 42, which read as if 42 were a valid fallback. §4.6 and §9.1 have no such fallback. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 - src/submission_checker/data/messages.yaml | 8 ---- .../models/file/point_config.py | 41 ------------------- tests/submission_checker/test_checks_file.py | 24 +++++++++++ 4 files changed, 24 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 610b641..c1277ac 100644 --- a/README.md +++ b/README.md @@ -510,7 +510,6 @@ validation by the service or reviewers. | `seed-runtime-match` | §2.1.1 | The RNG seeds equal the bound set's values | | `target-cohort` | §4.6 | `target_cohort` matches `YYYY-MM-C0` / `YYYY-MM-C1` | | `seed-set-adoption` | §4.6 | `target_cohort` falls inside the set's four-cohort adoption window | -| `seed-config-legacy` | §4.6 | v0.7 fallback: seeds == 42 when no `seed_set` is declared | | `seed-set-registry` | §4.6 | Warns when the seed-set file itself cannot be read | ### Speculative decoding (§2.9.4) diff --git a/src/submission_checker/data/messages.yaml b/src/submission_checker/data/messages.yaml index a4ead9b..99fa1ef 100644 --- a/src/submission_checker/data/messages.yaml +++ b/src/submission_checker/data/messages.yaml @@ -630,14 +630,6 @@ result-summary-present: fail-2: text: 'Missing result_summary.json for {point_dir_name}: {relative_to}' fix: Add the point's result_summary.json, as the benchmark writes it. -seed-config-legacy: - title: Seed config legacy - spec: §4.6 - messages: - fail: 'Point {concurrency}: {field_name} = {val!r}; v0.7 required 42 and v1.0 requires a declared - seed_set (§4.6) — neither is satisfied' - warn: 'Point {concurrency}: {field_name} = 42 with no seed_set declared; v1.0 binds seeds to a published - set (§4.6)' seed-runtime-match: title: Seed runtime match spec: §4.6 diff --git a/src/submission_checker/models/file/point_config.py b/src/submission_checker/models/file/point_config.py index 119a08b..a1b5cdd 100644 --- a/src/submission_checker/models/file/point_config.py +++ b/src/submission_checker/models/file/point_config.py @@ -490,47 +490,6 @@ def _check_target_cohort(self, info: ValidationInfo) -> PointConfig: ) return self - @model_validator(mode="after") - def _check_seeds_legacy(self, info: ValidationInfo) -> PointConfig: - """v0.7 fallback: seeds must equal 42 when the point binds no seed set. - - v0.7 fixed every seed at 42. v1.0 rotates them (§4.6), so the constant is now - wrong for any point that names a ``seed_set`` — those are checked against the - published registry by - :class:`~submission_checker.models.aggregate.SeedBinding` instead. This rule - survives only to give a v0.7 bundle a comprehensible message. - """ - path: Path | None = (info.context or {}).get("yaml_path") - if self.seed_set is not None: - return self - rt = self.runtime_settings.runtime - for field_name, val in ( - ("runtime_settings.runtime.scheduler_rng_seed", rt.scheduler_rng_seed), - ("runtime_settings.runtime.sample_index_rng_seed", rt.sample_index_rng_seed), - ): - if val != 42: - self._check_results.append( - err( - "seed-config-legacy", - "fail", - path, - concurrency=self.concurrency, - field_name=field_name, - val=val, - ) - ) - else: - self._check_results.append( - warn( - "seed-config-legacy", - "warn", - path, - concurrency=self.concurrency, - field_name=field_name, - ) - ) - return self - @model_validator(mode="after") def _check_warmup(self, info: ValidationInfo) -> PointConfig: """§6.3.3: warmup declaration is required for every measurement point.""" diff --git a/tests/submission_checker/test_checks_file.py b/tests/submission_checker/test_checks_file.py index a8d0f15..ff7a7a3 100644 --- a/tests/submission_checker/test_checks_file.py +++ b/tests/submission_checker/test_checks_file.py @@ -169,6 +169,30 @@ def test_valid_region_values_accepted(self, tmp_path, region): ) +@pytest.mark.unit +class TestSeedSetMissing: + @pytest.mark.parametrize("seed", [42, 7]) + def test_one_disclosure_error_and_no_seed_value_check(self, tmp_path, seed): + """§4.6: no seed set means no binding; seed 42 is not a fallback.""" + config = PointConfig.model_validate( + { + "concurrency": 64, + "runtime_settings": { + "runtime": {"scheduler_rng_seed": seed, "sample_index_rng_seed": seed} + }, + }, + context={"yaml_path": tmp_path / "point_64.yaml"}, + ) + disclosure = [ + r + for r in config._check_results + if r.rule == "point-disclosure-complete" and r.severity == Severity.ERROR + ] + assert len(disclosure) == 1 + assert "seed_set" in disclosure[0].message + assert not [r for r in config._check_results if r.rule.startswith("seed")] + + @pytest.mark.unit class TestAccuracyResultModel: def test_scores_accessible(self):