Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -510,7 +510,6 @@ validation by the service or reviewers.
| `seed-runtime-match` | §2.1.1 | The RNG seeds equal the bound set's values |
| `target-cohort` | §4.6 | `target_cohort` matches `YYYY-MM-C0` / `YYYY-MM-C1` |
| `seed-set-adoption` | §4.6 | `target_cohort` falls inside the set's four-cohort adoption window |
| `seed-config-legacy` | §4.6 | v0.7 fallback: seeds == 42 when no `seed_set` is declared |
| `seed-set-registry` | §4.6 | Warns when the seed-set file itself cannot be read |

### Speculative decoding (§2.9.4)
Expand Down
8 changes: 0 additions & 8 deletions src/submission_checker/data/messages.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -630,14 +630,6 @@ result-summary-present:
fail-2:
text: 'Missing result_summary.json for {point_dir_name}: {relative_to}'
fix: Add the point's result_summary.json, as the benchmark writes it.
seed-config-legacy:
title: Seed config legacy
spec: §4.6
messages:
fail: 'Point {concurrency}: {field_name} = {val!r}; v0.7 required 42 and v1.0 requires a declared
seed_set (§4.6) — neither is satisfied'
warn: 'Point {concurrency}: {field_name} = 42 with no seed_set declared; v1.0 binds seeds to a published
set (§4.6)'
seed-runtime-match:
title: Seed runtime match
spec: §4.6
Expand Down
41 changes: 0 additions & 41 deletions src/submission_checker/models/file/point_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -490,47 +490,6 @@ def _check_target_cohort(self, info: ValidationInfo) -> PointConfig:
)
return self

@model_validator(mode="after")
def _check_seeds_legacy(self, info: ValidationInfo) -> PointConfig:
"""v0.7 fallback: seeds must equal 42 when the point binds no seed set.

v0.7 fixed every seed at 42. v1.0 rotates them (§4.6), so the constant is now
wrong for any point that names a ``seed_set`` — those are checked against the
published registry by
:class:`~submission_checker.models.aggregate.SeedBinding` instead. This rule
survives only to give a v0.7 bundle a comprehensible message.
"""
path: Path | None = (info.context or {}).get("yaml_path")
if self.seed_set is not None:
return self
rt = self.runtime_settings.runtime
for field_name, val in (
("runtime_settings.runtime.scheduler_rng_seed", rt.scheduler_rng_seed),
("runtime_settings.runtime.sample_index_rng_seed", rt.sample_index_rng_seed),
):
if val != 42:
self._check_results.append(
err(
"seed-config-legacy",
"fail",
path,
concurrency=self.concurrency,
field_name=field_name,
val=val,
)
)
else:
self._check_results.append(
warn(
"seed-config-legacy",
"warn",
path,
concurrency=self.concurrency,
field_name=field_name,
)
)
return self

@model_validator(mode="after")
def _check_warmup(self, info: ValidationInfo) -> PointConfig:
"""§6.3.3: warmup declaration is required for every measurement point."""
Expand Down
24 changes: 24 additions & 0 deletions tests/submission_checker/test_checks_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,30 @@ def test_valid_region_values_accepted(self, tmp_path, region):
)


@pytest.mark.unit
class TestSeedSetMissing:
@pytest.mark.parametrize("seed", [42, 7])
def test_one_disclosure_error_and_no_seed_value_check(self, tmp_path, seed):
"""§4.6: no seed set means no binding; seed 42 is not a fallback."""
config = PointConfig.model_validate(
{
"concurrency": 64,
"runtime_settings": {
"runtime": {"scheduler_rng_seed": seed, "sample_index_rng_seed": seed}
},
},
context={"yaml_path": tmp_path / "point_64.yaml"},
)
disclosure = [
r
for r in config._check_results
if r.rule == "point-disclosure-complete" and r.severity == Severity.ERROR
]
assert len(disclosure) == 1
assert "seed_set" in disclosure[0].message
assert not [r for r in config._check_results if r.rule.startswith("seed")]


@pytest.mark.unit
class TestAccuracyResultModel:
def test_scores_accessible(self):
Expand Down
Loading