Skip to content

feat(config): register MLPerf Endpoints v1.0 seed set B (cohort 2026-11-C0) #564

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arekay-nv merged 1 commit into from
Nov 14, 2026
Merged

feat(config): register MLPerf Endpoints v1.0 seed set B (cohort 2026-11-C0) #564

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion src/inference_endpoint/config/rulesets/mlcommons/rules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,9 +367,19 @@ def apply_user_config(
benchmark_rulesets={},
)

# Set B, cohort 2026-11-C0; `model_seed` unpinned as for set A:
# https://github.com/mlcommons/endpoints_policies/blob/aa7537c656a514db74b1dc607d1996c323954658/seedset.yaml
_endpoints_v1_0_2026_11_C0_B = RoundRuleset(
version="endpoints-v1.0-2026-11-C0-B",
scheduler_rng_seed=17312635550606430962,
sample_index_rng_seed=6679150691998295617,
benchmark_rulesets={},
)

# Every published cohort stays registered: a submission binds to one set for
# its full update window, so an older set must keep resolving after a newer
# one is published. Add new sets here, do not replace.
ENDPOINTS_ALL = [_endpoints_v1_0_2026_10_C1_A]
ENDPOINTS_ALL = [_endpoints_v1_0_2026_10_C1_A, _endpoints_v1_0_2026_11_C0_B]

# Stays on set A: advancing it would re-seed runs bound via mlperf-endpoints-current.
ENDPOINTS_CURRENT = _endpoints_v1_0_2026_10_C1_A
18 changes: 15 additions & 3 deletions tests/unit/config/rulesets/mlcommons/test_rules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,9 @@ def test_edge_ruleset_apply_user_config():
# Verbatim from mlcommons/endpoints_policies seedset.yaml, cohort 2026-10-C1 set A.
_EP_SCHED_SEED = 10487924139932647040
_EP_SAMPLE_SEED = 586478644936801402
# Verbatim from mlcommons/endpoints_policies seedset.yaml, cohort 2026-11-C0 set B.
_EP_B_SCHED_SEED = 17312635550606430962
_EP_B_SAMPLE_SEED = 6679150691998295617


@pytest.mark.unit
Expand All @@ -262,6 +265,13 @@ def test_endpoints_v1_0_official_seeds():
assert ep.sample_index_rng_seed == _EP_SAMPLE_SEED


@pytest.mark.unit
def test_endpoints_v1_0_set_b_official_seeds():
ep = get_ruleset("mlperf-endpoints-v1.0-2026-11-C0-B")
assert ep.scheduler_rng_seed == _EP_B_SCHED_SEED
assert ep.sample_index_rng_seed == _EP_B_SAMPLE_SEED


@pytest.mark.unit
def test_endpoints_ruleset_registered():
assert get_ruleset("mlperf-endpoints-v1.0-2026-10-C1-A") is ENDPOINTS_CURRENT
Expand All @@ -274,9 +284,11 @@ def test_endpoints_ruleset_registered():
def test_endpoints_seeds_differ_from_every_other_registered_ruleset():
"""Endpoints cohorts rotate independently of the other rulesets; a shared
value would mean one of the two was transcribed from the wrong source."""
for other in [*ALL_ROUNDS, EDGE_CURRENT]:
assert ENDPOINTS_CURRENT.scheduler_rng_seed != other.scheduler_rng_seed
assert ENDPOINTS_CURRENT.sample_index_rng_seed != other.sample_index_rng_seed
for ep in ENDPOINTS_ALL:
for other in [*ALL_ROUNDS, EDGE_CURRENT, *ENDPOINTS_ALL]:
if other is not ep:
assert ep.scheduler_rng_seed != other.scheduler_rng_seed
assert ep.sample_index_rng_seed != other.sample_index_rng_seed


@pytest.mark.unit
Expand Down
Loading