diff --git a/README.md b/README.md index 610b641..cd20ce8 100644 --- a/README.md +++ b/README.md @@ -491,9 +491,7 @@ validation by the service or reviewers. | `load-pattern` | §6.1 | `load_pattern` is `concurrency` or `agentic_inference`, with a positive level | | `streaming-config` | §6.5 | Reports client IPC chunk forwarding; either flag value is accepted and neither proves server streaming | | `point-duration` | §6.2 | Steady-state window's issue-time span meets the region minimum: 600 s at concurrency ≤ 32, otherwise 1,200 s, or 2,400 s for agentic (warn) | -| `steady-state-valid` | §4.4 | `status`, `verdict` and gating `state` use the spec's vocabulary | -| `steady-state-consistency` | §4.4 | The reported status agrees with the window it describes | -| `steady-state-basis` | §4.4 | Which basis supplies the official result; flags fallbacks and drift (warn) | +| `steady-state-basis` | §4.4 | Reads the client's verdict from `result_summary.json` and reports which basis supplies the official result; flags fallbacks and drift. Fixed-concurrency points only (warn) | | `min-query-count` | §6.4 | `n_samples_completed` meets the dataset minimum | | `warmup-present` | §6.3.3 | Warmup declaration present; concurrency zero is valid only with zero duration and request counts | | `warmup-logs-retained` | §6.3.2 | Warmup log retention declared when warmup was performed (warn) | diff --git a/src/submission_checker/data/messages.yaml b/src/submission_checker/data/messages.yaml index a4ead9b..b9eec45 100644 --- a/src/submission_checker/data/messages.yaml +++ b/src/submission_checker/data/messages.yaml @@ -710,8 +710,18 @@ steady-state-basis: title: Steady state basis spec: §4.4 messages: - warn: 'Point {c}: no steady_state block; metrics are whole-run, which §4.4 makes the fallback rather - than the official basis' + warn: + text: 'Point {c}: result_summary.json has no steady-state verdict (detection was off); metrics are + whole-run, which §4.4 makes the fallback rather than the official basis' + fix: 'Run the benchmark with settings.steady_state.enabled: true (--steady-state).' + warn-absent: + text: 'Point {c}: result_summary.json has no steady_state field; the client predates steady-state + reporting, so metrics are whole-run (§4.4 fallback)' + fix: 'Rerun with a client that reports steady state, with settings.steady_state.enabled: true.' + unreadable: + text: 'Point {c}: result_summary.json steady_state is not the block the client writes; metrics are + treated as whole-run' + fix: Ship result_summary.json exactly as the client wrote it. pass: 'Point {c}: official result is the steady-state window ({value} super-passes)' warn-2: 'Point {c}: status {status!r} — official result falls back to whole-run `total`; steady-state numbers are low-confidence (§4.4)' @@ -719,27 +729,6 @@ steady-state-basis: never as a point estimate' drifting-metrics: 'Point {c}: verdict {verdict!r} — a drifting gating metric is reported as a range or slope, never as a point estimate ({metrics})' -steady-state-consistency: - title: Steady state consistency - spec: §4.4 - messages: - pass: steady_state status {status!r} consistent with the reported window - fail: '{problems}' - status-windowable-over-super: status 'windowable' over {spans} super-pass(es); §4.4 needs ≥ {min_trend_n} - (MIN_TREND_N) - status-windowable-drifting-gating: 'status ''windowable'' with drifting gating metric(s): {drifting}; - §4.4 requires every one to be a Plateau' - status-insufficient-passes-over: status 'insufficient_passes' over {spans} super-pass(es), which meets - the ≥ {min_trend_n} floor -steady-state-valid: - title: Steady state valid - spec: §4.4 - messages: - pass: steady_state status={status!r} verdict={verdict!r} - fail: '{problems}' - status-expected-one: status {status!r} (expected one of {steady_state_statuses}) - verdict-expected-one: verdict {verdict!r} (expected one of {steady_state_verdicts}) - state-expected-one: state[{metric}] {state!r} (expected one of {gating_states}) streaming-config: title: Streaming config spec: §6.5 diff --git a/src/submission_checker/models/aggregate/point_result.py b/src/submission_checker/models/aggregate/point_result.py index 407c21f..3534d67 100644 --- a/src/submission_checker/models/aggregate/point_result.py +++ b/src/submission_checker/models/aggregate/point_result.py @@ -7,10 +7,18 @@ __all__ = ["MIN_QUERY_COUNT", "PointResult"] -from pydantic import BaseModel, ConfigDict, PrivateAttr, ValidationInfo, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + PrivateAttr, + ValidationError, + ValidationInfo, + model_validator, +) from ..file.point_config import OFFLINE_DEDICATED, PointConfig from ..file.point_summary import PointSummary +from ..file.steady_state import SteadyState, from_client_verdict from ..regions import classify_concurrency, min_duration_ms from ..results import CheckResult, err, ok, warn @@ -57,16 +65,13 @@ def _check_point_duration(self, info: ValidationInfo) -> PointResult: The point's own load pattern decides; a frontier whose points disagree is reported by ``benchmark-type-consistency``. """ - regions = (info.context or {}).get("regions") summary_path: Path | None = (info.context or {}).get("summary_path") - if regions is None: - return self + minimum = self._min_duration_ms(info) + if minimum is None: + return self # no regions, or already flagged by concurrency-in-range + region, min_ms = minimum c = self.config.concurrency - region = classify_concurrency(c, regions) - if region is None: - return self # already flagged by concurrency-in-range - min_ms = min_duration_ms(region, c, agentic=self.config.is_agentic) - duration_ms, basis = self._duration_basis() + duration_ms, basis = self._duration_basis(self._steady_state(info)) if duration_ms < min_ms: self._check_results.append( warn( @@ -107,13 +112,26 @@ def _check_steady_state_basis(self, info: ValidationInfo) -> PointResult: A ``drifting_up`` / ``drifting_down`` verdict is flagged separately: §4.4 says such a metric is reported "as drift (range/slope), never as a point estimate", which a percentile read out of ``result_summary.json`` silently is. + + The verdict is the one the client wrote to ``result_summary.json``, read by + :func:`~submission_checker.models.file.steady_state.from_client_verdict`. + §4.4 covers fixed-concurrency points only, so agentic points and a dedicated + Offline run are skipped. """ path: Path | None = (info.context or {}).get("summary_path") - block = self.config.steady_state c = self.config.concurrency - if block is None: + if self.config.is_agentic or self.config.offline == OFFLINE_DEDICATED: + return self + if not self.summary.has_steady_state_key: + self._check_results.append(warn("steady-state-basis", "warn-absent", path, c=c)) + return self + if self.summary.steady_state is None: self._check_results.append(warn("steady-state-basis", "warn", path, c=c)) return self + block = self._steady_state(info) + if block is None: + self._check_results.append(warn("steady-state-basis", "unreadable", path, c=c)) + return self if block.is_official: self._check_results.append( @@ -174,15 +192,41 @@ def _check_min_query_count(self, info: ValidationInfo) -> PointResult: ) return self - def _duration_basis(self) -> tuple[float, str]: + def _min_duration_ms(self, info: ValidationInfo) -> tuple[str, float] | None: + """Return ``(region, §6.2 minimum in ms)``, or None when the region is unknown.""" + regions = (info.context or {}).get("regions") + if regions is None: + return None + c = self.config.concurrency + region = classify_concurrency(c, regions) + if region is None: + return None + return region, min_duration_ms(region, c, agentic=self.config.is_agentic) + + def _steady_state(self, info: ValidationInfo) -> SteadyState | None: + """The client's §4.4 verdict, or None when it reports none or an unreadable one.""" + raw = self.summary.steady_state + if not isinstance(raw, dict): + return None + minimum = self._min_duration_ms(info) + try: + return from_client_verdict(raw, None if minimum is None else minimum[1] / 1000.0) + except ValidationError: + return None + + def _duration_basis(self, block: SteadyState | None) -> tuple[float, str]: """Return ``(duration in ms, what it measures)`` for the §6.2 comparison. §4.4 measures §6.2 against the steady-state window's issue-time span. A point - that reports no window — or one whose window states no ``duration_s`` — falls - back to the whole run, which is §4.4's own fallback basis. + whose client found no window falls back to the whole run, which is §4.4's own + fallback basis. A window the client rejected as too short is not a steady + window, so it does not count either. """ - block = self.config.steady_state - if block is not None and block.window.duration_s is not None: + if ( + block is not None + and block.verdict != "not found" + and block.window.duration_s is not None + ): return block.window.duration_s * 1000.0, "steady-state window" return self.summary.duration_ms, "whole-run" diff --git a/src/submission_checker/models/file/point_config.py b/src/submission_checker/models/file/point_config.py index 119a08b..69b583c 100644 --- a/src/submission_checker/models/file/point_config.py +++ b/src/submission_checker/models/file/point_config.py @@ -5,7 +5,7 @@ import re from pathlib import Path -from ...messages import Invalid, fragment +from ...messages import Invalid __all__ = ["DpShortfall", "NodesUsed", "PointConfig", "RuntimeSettings", "WarmupSpec"] @@ -20,14 +20,6 @@ from ..regions import REGION_NAMES, SUBMITTERS_CHOICE from ..results import CheckResult, err, ok, warn -from .steady_state import ( - GATING_STATES, - MIN_TREND_N, - OFFICIAL_STATUS, - STEADY_STATE_STATUSES, - STEADY_STATE_VERDICTS, - SteadyState, -) # §8.3's own table still lists the v0.7 *_throughput names, but §5.5's reference # algorithm and §9.1's check names both use *_concurrency. The algorithm wins. @@ -283,9 +275,6 @@ class PointConfig(BaseModel): #: §5.7 Offline-point declaration. Absent is equivalent to ``none``. offline: str | None = None - #: §4.4 reporting basis. Absent on a point that predates the steady-state rules. - steady_state: SteadyState | None = None - #: §8.3 speculative-decoding disclosure. Absent when the point used none. speculative_decoding: dict[str, object] | None = None @@ -352,111 +341,6 @@ def _check_offline_declared(self, info: ValidationInfo) -> PointConfig: self._check_results.append(ok("offline-declared", "pass", path, offline=self.offline)) return self - @model_validator(mode="after") - def _check_steady_state_vocabulary(self, info: ValidationInfo) -> PointConfig: - """§4.4: `status`, `verdict` and per-metric `state` use the spec's vocabulary.""" - path: Path | None = (info.context or {}).get("yaml_path") - block = self.steady_state - if block is None: - return self - - bad: list[str] = [] - if block.status is not None and block.status not in STEADY_STATE_STATUSES: - bad.append( - fragment( - "steady-state-valid", - "status-expected-one", - status=block.status, - steady_state_statuses=", ".join(STEADY_STATE_STATUSES), - ) - ) - if block.verdict is not None and block.verdict not in STEADY_STATE_VERDICTS: - bad.append( - fragment( - "steady-state-valid", - "verdict-expected-one", - verdict=block.verdict, - steady_state_verdicts=", ".join(STEADY_STATE_VERDICTS), - ) - ) - for metric, state in sorted(block.state.items()): - if state not in GATING_STATES: - bad.append( - fragment( - "steady-state-valid", - "state-expected-one", - metric=metric, - state=state, - gating_states=", ".join(GATING_STATES), - ) - ) - - if bad: - self._check_results.append( - err("steady-state-valid", "fail", path, problems="; ".join(bad)) - ) - else: - self._check_results.append( - ok("steady-state-valid", "pass", path, status=block.status, verdict=block.verdict) - ) - return self - - @model_validator(mode="after") - def _check_steady_state_consistency(self, info: ValidationInfo) -> PointConfig: - """§4.4: the reported status must agree with the window it describes. - - The status table keys on two things the block also states outright — how many - super-passes the window spans, and whether every gating metric is a Plateau. - A block claiming ``windowable`` over three super-passes is internally - inconsistent, and since ``windowable`` is what makes the windowed numbers - *official*, the disagreement decides which metrics get published. - """ - path: Path | None = (info.context or {}).get("yaml_path") - block = self.steady_state - if block is None or block.status is None: - return self - - problems: list[str] = [] - spans = block.window.super_passes - if block.status == OFFICIAL_STATUS: - if spans is not None and spans < MIN_TREND_N: - problems.append( - fragment( - "steady-state-consistency", - "status-windowable-over-super", - spans=spans, - min_trend_n=MIN_TREND_N, - ) - ) - drifting = block.drifting_metrics - if drifting: - problems.append( - fragment( - "steady-state-consistency", - "status-windowable-drifting-gating", - drifting=", ".join(drifting), - ) - ) - elif block.status == "insufficient_passes" and spans is not None and spans >= MIN_TREND_N: - problems.append( - fragment( - "steady-state-consistency", - "status-insufficient-passes-over", - spans=spans, - min_trend_n=MIN_TREND_N, - ) - ) - - if problems: - self._check_results.append( - err("steady-state-consistency", "fail", path, problems="; ".join(problems)) - ) - else: - self._check_results.append( - ok("steady-state-consistency", "pass", path, status=block.status) - ) - return self - @model_validator(mode="after") def _check_disclosure_complete(self, info: ValidationInfo) -> PointConfig: """§8.3 + §9.1 "Submission completeness": every disclosure field must be present.""" diff --git a/src/submission_checker/models/file/point_summary.py b/src/submission_checker/models/file/point_summary.py index 7b03c31..1978ded 100644 --- a/src/submission_checker/models/file/point_summary.py +++ b/src/submission_checker/models/file/point_summary.py @@ -3,6 +3,7 @@ from __future__ import annotations import math +from typing import Any from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator @@ -78,6 +79,17 @@ class PointSummary(BaseModel): output_tokens_per_turn_total: float | None = None e2e_turn_time_seconds_total: float | None = None + #: §4.4 verdict, as the client writes it. ``None`` when detection did not run; + #: absent altogether from clients that predate it (see :attr:`has_steady_state_key`). + #: Kept raw so a malformed block is reported by ``steady-state-basis`` rather than + #: failing the whole summary. + steady_state: Any = None + + @property + def has_steady_state_key(self) -> bool: + """True when the summary carries ``steady_state`` at all, even as ``null``.""" + return "steady_state" in self.model_fields_set + @computed_field # type: ignore[prop-decorator] @property def duration_ms(self) -> float: diff --git a/src/submission_checker/models/file/steady_state.py b/src/submission_checker/models/file/steady_state.py index 32b004e..8529034 100644 --- a/src/submission_checker/models/file/steady_state.py +++ b/src/submission_checker/models/file/steady_state.py @@ -8,14 +8,16 @@ result**, with the whole-run ``total`` metrics kept as supplementary. The checker cannot run the detector — that is a post-processing step over the durable -event log, and ``events.jsonl`` is not part of the submitted bundle. What it can do is -validate the block the submitter reports: that its vocabulary is the spec's, that the -status and shape agree with each other, and that the window it claims is long enough -to be the official result under §6.2. +event log, and ``events.jsonl`` is not part of the submitted bundle. The reference +client runs it when ``settings.steady_state.enabled`` is set and writes its verdict to +``result_summary.json``. :func:`from_client_verdict` reads that verdict into §4.4's +terms, so the checker never relies on a block copied by hand into ``point.yaml``. """ from __future__ import annotations +from typing import Any + from pydantic import BaseModel, ConfigDict, Field __all__ = [ @@ -24,8 +26,10 @@ "OFFICIAL_STATUS", "STEADY_STATE_STATUSES", "STEADY_STATE_VERDICTS", + "ClientSteadyState", "SteadyState", "SteadyStateWindow", + "from_client_verdict", ] #: §4.4's trend-test floor: a window must span at least this many super-passes. @@ -121,3 +125,116 @@ def drifting_metrics(self) -> list[str]: estimate", so a consumer of the reported percentile needs to know. """ return sorted(name for name, state in self.state.items() if state.startswith("Drifting")) + + +#: The client's per-metric trend (``global_trend``) in §4.4's gating-state terms. +#: ``insufficient`` (too few super-passes to test) has no §4.4 state and is left out. +_TREND_STATES = {"steady": "Plateau", "up": "Drifting Up", "down": "Drifting Down"} + + +class ClientWindow(BaseModel): + """The client's ``steady_state.window``. ``sp_hi`` is exclusive.""" + + model_config = ConfigDict(extra="allow") + + sp_lo: int + sp_hi: int + n_samples: int + start_ns: int + end_ns: int + + +class ClientAnomaly(BaseModel): + """The client's ``steady_state.anomaly``.""" + + model_config = ConfigDict(extra="allow") + + detected: bool = False + + +class ClientSteadyState(BaseModel): + """The ``steady_state`` block the reference client writes to ``result_summary.json``. + + Only the fields the checker reads are declared. ``window`` is the reported plateau; + when ``found`` is false it is the longest candidate the client rejected as too + short, or ``None`` when there was no plateau at all. + """ + + model_config = ConfigDict(extra="allow") + + found: bool + superpass_size: int + n_super_passes: int + warmup: int = 0 + window: ClientWindow | None = None + anomaly: ClientAnomaly = Field(default_factory=ClientAnomaly) + global_trend: dict[str, str] = Field(default_factory=dict) + + +def from_client_verdict(raw: dict[str, Any], min_duration_s: float | None) -> SteadyState: + """Read the client's verdict into a §4.4 block. + + ``status`` follows §4.4's status table. A run with no full super-pass after warmup + is ``partial_dataset``. A run with no accepted window of at least + :data:`MIN_TREND_N` super-passes is ``insufficient_passes``, unless the client + rejected a long enough window as too short, which is ``insufficient_duration``. An + accepted window shorter than *min_duration_s* (§6.2) is also + ``insufficient_duration``. Otherwise the point is ``windowable``. With + *min_duration_s* unknown, the duration test is skipped. + + ``verdict`` follows §4.4's shape table: ``not found`` when the client accepted no + window, then ``anomaly`` (the first plateau stays the result), then ``drifting_up`` + / ``drifting_down`` from the client's trend after the window, else + ``STEADY STATE``. + + Raises: + pydantic.ValidationError: *raw* is not the client's block. + """ + client = ClientSteadyState.model_validate(raw) + window = SteadyStateWindow() + passes = 0 + if client.window is not None: + cw = client.window + passes = cw.sp_hi - cw.sp_lo + window = SteadyStateWindow( + super_pass_start=cw.sp_lo, + super_pass_end=cw.sp_hi - 1, + super_pass_size=client.superpass_size, + n_samples=cw.n_samples, + duration_s=(cw.end_ns - cw.start_ns) / 1e9, + ) + state = { + metric: _TREND_STATES[trend] + for metric, trend in sorted(client.global_trend.items()) + if trend in _TREND_STATES + } + + if client.n_super_passes - client.warmup < 1: + status = "partial_dataset" + elif passes < MIN_TREND_N: + status = "insufficient_passes" + elif not client.found or ( + min_duration_s is not None and (window.duration_s or 0.0) < min_duration_s + ): + status = "insufficient_duration" + else: + status = OFFICIAL_STATUS + + if not client.found: + verdict = "not found" + elif client.anomaly.detected: + verdict = "anomaly" + elif "Drifting Up" in state.values(): + verdict = "drifting_up" + elif "Drifting Down" in state.values(): + verdict = "drifting_down" + else: + verdict = "STEADY STATE" + + return SteadyState( + status=status, + verdict=verdict, + window=window, + state=state, + anomaly=raw.get("anomaly") if client.anomaly.detected else None, + ) diff --git a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml index 7724cb8..328ee31 100644 --- a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml +++ b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16 diff --git a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json index e6e2bc2..d9a7528 100644 --- a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json +++ b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 800.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml index a7ecd2c..75eb9e3 100644 --- a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml +++ b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 38 diff --git a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json index 0fe18b5..21e34cc 100644 --- a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json +++ b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1500.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml index 4fa372c..6caefb1 100644 --- a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml +++ b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 88 diff --git a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json index 839c42f..e015125 100644 --- a/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json +++ b/test_submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2500.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml index 1554743..53ac363 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1024 diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json index 07c4aa7..935bed7 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 9734.9295, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml index 856b50a..b83958e 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json index d04481d..7a1299a 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 5687.0736, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml index df36161..93c7b99 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16 diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json index 41eff27..c7ddacc 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1888.2008, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml index 3a8b037..887909c 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 2048 diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json index db91311..9fb786c 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 9757.9893, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml index 1d9bf08..32b3929 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 4 diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json index 32bec22..db7a1e3 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 775.6654, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml index 2367f5b..f114d20 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json index 2fb9226..d31467a 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 9192.0955, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml index 3dd86c2..fe9ee5d 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json index f69318d..b622a38 100644 --- a/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json +++ b/test_submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 4308.1207, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml index 1554743..53ac363 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1024 diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json index ef0d7d5..8dea4de 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 8720.2384, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml index 856b50a..b83958e 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json index c10338b..8107c33 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 4883.5144, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml index df36161..93c7b99 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16 diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json index e6d7a27..00668ae 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1702.2933, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml index 3a8b037..887909c 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 2048 diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json index 7db6a29..ef8b24d 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 8643.1591, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml index 1d9bf08..32b3929 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 4 diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json index a8be486..2a0622d 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 734.3735, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml index 2367f5b..f114d20 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json index 6229216..94b81d9 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 8130.7031, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml index 3dd86c2..fe9ee5d 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json index c26fef0..3d0922d 100644 --- a/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json +++ b/test_submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 3515.8571, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml index a3aeda0..448fbc9 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json index fc8a189..fa58207 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1940.9001, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml index b1aecf4..a8c7e46 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16 diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json index 13e0192..88a6994 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 743.0109, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml index dffd958..a971337 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 256 diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json index e49af20..bb901c4 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2349.6027, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml index 866e7d5..abbed20 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 32 diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json index a53ebf6..872aabb 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 996.9355, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml index f857ef6..b6cb964 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json index d3f1c58..78d30f3 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2826.3269, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml index 81ed6e5..5e2bf7a 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json index d7c5651..743f66b 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1454.2447, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml index 9ad91e4..8bec364 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 8 diff --git a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json index 35c0965..9be1838 100644 --- a/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json +++ b/test_submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 419.8022, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml index d695ead..f28f0c1 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1024 diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json index d9903b6..844c91e 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2935.2758, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml index f7ac596..65f6096 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json index 6d128c4..3b03237 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1447.1003, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml index 32e0e6a..22cb646 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16 diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json index 9ba4737..564df5c 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 336.1901, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml index 0171d99..c820d07 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 256 diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json index 8fdf6e2..7bfcd45 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1930.3863, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml index 2f3f48c..da31539 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 32 diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json index bfb87ca..40b65c4 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 569.857, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml index b174385..f790532 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json index 4107afa..8c8f45c 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2521.2732, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml index 324b0fe..2e33d7e 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json index 7916313..fb1e23c 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 946.4836, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml index 92acc87..91f67d3 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 8 diff --git a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json index e825818..42c7683 100644 --- a/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json +++ b/test_submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 176.3306, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml index c6afd01..500d86c 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json index 30d450b..62dbe65 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 66.4212, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml index 7e04960..e9e12a0 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1024 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json index 64ae3c6..5f00945 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 3319.3792, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml index 33c4397..9400eaa 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json index 2971280..90a961f 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2727.5555, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml index ee2163a..a0ac8c3 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json index f2614d7..4bf4f9c 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 828.7308, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml index fe9d925..5ed98d4 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 2 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json index 800a347..20a1e8e 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 129.9917, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml index df0f631..e097cea 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 256 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json index 565ee9e..e269cb6 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 3148.1895, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml index d27c0e4..2afe9e6 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 32 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json index d2d1805..b6cd854 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1413.1885, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml index fc29131..1abf4b1 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 4 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json index 1e9c9a5..355935a 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 247.4485, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml index 5dc64d7..8ed62d2 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json index f5048b1..d042b9a 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 3329.9015, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml index 76ce106..d8555f3 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json index 0626373..5b54077 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2092.0486, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml index 755c228..36376d9 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 8 diff --git a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json index 2cbec8f..6dc8060 100644 --- a/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json +++ b/test_submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 459.6069, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml index c6afd01..500d86c 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json index 4bb4b90..8627cbf 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 37.5378, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml index 7e04960..e9e12a0 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1024 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json index f96d278..3910d35 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1690.9752, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml index 33c4397..9400eaa 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json index 9fd9647..8ea4e20 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1434.7653, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml index ee2163a..a0ac8c3 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json index 9c90716..426ab16 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 488.1576, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml index fe9d925..5ed98d4 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 2 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json index 29b907f..8204126 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 74.3196, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml index df0f631..e097cea 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 256 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json index 69e9f5c..f5faf8d 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1637.236, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml index d27c0e4..2afe9e6 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 32 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json index 744dda3..c2d1364 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 784.1499, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml index fc29131..1abf4b1 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 4 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json index 8e9864b..4f46b06 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 147.2254, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml index 5dc64d7..8ed62d2 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json index a2c730b..2831196 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1698.9345, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml index 76ce106..d8555f3 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json index 8108943..03f55c3 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1119.157, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml index 755c228..36376d9 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 8 diff --git a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json index 667db87..3423c70 100644 --- a/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json +++ b/test_submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 275.4528, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml index ad7c5d2..7c1bd37 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1024 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json index 0e75738..fd982d7 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 16663.052, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml index 488a177..d6248de 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json index 7bb8379..ec8aceb 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 5156.1156, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml index b8e40a8..130ade6 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1536 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json index c57070a..4427ed1 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 16080.1285, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml index d85fa3a..bb4f36a 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 192 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json index 1a0978c..61eb0aa 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 5918.8286, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml index ec3f96a..56ce9da 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 2048 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json index bb46230..538f4b0 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 17020.065, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml index 399f689..f1fc313 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 256 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json index 36daea8..df72b81 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 7679.78, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml index 3d504e2..59136f2 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 384 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json index ea56360..6bcc038 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 11517.7175, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml index 8f1b341..1d3a342 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json index ee10d72..439385e 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 13407.9619, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml index 39a2a8e..c9250ce 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json index 878b5a8..fc075d2 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2786.3735, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml index 9f9a1f4..aa17faa 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 768 diff --git a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json index 24c446c..5bab489 100644 --- a/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json +++ b/test_submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 15382.5387, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml index ad7c5d2..7c1bd37 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1024 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json index fcbea3f..3546467 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 15190.3577, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml index 488a177..d6248de 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json index 8428958..0f1629a 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 4240.8096, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml index b8e40a8..130ade6 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1536 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json index 6de815a..34e7fc8 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 14820.5046, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml index d85fa3a..bb4f36a 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 192 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json index 9410fd0..ac41f37 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 5945.235, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml index ec3f96a..56ce9da 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 2048 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json index abd1561..e8c78a4 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 14549.0024, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml index 399f689..f1fc313 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 256 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json index 7129f09..1f68c64 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 7453.2487, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml index 3d504e2..59136f2 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 384 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json index 2cdfd5d..566bc43 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 9928.3767, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml index 8f1b341..1d3a342 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json index 5bbd690..87c41e1 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 11900.548, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml index 39a2a8e..c9250ce 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json index 1ecf29e..862f14a 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2286.3769, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml index 9f9a1f4..aa17faa 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 768 diff --git a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json index 643f698..60d27a8 100644 --- a/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json +++ b/test_submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 11611.4547, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml index 1702327..accf099 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json index 57e2555..a4a5e1f 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 93.013, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml index 006ab14..3504e19 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json index 62ee53f..6f3e096 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2629.9466, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml index efda497..515b031 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json index d857471..7d7f1cf 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 688.0213, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml index b0344d7..434bda4 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 2 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json index 0d10e71..d9a2c4a 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 162.2332, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml index 28202d2..b34a977 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 256 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json index 3818268..b2ec967 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 4339.5446, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml index 3990136..8e13ca4 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 32 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json index 000263e..69aca93 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 944.9684, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml index 9dc2bcd..9c1d542 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 4 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json index ee85740..695257e 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 274.8203, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml index aefafe7..a4fe3c2 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json index e5a7060..936a045 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 6541.1293, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml index bee059f..ab5c7c7 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json index 8463177..4c9716d 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1539.3602, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml index 48698c9..303ed66 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 8 diff --git a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json index cac202f..27210c9 100644 --- a/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json +++ b/test_submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 451.6703, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml index cb5b9a5..0dd7938 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1024 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json index 5fa3c1f..6da24cf 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 106193.0063, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml index e09e64f..e37cb86 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 128 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json index e0332ec..a7acdef 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 25496.871, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml index 9e90cbf..4147642 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16384 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json index 4a036b1..1b8e4f7 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 375039.2793, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml index 2f419b9..a63ade8 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 2048 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json index dabf292..99480dd 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 137046.5114, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml index 8db22e8..5d52e3a 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 256 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json index 7b8c8f2..0450a17 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 43305.2862, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml index 28ed9c8..65ce119 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 32 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json index 2bfc7a8..5c5555f 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 6683.0628, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml index 3c50d9f..ba1b66d 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 4096 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json index f066438..dba6804 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 172665.1461, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml index 3282309..5d3ff26 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json index 65c5ee0..c1c4b22 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 69152.4701, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml index 0958c75..630abe9 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 64 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json index 051a5ab..9e55269 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 13135.2154, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml index a55bc8a..c0fba55 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 8192 diff --git a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json index 7d989d7..13b127b 100644 --- a/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json +++ b/test_submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 251036.0844, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml index 1fda013..8846b6e 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml @@ -22,20 +22,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 1000 diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json index 793001e..e5c187f 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 4500.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml index e0ef921..f38a28e 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 16 diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json index e6e2bc2..d9a7528 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 800.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 1600000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml index 7ef3f56..468881f 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 20 diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json index 28f9c21..8d7ca88 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 1500.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml index 20beabb..6a3bbb4 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 256 diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json index b9aa245..2ca07c9 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 4000.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml index d0bf5c0..793fbc6 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 512 diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json index 5c94bee..69bb35d 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 4500.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml index a54586a..18c2606 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 768 diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json index c44d18b..cf3fd87 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 4600.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml index 7a81603..7504678 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml @@ -21,20 +21,6 @@ runtime_settings: seed_set: A shared_docs: docs shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 target_cohort: 2026-10-C1 warmup: concurrency: 88 diff --git a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json index aee0f8a..83c55f3 100644 --- a/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json +++ b/test_submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json @@ -25,5 +25,42 @@ } }, "system_tps": 2500.0, - "tps_per_user": 181.8182 + "tps_per_user": 181.8182, + "steady_state": { + "found": true, + "reason": null, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": 1000000000000, + "end_ns": 2200000000000, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0 + }, + "anomaly": { + "detected": false, + "change_point_sp": null, + "delta_pct": 0.0, + "pettitt": null, + "plateaus": [ + [ + 0, + 4 + ] + ] + }, + "global_trend": { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady" + }, + "drifting_up": [] + } } diff --git a/tests/submission_checker/test_checks_steady_state.py b/tests/submission_checker/test_checks_steady_state.py index e6d91ba..5e8a992 100644 --- a/tests/submission_checker/test_checks_steady_state.py +++ b/tests/submission_checker/test_checks_steady_state.py @@ -3,60 +3,109 @@ """Tests for the §4.4 steady-state reporting basis. The checker cannot run the detector — that is post-processing over `events.jsonl`, -which is not in the bundle — so what is asserted here is that the *reported* block is -internally coherent, and that §6.2 is measured against the window rather than the run. +which is not in the bundle. The reference client runs it and writes its verdict to +`result_summary.json`; what is asserted here is how that verdict is read into §4.4's +terms, and that §6.2 is measured against the window rather than the run. """ from __future__ import annotations from pathlib import Path +from typing import Any import pytest +from pydantic import ValidationError from submission_checker.models import PointConfig, PointResult, Severity from submission_checker.models.file.steady_state import ( MIN_TREND_N, SteadyState, SteadyStateWindow, + from_client_verdict, ) from .conftest import _REGIONS, _summary +_ABSENT = object() -def _block( - status: str = "windowable", - verdict: str = "STEADY STATE", - super_passes: int = 4, + +def _verdict( + *, + found: bool = True, + passes: int | None = 4, duration_s: float = 1200.0, - state: dict[str, str] | None = None, -) -> dict: + n_super_passes: int = 5, + warmup: int = 1, + trend: dict[str, str] | None = None, + anomaly: bool = False, +) -> dict[str, Any]: + """A `steady_state` block in the shape the client writes (`sp_hi` is exclusive).""" + start_ns = 1_000_000_000_000 + window = ( + None + if passes is None + else { + "sp_lo": 0, + "sp_hi": passes, + "n_super_passes": passes, + "n_samples": 1000 * passes, + "start_ns": start_ns, + "end_ns": start_ns + int(duration_s * 1e9), + } + ) return { - "status": status, - "verdict": verdict, - "window": { - "super_pass_start": 1, - "super_pass_end": super_passes, - "super_pass_size": 1000, - "n_samples": 1000 * super_passes, - "duration_s": duration_s, + "found": found, + "reason": None if found else "no admissible steady plateau", + "superpass_size": 1000, + "n_super_passes": n_super_passes, + "warmup": warmup, + "window": window, + "anomaly": {"detected": anomaly, "change_point_sp": 3 if anomaly else None}, + "global_trend": trend + if trend is not None + else { + "ttft_p50": "steady", + "ttft_p90": "steady", + "tpot_p50": "steady", + "tpot_p90": "steady", }, - "state": state if state is not None else {"tpot_p50": "Plateau", "tpot_p90": "Plateau"}, + "drifting_up": [k for k, v in (trend or {}).items() if v == "up"], } -def _point(tmp_path: Path, steady_state: dict | None, concurrency: int = 64) -> PointConfig: +def _point( + tmp_path: Path, + concurrency: int = 64, + load_pattern: str = "concurrency", + **extra: Any, +) -> PointConfig: payload = { "concurrency": concurrency, "dataset": "mlperf-perf-dataset-v1", - "runtime_settings": {"load_pattern": "concurrency", "runtime": {}}, + "runtime_settings": {"load_pattern": load_pattern, "runtime": {}}, + **extra, } - if steady_state is not None: - payload["steady_state"] = steady_state return PointConfig.model_validate(payload, context={"yaml_path": tmp_path / "point.yaml"}) -def _errors(config: PointConfig, rule: str) -> list: - return [r for r in config._check_results if r.rule == rule and r.severity == Severity.ERROR] +def _result( + tmp_path: Path, + steady_state: Any = _ABSENT, + *, + run_duration_ns: float = 1_200_000_000_000.0, + config: PointConfig | None = None, +) -> PointResult: + summary = _summary(duration_ns=run_duration_ns) + if steady_state is not _ABSENT: + summary = summary.model_copy(update={"steady_state": steady_state}) + return PointResult.model_validate( + { + "config": config if config is not None else _point(tmp_path), + "summary": summary, + "yaml_path": tmp_path / "point.yaml", + }, + context={"regions": _REGIONS, "summary_path": tmp_path / "summary.json"}, + ) @pytest.mark.unit @@ -81,149 +130,179 @@ def test_drifting_metrics_are_listed(self) -> None: @pytest.mark.unit -class TestVocabulary: - def test_spec_values_accepted(self, tmp_path: Path) -> None: - assert not _errors(_point(tmp_path, _block()), "steady-state-valid") - - @pytest.mark.parametrize( - "field, value", - [("status", "great"), ("verdict", "looks_fine")], - ) - def test_unknown_value_errors(self, tmp_path: Path, field: str, value: str) -> None: - block = _block() - block[field] = value - assert _errors(_point(tmp_path, block), "steady-state-valid") - - def test_unknown_gating_state_errors(self, tmp_path: Path) -> None: - block = _block(state={"tpot_p90": "wobbling"}) - assert _errors(_point(tmp_path, block), "steady-state-valid") - - @pytest.mark.parametrize( - "verdict", ["STEADY STATE", "drifting_up", "drifting_down", "anomaly", "not found"] - ) - def test_every_spec_verdict_is_accepted(self, tmp_path: Path, verdict: str) -> None: - assert not _errors(_point(tmp_path, _block(verdict=verdict)), "steady-state-valid") - - def test_absent_block_is_not_a_finding(self, tmp_path: Path) -> None: - config = _point(tmp_path, None) - assert not [r for r in config._check_results if r.rule == "steady-state-valid"] - - -@pytest.mark.unit -class TestInternalConsistency: - """The status table keys on facts the block also states outright.""" - - def test_windowable_below_the_trend_floor_errors(self, tmp_path: Path) -> None: - block = _block(super_passes=MIN_TREND_N - 1) - hits = _errors(_point(tmp_path, block), "steady-state-consistency") - assert hits and "MIN_TREND_N" in hits[0].message - - def test_windowable_at_the_floor_passes(self, tmp_path: Path) -> None: - assert not _errors( - _point(tmp_path, _block(super_passes=MIN_TREND_N)), "steady-state-consistency" - ) - - def test_windowable_with_a_drifting_metric_errors(self, tmp_path: Path) -> None: - """§4.4 requires every gating metric to be a Plateau for the official result.""" - block = _block(state={"tpot_p50": "Plateau", "tpot_p90": "Drifting Up"}) - hits = _errors(_point(tmp_path, block), "steady-state-consistency") - assert hits and "tpot_p90" in hits[0].message - - def test_insufficient_passes_above_the_floor_errors(self, tmp_path: Path) -> None: - block = _block(status="insufficient_passes", super_passes=6) - assert _errors(_point(tmp_path, block), "steady-state-consistency") - - def test_insufficient_duration_below_the_floor_is_fine(self, tmp_path: Path) -> None: - """That status is about time, not passes, so a short window is consistent.""" - block = _block(status="insufficient_duration", super_passes=5, duration_s=10.0) - assert not _errors(_point(tmp_path, block), "steady-state-consistency") +class TestFromClientVerdict: + """The client's verdict, read into §4.4's status and shape tables.""" + + def test_window_end_is_converted_to_inclusive(self) -> None: + """The client's `sp_hi` is exclusive: [0, 4) is four super-passes, 0..3.""" + block = from_client_verdict(_verdict(passes=4), 1200.0) + assert block.window.super_pass_start == 0 + assert block.window.super_pass_end == 3 + assert block.window.super_passes == 4 + + def test_window_duration_is_the_issue_span(self) -> None: + block = from_client_verdict(_verdict(duration_s=1500.0), 1200.0) + assert block.window.duration_s == pytest.approx(1500.0) + + def test_clean_window_is_windowable(self) -> None: + block = from_client_verdict(_verdict(), 1200.0) + assert (block.status, block.verdict) == ("windowable", "STEADY STATE") + assert set(block.state.values()) == {"Plateau"} + + def test_window_below_the_trend_floor_is_insufficient_passes(self) -> None: + block = from_client_verdict(_verdict(passes=MIN_TREND_N - 1), 1200.0) + assert block.status == "insufficient_passes" + + def test_no_plateau_is_insufficient_passes_and_not_found(self) -> None: + block = from_client_verdict(_verdict(found=False, passes=None), 1200.0) + assert (block.status, block.verdict) == ("insufficient_passes", "not found") + + def test_under_one_super_pass_is_partial_dataset(self) -> None: + raw = _verdict(found=False, passes=None, n_super_passes=1, warmup=1) + assert from_client_verdict(raw, 1200.0).status == "partial_dataset" + + def test_window_shorter_than_section_6_2_is_insufficient_duration(self) -> None: + block = from_client_verdict(_verdict(duration_s=900.0), 1200.0) + assert (block.status, block.verdict) == ("insufficient_duration", "STEADY STATE") + + def test_window_the_client_rejected_as_short_is_insufficient_duration(self) -> None: + """`found: false` with a window is the longest plateau the client found too short.""" + block = from_client_verdict(_verdict(found=False, duration_s=1500.0), 1200.0) + assert (block.status, block.verdict) == ("insufficient_duration", "not found") + + def test_unknown_minimum_skips_the_duration_test(self) -> None: + assert from_client_verdict(_verdict(duration_s=10.0), None).status == "windowable" + + def test_drift_after_the_window_is_drifting_up(self) -> None: + """A local plateau with a metric climbing later (§4.4 shape table).""" + trend = {"ttft_p50": "up", "ttft_p90": "up", "tpot_p50": "steady", "tpot_p90": "steady"} + block = from_client_verdict(_verdict(trend=trend), 1200.0) + assert block.verdict == "drifting_up" + assert block.drifting_metrics == ["ttft_p50", "ttft_p90"] + + def test_downward_trend_is_drifting_down(self) -> None: + trend = {"tpot_p50": "down", "tpot_p90": "steady"} + assert from_client_verdict(_verdict(trend=trend), 1200.0).verdict == "drifting_down" + + def test_anomaly_takes_precedence_over_drift(self) -> None: + """§4.4 keeps the first plateau of a staircase as the result.""" + trend = {"tpot_p50": "up", "tpot_p90": "up"} + block = from_client_verdict(_verdict(trend=trend, anomaly=True), 1200.0) + assert block.verdict == "anomaly" + assert block.anomaly is not None + + def test_untestable_trend_has_no_state(self) -> None: + trend = {"tpot_p50": "steady", "tpot_p90": "insufficient"} + assert from_client_verdict(_verdict(trend=trend), 1200.0).state == {"tpot_p50": "Plateau"} + + def test_a_block_not_in_the_client_shape_raises(self) -> None: + with pytest.raises(ValidationError): + from_client_verdict({"status": "windowable", "verdict": "STEADY STATE"}, 1200.0) @pytest.mark.unit class TestDurationBasis: """§6.2's minimum is measured over the window's issue-time span (§4.4).""" - def _result(self, tmp_path: Path, steady_state: dict | None, run_duration_ns: float): - config = _point(tmp_path, steady_state, concurrency=64) - return PointResult.model_validate( - { - "config": config, - "summary": _summary(duration_ns=run_duration_ns), - "yaml_path": tmp_path / "point.yaml", - }, - context={"regions": _REGIONS, "summary_path": tmp_path / "summary.json"}, - ) - def _duration_hits(self, result: PointResult) -> list: return [r for r in result._check_results if r.rule == "point-duration"] def test_window_span_is_used_when_present(self, tmp_path: Path) -> None: """A long wall-clock run whose window is short must still be flagged.""" - result = self._result( - tmp_path, _block(duration_s=10.0), run_duration_ns=3_600_000_000_000.0 - ) + result = _result(tmp_path, _verdict(duration_s=10.0), run_duration_ns=3_600_000_000_000.0) hits = self._duration_hits(result) assert any(r.severity == Severity.WARNING for r in hits) assert any("steady-state window" in r.message for r in hits) def test_long_window_passes(self, tmp_path: Path) -> None: - result = self._result( - tmp_path, _block(duration_s=1200.0), run_duration_ns=1_200_000_000_000.0 - ) + result = _result(tmp_path, _verdict(duration_s=1200.0)) assert not [r for r in self._duration_hits(result) if r.severity == Severity.WARNING] - def test_falls_back_to_whole_run_without_a_block(self, tmp_path: Path) -> None: - result = self._result(tmp_path, None, run_duration_ns=1_200_000_000_000.0) - hits = self._duration_hits(result) + def test_falls_back_to_whole_run_without_a_verdict(self, tmp_path: Path) -> None: + hits = self._duration_hits(_result(tmp_path, None)) assert hits and any("whole-run" in r.message for r in hits) + def test_a_rejected_window_is_not_the_basis(self, tmp_path: Path) -> None: + """A window the client found too short is not a steady window.""" + result = _result(tmp_path, _verdict(found=False, duration_s=10.0)) + assert any("whole-run" in r.message for r in self._duration_hits(result)) + def test_wall_clock_no_longer_rescues_a_short_window(self, tmp_path: Path) -> None: """The drain is excluded by construction, so wall-clock overstates the basis.""" - short_window = self._result( - tmp_path, _block(duration_s=60.0), run_duration_ns=9_999_000_000_000.0 + short_window = _result( + tmp_path, _verdict(duration_s=60.0), run_duration_ns=9_999_000_000_000.0 ) assert [r for r in self._duration_hits(short_window) if r.severity == Severity.WARNING] @pytest.mark.unit class TestReportingBasis: - def _result(self, tmp_path: Path, steady_state: dict | None): - return PointResult.model_validate( - { - "config": _point(tmp_path, steady_state), - "summary": _summary(), - "yaml_path": tmp_path / "point.yaml", - }, - context={"regions": _REGIONS, "summary_path": tmp_path / "summary.json"}, - ) - def _basis(self, result: PointResult) -> list: return [r for r in result._check_results if r.rule == "steady-state-basis"] def test_windowable_is_official(self, tmp_path: Path) -> None: - hits = self._basis(self._result(tmp_path, _block())) + hits = self._basis(_result(tmp_path, _verdict())) assert hits and all(r.severity == Severity.INFO for r in hits) @pytest.mark.parametrize( - "status", ["insufficient_duration", "insufficient_passes", "partial_dataset"] + "verdict", + [ + _verdict(duration_s=900.0), + _verdict(passes=2), + _verdict(found=False, passes=None, n_super_passes=1, warmup=1), + ], + ids=["insufficient_duration", "insufficient_passes", "partial_dataset"], ) - def test_fallback_statuses_are_flagged(self, tmp_path: Path, status: str) -> None: - block = _block(status=status, super_passes=2 if status == "insufficient_passes" else 4) - hits = self._basis(self._result(tmp_path, block)) + def test_fallback_statuses_are_flagged(self, tmp_path: Path, verdict: dict) -> None: + hits = self._basis(_result(tmp_path, verdict)) assert any(r.severity == Severity.WARNING for r in hits) - def test_missing_block_is_flagged(self, tmp_path: Path) -> None: - hits = self._basis(self._result(tmp_path, None)) - assert any(r.severity == Severity.WARNING for r in hits) + def test_detection_off_is_flagged(self, tmp_path: Path) -> None: + hits = self._basis(_result(tmp_path, None)) + assert [r.message for r in hits if r.severity == Severity.WARNING] + assert "detection was off" in hits[0].message - @pytest.mark.parametrize("verdict", ["drifting_up", "drifting_down"]) - def test_drift_is_flagged_separately(self, tmp_path: Path, verdict: str) -> None: + def test_client_without_the_field_is_flagged(self, tmp_path: Path) -> None: + hits = self._basis(_result(tmp_path)) + assert hits and "predates steady-state reporting" in hits[0].message + + @pytest.mark.parametrize("raw", [{"status": "windowable"}, "windowable", 4]) + def test_unreadable_verdict_is_flagged(self, tmp_path: Path, raw: Any) -> None: + hits = self._basis(_result(tmp_path, raw)) + assert hits and "not the block the client writes" in hits[0].message + + @pytest.mark.parametrize( + "trend", + [{"tpot_p50": "up", "tpot_p90": "steady"}, {"tpot_p50": "down", "tpot_p90": "steady"}], + ids=["drifting_up", "drifting_down"], + ) + def test_drift_is_flagged_separately(self, tmp_path: Path, trend: dict) -> None: """§4.4: a drifting metric is a range or slope, never a point estimate.""" - block = _block(status="insufficient_passes", verdict=verdict, super_passes=2) - messages = [r.message for r in self._basis(self._result(tmp_path, block))] + messages = [r.message for r in self._basis(_result(tmp_path, _verdict(trend=trend)))] assert any("never as a point estimate" in m for m in messages) def test_anomaly_verdict_is_still_official(self, tmp_path: Path) -> None: """§4.4 accepts the first plateau of a staircase as the steady state.""" - hits = self._basis(self._result(tmp_path, _block(verdict="anomaly"))) + hits = self._basis(_result(tmp_path, _verdict(anomaly=True))) assert all(r.severity == Severity.INFO for r in hits) + + def test_a_point_yaml_block_is_not_read(self, tmp_path: Path) -> None: + """The verdict comes from result_summary.json, not a copy in point.yaml.""" + copied = {"status": "windowable", "verdict": "STEADY STATE"} + config = _point(tmp_path, steady_state=copied) + hits = self._basis(_result(tmp_path, None, config=config)) + assert any(r.severity == Severity.WARNING for r in hits) + + def test_agentic_points_are_out_of_scope(self, tmp_path: Path) -> None: + """§4.4 covers fixed-concurrency points; the client does not run it for agentic.""" + config = _point(tmp_path, load_pattern="agentic_inference") + assert not self._basis(_result(tmp_path, None, config=config)) + + def test_dedicated_offline_point_is_out_of_scope(self, tmp_path: Path) -> None: + config = _point(tmp_path, offline="dedicated") + assert not self._basis(_result(tmp_path, None, config=config)) + + def test_elected_offline_point_is_still_checked(self, tmp_path: Path) -> None: + """An elected C_max point remains a fixed-concurrency point (§5.7.2).""" + config = _point(tmp_path, offline="elected") + assert self._basis(_result(tmp_path, None, config=config)) diff --git a/tests/submission_checker/test_report_completeness.py b/tests/submission_checker/test_report_completeness.py index 6dab2ce..1228b07 100644 --- a/tests/submission_checker/test_report_completeness.py +++ b/tests/submission_checker/test_report_completeness.py @@ -146,7 +146,7 @@ def test_unreadable_c_max_skips_only_the_region_rules(self, submission: Path) -> # Rules that need neither the description nor the regions still run. assert _results(report, "point-count") assert _results(report, "ultra-low-concurrency-coverage") - assert _results(report, "steady-state-valid") + assert _results(report, "steady-state-basis") @pytest.mark.unit diff --git a/tests/tools/regenerate_fixtures.py b/tests/tools/regenerate_fixtures.py index ccf3030..424c746 100644 --- a/tests/tools/regenerate_fixtures.py +++ b/tests/tools/regenerate_fixtures.py @@ -45,6 +45,9 @@ 11. **Exactly ``N`` accuracy results** (§4.3, §5.3). v0.7 ran accuracy at most points; v1.0 allows one per mandatory band plus the Offline point and none at a submitter's-choice point, so the surplus ``accuracy_results.json`` files are removed. +12. **§4.4 verdict in ``result_summary.json``.** The checker reads the steady-state + verdict where the client writes it, so each summary gains a clean ``windowable`` + block in the client's shape and ``point.yaml`` loses the hand-written one. Run with ``uv run python tests/tools/regenerate_fixtures.py``; ``--check`` exits 1 if anything would change, which is what the idempotence test asserts. @@ -237,6 +240,7 @@ def _regenerate_curve( changes += _write_json(point_dir / layout.SYSTEM_DESC_JSON, desc, dry_run=dry_run) summary = _modernise_summary(summaries[point_dir], legacy_meta) + summary = _with_steady_state(summary, point_dir) changes += _write_json(point_dir / layout.RESULT_SUMMARY_JSON, summary, dry_run=dry_run) changes += _write_point_yaml(point_dir, concurrency, base_desc, seed_set, dry_run=dry_run) @@ -604,6 +608,60 @@ def _modernise_summary(summary: dict[str, Any], legacy_meta: dict[str, Any]) -> return out +def _with_steady_state(summary: dict[str, Any], point_dir: Path) -> dict[str, Any]: + """Give the summary a §4.4 verdict in the shape the client writes. + + The corpus is synthetic, so the verdict is a clean ``windowable`` result: the point + of the fixtures is to exercise the rules, and a corpus that fell back to ``total`` + everywhere would never reach them. The window spans four super-passes after one + warmup super-pass and lasts the point's ``min_duration_ms`` (1,200 s when unset). + """ + if "steady_state" in summary: + return summary + path = point_dir / layout.POINT_YAML + point = yaml.safe_load(path.read_text()) if path.exists() else None + runtime_settings = point.get("runtime_settings") if isinstance(point, dict) else None + min_ms = (runtime_settings or {}).get("min_duration_ms") or 0 + duration_ns = int(float(min_ms) * 1e6) or 1_200_000_000_000 + start_ns = 1_000_000_000_000 + steady = "steady" + return { + **summary, + "steady_state": { + "found": True, + "reason": None, + "superpass_size": 1000, + "n_super_passes": 5, + "warmup": 1, + "window": { + "sp_lo": 0, + "sp_hi": 4, + "n_super_passes": 4, + "n_samples": 4000, + "start_ns": start_ns, + "end_ns": start_ns + duration_ns, + "plateau_index": 0, + "n_plateaus": 1, + "skipped_short": 0, + }, + "anomaly": { + "detected": False, + "change_point_sp": None, + "delta_pct": 0.0, + "pettitt": None, + "plateaus": [[0, 4]], + }, + "global_trend": { + "ttft_p50": steady, + "ttft_p90": steady, + "tpot_p50": steady, + "tpot_p90": steady, + }, + "drifting_up": [], + }, + } + + def _dataset_type(value: object) -> str: """Normalise a dataset_type to one of §8.3's three spellings.""" text = str(value or "Performance").strip().lower() @@ -680,29 +738,8 @@ def _write_point_yaml( runtime_settings["runtime"] = runtime point["runtime_settings"] = runtime_settings - # §4.4 makes the steady-state window the official reporting basis. The corpus is - # synthetic, so the block is synthesised as a clean `windowable` result: the point - # of the fixtures is to exercise the rules, and a corpus that fell back to `total` - # everywhere would never reach them. - if "steady_state" not in point: - duration_s = float(point.get("runtime_settings", {}).get("min_duration_ms", 0)) / 1000.0 - point["steady_state"] = { - "status": "windowable", - "verdict": "STEADY STATE", - "window": { - "super_pass_start": 1, - "super_pass_end": 4, - "super_pass_size": 1000, - "n_samples": 4000, - "duration_s": duration_s or 1200.0, - }, - "state": { - "ttft_p50": "Plateau", - "ttft_p90": "Plateau", - "tpot_p50": "Plateau", - "tpot_p90": "Plateau", - }, - } + # The §4.4 verdict now lives in result_summary.json (see _with_steady_state). + point.pop("steady_state", None) warmup = point.get("warmup") if isinstance(warmup, dict):