Read the steady-state verdict from result_summary.json

README.md

@@ -491,9 +491,7 @@ validation by the service or reviewers.
 | `load-pattern` | §6.1 | `load_pattern` is `concurrency` or `agentic_inference`, with a positive level |
 | `streaming-config` | §6.5 | Reports client IPC chunk forwarding; either flag value is accepted and neither proves server streaming |
 | `point-duration` | §6.2 | Steady-state window's issue-time span meets the region minimum: 600 s at concurrency ≤ 32, otherwise 1,200 s, or 2,400 s for agentic (warn) |
-| `steady-state-valid` | §4.4 | `status`, `verdict` and gating `state` use the spec's vocabulary |
-| `steady-state-consistency` | §4.4 | The reported status agrees with the window it describes |
-| `steady-state-basis` | §4.4 | Which basis supplies the official result; flags fallbacks and drift (warn) |
+| `steady-state-basis` | §4.4 | Reads the client's verdict from `result_summary.json` and reports which basis supplies the official result; flags fallbacks and drift. Fixed-concurrency points only (warn) |
 | `min-query-count` | §6.4 | `n_samples_completed` meets the dataset minimum |
 | `warmup-present` | §6.3.3 | Warmup declaration present; concurrency zero is valid only with zero duration and request counts |
 | `warmup-logs-retained` | §6.3.2 | Warmup log retention declared when warmup was performed (warn) |

src/submission_checker/data/messages.yaml

@@ -710,36 +710,25 @@ steady-state-basis:
   title: Steady state basis
   spec: §4.4
   messages:
-    warn: 'Point {c}: no steady_state block; metrics are whole-run, which §4.4 makes the fallback rather
-      than the official basis'
+    warn:
+      text: 'Point {c}: result_summary.json has no steady-state verdict (detection was off); metrics are
+        whole-run, which §4.4 makes the fallback rather than the official basis'
+      fix: 'Run the benchmark with settings.steady_state.enabled: true (--steady-state).'
+    warn-absent:
+      text: 'Point {c}: result_summary.json has no steady_state field; the client predates steady-state
+        reporting, so metrics are whole-run (§4.4 fallback)'
+      fix: 'Rerun with a client that reports steady state, with settings.steady_state.enabled: true.'
+    unreadable:
+      text: 'Point {c}: result_summary.json steady_state is not the block the client writes; metrics are
+        treated as whole-run'
+      fix: Ship result_summary.json exactly as the client wrote it.
     pass: 'Point {c}: official result is the steady-state window ({value} super-passes)'
     warn-2: 'Point {c}: status {status!r} — official result falls back to whole-run `total`; steady-state
       numbers are low-confidence (§4.4)'
     drifting: 'Point {c}: verdict {verdict!r} — a drifting gating metric is reported as a range or slope,
       never as a point estimate'
     drifting-metrics: 'Point {c}: verdict {verdict!r} — a drifting gating metric is reported as a range
       or slope, never as a point estimate ({metrics})'
-steady-state-consistency:
-  title: Steady state consistency
-  spec: §4.4
-  messages:
-    pass: steady_state status {status!r} consistent with the reported window
-    fail: '{problems}'
-    status-windowable-over-super: status 'windowable' over {spans} super-pass(es); §4.4 needs ≥ {min_trend_n}
-      (MIN_TREND_N)
-    status-windowable-drifting-gating: 'status ''windowable'' with drifting gating metric(s): {drifting};
-      §4.4 requires every one to be a Plateau'
-    status-insufficient-passes-over: status 'insufficient_passes' over {spans} super-pass(es), which meets
-      the ≥ {min_trend_n} floor
-steady-state-valid:
-  title: Steady state valid
-  spec: §4.4
-  messages:
-    pass: steady_state status={status!r} verdict={verdict!r}
-    fail: '{problems}'
-    status-expected-one: status {status!r} (expected one of {steady_state_statuses})
-    verdict-expected-one: verdict {verdict!r} (expected one of {steady_state_verdicts})
-    state-expected-one: state[{metric}] {state!r} (expected one of {gating_states})
 streaming-config:
   title: Streaming config
   spec: §6.5

src/submission_checker/models/aggregate/point_result.py

@@ -7,10 +7,18 @@
 
 __all__ = ["MIN_QUERY_COUNT", "PointResult"]
 
-from pydantic import BaseModel, ConfigDict, PrivateAttr, ValidationInfo, model_validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    PrivateAttr,
+    ValidationError,
+    ValidationInfo,
+    model_validator,
+)
 
 from ..file.point_config import OFFLINE_DEDICATED, PointConfig
 from ..file.point_summary import PointSummary
+from ..file.steady_state import SteadyState, from_client_verdict
 from ..regions import classify_concurrency, min_duration_ms
 from ..results import CheckResult, err, ok, warn
 
@@ -57,16 +65,13 @@ def _check_point_duration(self, info: ValidationInfo) -> PointResult:
         The point's own load pattern decides; a frontier whose points disagree is
         reported by ``benchmark-type-consistency``.
         """
-        regions = (info.context or {}).get("regions")
         summary_path: Path | None = (info.context or {}).get("summary_path")
-        if regions is None:
-            return self
+        minimum = self._min_duration_ms(info)
+        if minimum is None:
+            return self  # no regions, or already flagged by concurrency-in-range
+        region, min_ms = minimum
         c = self.config.concurrency
-        region = classify_concurrency(c, regions)
-        if region is None:
-            return self  # already flagged by concurrency-in-range
-        min_ms = min_duration_ms(region, c, agentic=self.config.is_agentic)
-        duration_ms, basis = self._duration_basis()
+        duration_ms, basis = self._duration_basis(self._steady_state(info))
         if duration_ms < min_ms:
             self._check_results.append(
                 warn(
@@ -107,13 +112,26 @@ def _check_steady_state_basis(self, info: ValidationInfo) -> PointResult:
         A ``drifting_up`` / ``drifting_down`` verdict is flagged separately: §4.4 says
         such a metric is reported "as drift (range/slope), never as a point estimate",
         which a percentile read out of ``result_summary.json`` silently is.
+
+        The verdict is the one the client wrote to ``result_summary.json``, read by
+        :func:`~submission_checker.models.file.steady_state.from_client_verdict`.
+        §4.4 covers fixed-concurrency points only, so agentic points and a dedicated
+        Offline run are skipped.
         """
         path: Path | None = (info.context or {}).get("summary_path")
-        block = self.config.steady_state
         c = self.config.concurrency
-        if block is None:
+        if self.config.is_agentic or self.config.offline == OFFLINE_DEDICATED:
+            return self
+        if not self.summary.has_steady_state_key:
+            self._check_results.append(warn("steady-state-basis", "warn-absent", path, c=c))
+            return self
+        if self.summary.steady_state is None:
             self._check_results.append(warn("steady-state-basis", "warn", path, c=c))
             return self
+        block = self._steady_state(info)
+        if block is None:
+            self._check_results.append(warn("steady-state-basis", "unreadable", path, c=c))
+            return self
 
         if block.is_official:
             self._check_results.append(
@@ -174,15 +192,41 @@ def _check_min_query_count(self, info: ValidationInfo) -> PointResult:
             )
         return self
 
-    def _duration_basis(self) -> tuple[float, str]:
+    def _min_duration_ms(self, info: ValidationInfo) -> tuple[str, float] | None:
+        """Return ``(region, §6.2 minimum in ms)``, or None when the region is unknown."""
+        regions = (info.context or {}).get("regions")
+        if regions is None:
+            return None
+        c = self.config.concurrency
+        region = classify_concurrency(c, regions)
+        if region is None:
+            return None
+        return region, min_duration_ms(region, c, agentic=self.config.is_agentic)
+
+    def _steady_state(self, info: ValidationInfo) -> SteadyState | None:
+        """The client's §4.4 verdict, or None when it reports none or an unreadable one."""
+        raw = self.summary.steady_state
+        if not isinstance(raw, dict):
+            return None
+        minimum = self._min_duration_ms(info)
+        try:
+            return from_client_verdict(raw, None if minimum is None else minimum[1] / 1000.0)
+        except ValidationError:
+            return None
+
+    def _duration_basis(self, block: SteadyState | None) -> tuple[float, str]:
         """Return ``(duration in ms, what it measures)`` for the §6.2 comparison.
 
         §4.4 measures §6.2 against the steady-state window's issue-time span. A point
-        that reports no window — or one whose window states no ``duration_s`` — falls
-        back to the whole run, which is §4.4's own fallback basis.
+        whose client found no window falls back to the whole run, which is §4.4's own
+        fallback basis. A window the client rejected as too short is not a steady
+        window, so it does not count either.
         """
-        block = self.config.steady_state
-        if block is not None and block.window.duration_s is not None:
+        if (
+            block is not None
+            and block.verdict != "not found"
+            and block.window.duration_s is not None
+        ):
             return block.window.duration_s * 1000.0, "steady-state window"
         return self.summary.duration_ms, "whole-run"
 

src/submission_checker/models/file/point_config.py

@@ -5,7 +5,7 @@
 import re
 from pathlib import Path
 
-from ...messages import Invalid, fragment
+from ...messages import Invalid
 
 __all__ = ["DpShortfall", "NodesUsed", "PointConfig", "RuntimeSettings", "WarmupSpec"]
 
@@ -20,14 +20,6 @@
 
 from ..regions import REGION_NAMES, SUBMITTERS_CHOICE
 from ..results import CheckResult, err, ok, warn
-from .steady_state import (
-    GATING_STATES,
-    MIN_TREND_N,
-    OFFICIAL_STATUS,
-    STEADY_STATE_STATUSES,
-    STEADY_STATE_VERDICTS,
-    SteadyState,
-)
 
 # §8.3's own table still lists the v0.7 *_throughput names, but §5.5's reference
 # algorithm and §9.1's check names both use *_concurrency. The algorithm wins.
@@ -283,9 +275,6 @@ class PointConfig(BaseModel):
     #: §5.7 Offline-point declaration. Absent is equivalent to ``none``.
     offline: str | None = None
 
-    #: §4.4 reporting basis. Absent on a point that predates the steady-state rules.
-    steady_state: SteadyState | None = None
-
     #: §8.3 speculative-decoding disclosure. Absent when the point used none.
     speculative_decoding: dict[str, object] | None = None
 
@@ -352,111 +341,6 @@ def _check_offline_declared(self, info: ValidationInfo) -> PointConfig:
             self._check_results.append(ok("offline-declared", "pass", path, offline=self.offline))
         return self
 
-    @model_validator(mode="after")
-    def _check_steady_state_vocabulary(self, info: ValidationInfo) -> PointConfig:
-        """§4.4: `status`, `verdict` and per-metric `state` use the spec's vocabulary."""
-        path: Path | None = (info.context or {}).get("yaml_path")
-        block = self.steady_state
-        if block is None:
-            return self
-
-        bad: list[str] = []
-        if block.status is not None and block.status not in STEADY_STATE_STATUSES:
-            bad.append(
-                fragment(
-                    "steady-state-valid",
-                    "status-expected-one",
-                    status=block.status,
-                    steady_state_statuses=", ".join(STEADY_STATE_STATUSES),
-                )
-            )
-        if block.verdict is not None and block.verdict not in STEADY_STATE_VERDICTS:
-            bad.append(
-                fragment(
-                    "steady-state-valid",
-                    "verdict-expected-one",
-                    verdict=block.verdict,
-                    steady_state_verdicts=", ".join(STEADY_STATE_VERDICTS),
-                )
-            )
-        for metric, state in sorted(block.state.items()):
-            if state not in GATING_STATES:
-                bad.append(
-                    fragment(
-                        "steady-state-valid",
-                        "state-expected-one",
-                        metric=metric,
-                        state=state,
-                        gating_states=", ".join(GATING_STATES),
-                    )
-                )
-
-        if bad:
-            self._check_results.append(
-                err("steady-state-valid", "fail", path, problems="; ".join(bad))
-            )
-        else:
-            self._check_results.append(
-                ok("steady-state-valid", "pass", path, status=block.status, verdict=block.verdict)
-            )
-        return self
-
-    @model_validator(mode="after")
-    def _check_steady_state_consistency(self, info: ValidationInfo) -> PointConfig:
-        """§4.4: the reported status must agree with the window it describes.
-
-        The status table keys on two things the block also states outright — how many
-        super-passes the window spans, and whether every gating metric is a Plateau.
-        A block claiming ``windowable`` over three super-passes is internally
-        inconsistent, and since ``windowable`` is what makes the windowed numbers
-        *official*, the disagreement decides which metrics get published.
-        """
-        path: Path | None = (info.context or {}).get("yaml_path")
-        block = self.steady_state
-        if block is None or block.status is None:
-            return self
-
-        problems: list[str] = []
-        spans = block.window.super_passes
-        if block.status == OFFICIAL_STATUS:
-            if spans is not None and spans < MIN_TREND_N:
-                problems.append(
-                    fragment(
-                        "steady-state-consistency",
-                        "status-windowable-over-super",
-                        spans=spans,
-                        min_trend_n=MIN_TREND_N,
-                    )
-                )
-            drifting = block.drifting_metrics
-            if drifting:
-                problems.append(
-                    fragment(
-                        "steady-state-consistency",
-                        "status-windowable-drifting-gating",
-                        drifting=", ".join(drifting),
-                    )
-                )
-        elif block.status == "insufficient_passes" and spans is not None and spans >= MIN_TREND_N:
-            problems.append(
-                fragment(
-                    "steady-state-consistency",
-                    "status-insufficient-passes-over",
-                    spans=spans,
-                    min_trend_n=MIN_TREND_N,
-                )
-            )
-
-        if problems:
-            self._check_results.append(
-                err("steady-state-consistency", "fail", path, problems="; ".join(problems))
-            )
-        else:
-            self._check_results.append(
-                ok("steady-state-consistency", "pass", path, status=block.status)
-            )
-        return self
-
     @model_validator(mode="after")
     def _check_disclosure_complete(self, info: ValidationInfo) -> PointConfig:
         """§8.3 + §9.1 "Submission completeness": every disclosure field must be present."""

src/submission_checker/models/file/point_summary.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import math
+from typing import Any
 
 from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator
 
@@ -78,6 +79,17 @@ class PointSummary(BaseModel):
     output_tokens_per_turn_total: float | None = None
     e2e_turn_time_seconds_total: float | None = None
 
+    #: §4.4 verdict, as the client writes it. ``None`` when detection did not run;
+    #: absent altogether from clients that predate it (see :attr:`has_steady_state_key`).
+    #: Kept raw so a malformed block is reported by ``steady-state-basis`` rather than
+    #: failing the whole summary.
+    steady_state: Any = None
+
+    @property
+    def has_steady_state_key(self) -> bool:
+        """True when the summary carries ``steady_state`` at all, even as ``null``."""
+        return "steady_state" in self.model_fields_set
+
     @computed_field  # type: ignore[prop-decorator]
     @property
     def duration_ms(self) -> float: