fix(scenario): make any context overflow invalidate agentperf-v2 runs

docs/tutorials/agentperf-v2.md

@@ -40,6 +40,9 @@ to 12 hours.
 At high concurrency, cap aiperf's session cache so finished sessions cannot use up the client's memory: set
 `AIPERF_WORKER_SESSION_CACHE_MAX_ENTRIES` to about 8 × users ÷ workers. The default was fine at 16 users.
 
+Serve the model with a context window of at least 1M tokens. Every recorded request fits in 1M, so any context overflow
+marks the run `submission_valid=false`.
+
 Use the recordings of the model your server is serving. Your server must report cached tokens in its usage figures for the
 cache results to appear: for example, `--enable-cache-report` on SGLang.
 

src/aiperf/common/scenario/agentperf_v2.py

@@ -17,6 +17,9 @@
     default_trajectory_start_min_ratio=0.0,
     default_trajectory_start_max_ratio=1.0,
     require_cache_bust=CacheBustTarget.FIRST_TURN_PREFIX,
+    # Every recorded request fits a 1M-token window, so a correctly
+    # configured server never overflows; any overflow means skipped turns.
+    max_context_overflow_rate=0.0,
     loop=LoopRules(
         burn_in_seconds=900,
         ramp_seconds=300,

src/aiperf/common/scenario/base.py

@@ -154,6 +154,21 @@ class ScenarioSpec(AIPerfBaseModel):
         description="Space lanes evenly on one loop of the dataset's sessions, and "
         "measure one slice per lane, instead of sampling a random start per lane.",
     )
+    max_context_overflow_rate: float | None = Field(
+        default=None,
+        ge=0.0,
+        le=1.0,
+        description=(
+            "Scenario ceiling on the context-overflow rate (overflow responses / "
+            "total responses). A rate strictly above it flips submission_valid "
+            "to False. None falls back to "
+            "Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT. Set 0.0 when every "
+            "request in the dataset fits the required context window, so any "
+            "overflow (e.g. a server started with a shorter max context) "
+            "invalidates the run. The env override can only tighten, never "
+            "loosen, a scenario-set ceiling."
+        ),
+    )
     require_cache_bust: CacheBustTarget | None = Field(
         default=None,
         description=(

src/aiperf/exporters/aggregate/aggregate_base_exporter.py

@@ -11,6 +11,8 @@
 
 from aiperf.common.environment import Environment
 from aiperf.common.mixins import AIPerfLoggerMixin
+from aiperf.common.scenario.base import UnknownScenarioError
+from aiperf.common.scenario.registry import get_scenario
 from aiperf.orchestrator.aggregation.base import AggregateResult
 
 
@@ -31,6 +33,23 @@ def __getattr__(name: str) -> float:
 RUN_CANCELLED_REASON = "run_cancelled"
 
 
+def _context_overflow_rate_limit(scenario_name: str) -> float:
+    """Resolve the overflow-rate ceiling for ``scenario_name``.
+
+    The env limit (``AIPERF_AGENTX_CONTEXT_OVERFLOW_RATE_LIMIT``) is the
+    default. A scenario that declares ``max_context_overflow_rate`` gets the
+    stricter of the two, so the env var cannot loosen a scenario lock.
+    """
+    limit = Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT
+    try:
+        spec_limit = get_scenario(scenario_name).max_context_overflow_rate
+    except UnknownScenarioError:
+        # Unknown scenario names are rejected by the validator before a run;
+        # fall back to the env limit here rather than failing the export.
+        spec_limit = None
+    return limit if spec_limit is None else min(limit, spec_limit)
+
+
 def _build_run_metadata_dict(
     *,
     scenario_name: str | None,
@@ -79,14 +98,17 @@ def compute_submission_outcome(
 
     The validator-side outcome covers static config violations (handled at
     UserConfig.model_post_init by ``validate_scenario``). This helper folds
-    in runtime-only signals that are only knowable post-run -- the >1%
-    context-overflow rate per spec §7, and early cancellation (Ctrl+C): a
-    cancelled run produces partial metrics and is never a valid submission.
-
-    Rate semantics: strictly greater than
-    ``Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT`` (default 0.01 per
-    spec §7, override via ``AIPERF_AGENTX_CONTEXT_OVERFLOW_RATE_LIMIT``)
-    flips ``submission_valid`` to False; equal-to is accepted (boundary
+    in runtime-only signals that are only knowable post-run -- the
+    context-overflow rate, and early cancellation (Ctrl+C): a cancelled run
+    produces partial metrics and is never a valid submission.
+
+    Rate semantics: strictly greater than the ceiling from
+    ``_context_overflow_rate_limit`` flips ``submission_valid`` to False.
+    The ceiling is ``Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT``
+    (default 0.01 per spec §7, override via
+    ``AIPERF_AGENTX_CONTEXT_OVERFLOW_RATE_LIMIT``), or the scenario's
+    ``max_context_overflow_rate`` when that is stricter (0.0 for
+    agentperf-v2, so any overflow invalidates); equal-to is accepted (boundary
     behavior pinned by tests). When ``total_responses == 0`` the rate is
     treated as 0 (undefined / no successful responses), so the overflow
     rule does not flip submission validity in that case -- other failure
@@ -128,7 +150,7 @@ def compute_submission_outcome(
 
     if total_responses > 0:
         rate = context_overflow_count / total_responses
-        if rate > Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT:
+        if rate > _context_overflow_rate_limit(scenario_name):
             valid = False
             if CONTEXT_OVERFLOW_REASON not in reasons:
                 reasons.append(CONTEXT_OVERFLOW_REASON)

tests/unit/exporters/test_submission_valid_field.py

@@ -5,6 +5,7 @@
 import pytest
 from pytest import param
 
+from aiperf.common.environment import Environment
 from aiperf.exporters.aggregate.aggregate_base_exporter import (
     CONTEXT_OVERFLOW_REASON,
     RUN_CANCELLED_REASON,
@@ -115,3 +116,39 @@ def test_overflow_rate_boundary_without_double_count() -> None:
     )
     assert valid_inflated is True
     assert reasons_inflated == []
+
+
+@pytest.mark.parametrize(
+    "scenario_name, overflow, total, expected_valid",
+    [
+        param("agentperf-v2", 0, 10_000, True, id="agentperf_v2_zero_overflow_valid"),
+        param("agentperf-v2", 1, 10_000, False, id="agentperf_v2_single_overflow_invalid"),
+        param("inferencex-agentx-mvp", 1, 10_000, True, id="agentx_keeps_one_percent_rule"),
+        param("inferencex-agentx-mvp", 101, 10_000, False, id="agentx_over_one_percent_invalid"),
+    ],
+)  # fmt: skip
+def test_scenario_overflow_rate_ceiling(
+    scenario_name: str, overflow: int, total: int, expected_valid: bool
+) -> None:
+    valid, reasons = compute_submission_outcome(
+        scenario_name=scenario_name,
+        validator_submission_valid=True,
+        total_responses=total,
+        context_overflow_count=overflow,
+    )
+    assert valid is expected_valid
+    assert (CONTEXT_OVERFLOW_REASON in reasons) is (not expected_valid)
+
+
+def test_env_limit_cannot_loosen_scenario_ceiling(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(Environment.AGENTX, "CONTEXT_OVERFLOW_RATE_LIMIT", 0.5)
+    valid, reasons = compute_submission_outcome(
+        scenario_name="agentperf-v2",
+        validator_submission_valid=True,
+        total_responses=10_000,
+        context_overflow_count=1,
+    )
+    assert valid is False
+    assert CONTEXT_OVERFLOW_REASON in reasons