From d915cffbae10407b9a3180a0fe24f2fbb7ec8c2c Mon Sep 17 00:00:00 2001 From: Bofeng Luo Date: Thu, 12 Nov 2026 15:24:49 -0800 Subject: [PATCH 1/2] fix(scenario): make any context overflow invalidate agentperf-v2 runs Context-overflow responses in agentic replay are skipped: the session ends and the skipped turns never reach the metrics, while the validity gate only checks overflow events against a 1% rate. A server started with a context window shorter than the dataset needs could skip the longest turns and still report submission_valid=true. Every request in the agentperf-v2 recordings fits a 1M-token window, so a correctly configured server never overflows. Add a per-scenario max_context_overflow_rate ceiling and set it to 0.0 for agentperf-v2. The AIPERF_AGENTX_CONTEXT_OVERFLOW_RATE_LIMIT env var can only tighten a scenario's ceiling, never loosen it. Other scenarios keep the 1% rule. Signed-off-by: Bofeng Luo --- src/aiperf/common/scenario/agentperf_v2.py | 3 ++ src/aiperf/common/scenario/base.py | 15 ++++++++ .../aggregate/aggregate_base_exporter.py | 21 ++++++++++- .../exporters/test_submission_valid_field.py | 35 +++++++++++++++++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/aiperf/common/scenario/agentperf_v2.py b/src/aiperf/common/scenario/agentperf_v2.py index 3e79adfc52..bad7c51e2c 100644 --- a/src/aiperf/common/scenario/agentperf_v2.py +++ b/src/aiperf/common/scenario/agentperf_v2.py @@ -17,6 +17,9 @@ default_trajectory_start_min_ratio=0.0, default_trajectory_start_max_ratio=1.0, require_cache_bust=CacheBustTarget.FIRST_TURN_PREFIX, + # Every recorded request fits a 1M-token window, so a correctly + # configured server never overflows; any overflow means skipped turns. + max_context_overflow_rate=0.0, loop=LoopRules( burn_in_seconds=900, ramp_seconds=300, diff --git a/src/aiperf/common/scenario/base.py b/src/aiperf/common/scenario/base.py index c7ccc92559..f12256970a 100644 --- a/src/aiperf/common/scenario/base.py +++ b/src/aiperf/common/scenario/base.py @@ -154,6 +154,21 @@ class ScenarioSpec(AIPerfBaseModel): description="Space lanes evenly on one loop of the dataset's sessions, and " "measure one slice per lane, instead of sampling a random start per lane.", ) + max_context_overflow_rate: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Scenario ceiling on the context-overflow rate (overflow responses / " + "total responses). A rate strictly above it flips submission_valid " + "to False. None falls back to " + "Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT. Set 0.0 when every " + "request in the dataset fits the required context window, so any " + "overflow (e.g. a server started with a shorter max context) " + "invalidates the run. The env override can only tighten, never " + "loosen, a scenario-set ceiling." + ), + ) require_cache_bust: CacheBustTarget | None = Field( default=None, description=( diff --git a/src/aiperf/exporters/aggregate/aggregate_base_exporter.py b/src/aiperf/exporters/aggregate/aggregate_base_exporter.py index 3d44934901..bb6d324fd8 100644 --- a/src/aiperf/exporters/aggregate/aggregate_base_exporter.py +++ b/src/aiperf/exporters/aggregate/aggregate_base_exporter.py @@ -11,6 +11,8 @@ from aiperf.common.environment import Environment from aiperf.common.mixins import AIPerfLoggerMixin +from aiperf.common.scenario.base import UnknownScenarioError +from aiperf.common.scenario.registry import get_scenario from aiperf.orchestrator.aggregation.base import AggregateResult @@ -31,6 +33,23 @@ def __getattr__(name: str) -> float: RUN_CANCELLED_REASON = "run_cancelled" +def _context_overflow_rate_limit(scenario_name: str) -> float: + """Resolve the overflow-rate ceiling for ``scenario_name``. + + The env limit (``AIPERF_AGENTX_CONTEXT_OVERFLOW_RATE_LIMIT``) is the + default. A scenario that declares ``max_context_overflow_rate`` gets the + stricter of the two, so the env var cannot loosen a scenario lock. + """ + limit = Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT + try: + spec_limit = get_scenario(scenario_name).max_context_overflow_rate + except UnknownScenarioError: + # Unknown scenario names are rejected by the validator before a run; + # fall back to the env limit here rather than failing the export. + spec_limit = None + return limit if spec_limit is None else min(limit, spec_limit) + + def _build_run_metadata_dict( *, scenario_name: str | None, @@ -128,7 +147,7 @@ def compute_submission_outcome( if total_responses > 0: rate = context_overflow_count / total_responses - if rate > Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT: + if rate > _context_overflow_rate_limit(scenario_name): valid = False if CONTEXT_OVERFLOW_REASON not in reasons: reasons.append(CONTEXT_OVERFLOW_REASON) diff --git a/tests/unit/exporters/test_submission_valid_field.py b/tests/unit/exporters/test_submission_valid_field.py index 37c8ac02b0..6d0610f8d0 100644 --- a/tests/unit/exporters/test_submission_valid_field.py +++ b/tests/unit/exporters/test_submission_valid_field.py @@ -5,6 +5,7 @@ import pytest from pytest import param +from aiperf.common.environment import Environment from aiperf.exporters.aggregate.aggregate_base_exporter import ( CONTEXT_OVERFLOW_REASON, RUN_CANCELLED_REASON, @@ -115,3 +116,37 @@ def test_overflow_rate_boundary_without_double_count() -> None: ) assert valid_inflated is True assert reasons_inflated == [] + + +@pytest.mark.parametrize( + "scenario_name, overflow, total, expected_valid", + [ + param("agentperf-v2", 0, 10_000, True, id="agentperf_v2_zero_overflow_valid"), + param("agentperf-v2", 1, 10_000, False, id="agentperf_v2_single_overflow_invalid"), + param("inferencex-agentx-mvp", 1, 10_000, True, id="agentx_keeps_one_percent_rule"), + param("inferencex-agentx-mvp", 101, 10_000, False, id="agentx_over_one_percent_invalid"), + ], +) # fmt: skip +def test_scenario_overflow_rate_ceiling( + scenario_name: str, overflow: int, total: int, expected_valid: bool +) -> None: + valid, reasons = compute_submission_outcome( + scenario_name=scenario_name, + validator_submission_valid=True, + total_responses=total, + context_overflow_count=overflow, + ) + assert valid is expected_valid + assert (CONTEXT_OVERFLOW_REASON in reasons) is (not expected_valid) + + +def test_env_limit_cannot_loosen_scenario_ceiling(monkeypatch) -> None: + monkeypatch.setattr(Environment.AGENTX, "CONTEXT_OVERFLOW_RATE_LIMIT", 0.5) + valid, reasons = compute_submission_outcome( + scenario_name="agentperf-v2", + validator_submission_valid=True, + total_responses=10_000, + context_overflow_count=1, + ) + assert valid is False + assert CONTEXT_OVERFLOW_REASON in reasons From 6dd4f37cdba1740c3561ea7c7a08ffd274b8e5c4 Mon Sep 17 00:00:00 2001 From: Bofeng Luo Date: Thu, 12 Nov 2026 15:40:08 -0800 Subject: [PATCH 2/2] docs: describe the scenario overflow ceiling and the 1M context requirement Signed-off-by: Bofeng Luo --- docs/tutorials/agentperf-v2.md | 3 +++ .../aggregate/aggregate_base_exporter.py | 19 +++++++++++-------- .../exporters/test_submission_valid_field.py | 4 +++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/tutorials/agentperf-v2.md b/docs/tutorials/agentperf-v2.md index fc311b0320..1db2bc7c51 100644 --- a/docs/tutorials/agentperf-v2.md +++ b/docs/tutorials/agentperf-v2.md @@ -40,6 +40,9 @@ to 12 hours. At high concurrency, cap aiperf's session cache so finished sessions cannot use up the client's memory: set `AIPERF_WORKER_SESSION_CACHE_MAX_ENTRIES` to about 8 × users ÷ workers. The default was fine at 16 users. +Serve the model with a context window of at least 1M tokens. Every recorded request fits in 1M, so any context overflow +marks the run `submission_valid=false`. + Use the recordings of the model your server is serving. Your server must report cached tokens in its usage figures for the cache results to appear: for example, `--enable-cache-report` on SGLang. diff --git a/src/aiperf/exporters/aggregate/aggregate_base_exporter.py b/src/aiperf/exporters/aggregate/aggregate_base_exporter.py index bb6d324fd8..b8e768895d 100644 --- a/src/aiperf/exporters/aggregate/aggregate_base_exporter.py +++ b/src/aiperf/exporters/aggregate/aggregate_base_exporter.py @@ -98,14 +98,17 @@ def compute_submission_outcome( The validator-side outcome covers static config violations (handled at UserConfig.model_post_init by ``validate_scenario``). This helper folds - in runtime-only signals that are only knowable post-run -- the >1% - context-overflow rate per spec §7, and early cancellation (Ctrl+C): a - cancelled run produces partial metrics and is never a valid submission. - - Rate semantics: strictly greater than - ``Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT`` (default 0.01 per - spec §7, override via ``AIPERF_AGENTX_CONTEXT_OVERFLOW_RATE_LIMIT``) - flips ``submission_valid`` to False; equal-to is accepted (boundary + in runtime-only signals that are only knowable post-run -- the + context-overflow rate, and early cancellation (Ctrl+C): a cancelled run + produces partial metrics and is never a valid submission. + + Rate semantics: strictly greater than the ceiling from + ``_context_overflow_rate_limit`` flips ``submission_valid`` to False. + The ceiling is ``Environment.AGENTX.CONTEXT_OVERFLOW_RATE_LIMIT`` + (default 0.01 per spec §7, override via + ``AIPERF_AGENTX_CONTEXT_OVERFLOW_RATE_LIMIT``), or the scenario's + ``max_context_overflow_rate`` when that is stricter (0.0 for + agentperf-v2, so any overflow invalidates); equal-to is accepted (boundary behavior pinned by tests). When ``total_responses == 0`` the rate is treated as 0 (undefined / no successful responses), so the overflow rule does not flip submission validity in that case -- other failure diff --git a/tests/unit/exporters/test_submission_valid_field.py b/tests/unit/exporters/test_submission_valid_field.py index 6d0610f8d0..bdfabe0a7b 100644 --- a/tests/unit/exporters/test_submission_valid_field.py +++ b/tests/unit/exporters/test_submission_valid_field.py @@ -140,7 +140,9 @@ def test_scenario_overflow_rate_ceiling( assert (CONTEXT_OVERFLOW_REASON in reasons) is (not expected_valid) -def test_env_limit_cannot_loosen_scenario_ceiling(monkeypatch) -> None: +def test_env_limit_cannot_loosen_scenario_ceiling( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setattr(Environment.AGENTX, "CONTEXT_OVERFLOW_RATE_LIMIT", 0.5) valid, reasons = compute_submission_outcome( scenario_name="agentperf-v2",