From 9fa21e2676e7b9552e51b1a7cb44f71196d87f61 Mon Sep 17 00:00:00 2001 From: Bofeng Luo Date: Thu, 12 Nov 2026 15:37:41 -0800 Subject: [PATCH 1/4] feat(scenario): replay recorded output lengths in agentperf-v2 Live replies are discarded and the next turn is built from the recording, so a longer or shorter live reply changes the output token count and the throughput without changing anything else. The scenario allows EOS and does not lock output-shaping --extra-inputs. - agentperf-v2 requires ignore_eos, so each reply runs to the turn's max_tokens. - New require_turn_max_tokens scenario field: the DAG loader rejects a dataset in which any turn lacks max_tokens. - New forbid_extra_input_keys scenario field: agentperf-v2 rejects --extra-inputs keys that would override output length, stop, reasoning or sampling settings. Requires a dataset release that sets each turn's max_tokens to its recorded output token count. Signed-off-by: Bofeng Luo --- docs/tutorials/agentperf-v2.md | 4 ++ src/aiperf/common/scenario/agentperf_v2.py | 19 +++++- src/aiperf/common/scenario/base.py | 16 +++++ src/aiperf/common/scenario/validator.py | 33 +++++++++++ src/aiperf/dataset/loader/dag_jsonl.py | 27 +++++++++ .../scenario/test_scenario_validator.py | 36 +++++++++++ .../loader/test_dag_jsonl_turn_max_tokens.py | 59 +++++++++++++++++++ 7 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 tests/unit/dataset/loader/test_dag_jsonl_turn_max_tokens.py diff --git a/docs/tutorials/agentperf-v2.md b/docs/tutorials/agentperf-v2.md index fc311b0320..6cc663107d 100644 --- a/docs/tutorials/agentperf-v2.md +++ b/docs/tutorials/agentperf-v2.md @@ -40,6 +40,10 @@ to 12 hours. At high concurrency, cap aiperf's session cache so finished sessions cannot use up the client's memory: set `AIPERF_WORKER_SESSION_CACHE_MAX_ENTRIES` to about 8 × users ÷ workers. The default was fine at 16 users. +Every request carries its turn's recorded output length as `max_tokens` and is sent with `ignore_eos`, so each reply is +exactly as long as the recorded one and reply length cannot move the results. The dataset must set `max_tokens` on every +turn; the loader rejects a turn without one. `--extra-inputs` cannot set output length, stop, reasoning or sampling options. + Use the recordings of the model your server is serving. Your server must report cached tokens in its usage figures for the cache results to appear: for example, `--enable-cache-report` on SGLang. diff --git a/src/aiperf/common/scenario/agentperf_v2.py b/src/aiperf/common/scenario/agentperf_v2.py index 3e79adfc52..6eb97e1d23 100644 --- a/src/aiperf/common/scenario/agentperf_v2.py +++ b/src/aiperf/common/scenario/agentperf_v2.py @@ -7,7 +7,24 @@ AGENTPERF_V2 = ScenarioSpec( name="agentperf-v2", timing_mode=TimingMode.AGENTIC_REPLAY, - require_ignore_eos=False, + # Every turn carries its recorded output length as max_tokens, and + # ignore_eos makes each reply exactly that long, so reply length cannot + # move the score. + require_ignore_eos=True, + require_turn_max_tokens=True, + forbid_extra_input_keys=( + "max_tokens", + "max_completion_tokens", + "min_tokens", + "min_new_tokens", + "stop", + "stop_token_ids", + "reasoning_effort", + "chat_template_kwargs", + "temperature", + "top_p", + "top_k", + ), require_streaming=True, forbid_ignore_trace_delays=True, forbid_input_truncation=True, diff --git a/src/aiperf/common/scenario/base.py b/src/aiperf/common/scenario/base.py index c7ccc92559..30b5103af4 100644 --- a/src/aiperf/common/scenario/base.py +++ b/src/aiperf/common/scenario/base.py @@ -149,6 +149,22 @@ class ScenarioSpec(AIPerfBaseModel): "each trace's original inter-turn timing." ), ) + require_turn_max_tokens: bool = Field( + default=False, + description=( + "Reject a DAG dataset in which any turn lacks max_tokens. Pair with " + "require_ignore_eos so every reply has exactly the length the " + "dataset sets for that turn (its recorded output length)." + ), + ) + forbid_extra_input_keys: tuple[str, ...] = Field( + default=(), + description=( + "Keys rejected in --extra-inputs. Extra inputs are merged into " + "every request body after the per-turn max_tokens, so these keys " + "would override the dataset's output length or decoding settings." + ), + ) loop: LoopRules | None = Field( default=None, description="Space lanes evenly on one loop of the dataset's sessions, and " diff --git a/src/aiperf/common/scenario/validator.py b/src/aiperf/common/scenario/validator.py index c82945efe8..3bcfdacca0 100644 --- a/src/aiperf/common/scenario/validator.py +++ b/src/aiperf/common/scenario/validator.py @@ -84,6 +84,7 @@ def apply_scenario(run: BenchmarkRun) -> ScenarioOutcome: _apply_timing_mode(run, spec, violations, applied) _apply_require_streaming(run, spec, violations, applied) _apply_require_ignore_eos(run, spec, violations, applied) + _apply_forbid_extra_input_keys(run, spec, violations, applied) _apply_forbid_ignore_trace_delays(run, spec, violations, applied) _apply_use_think_time_only(run, spec, violations, applied) _apply_forbid_input_truncation(run, spec, violations, applied) @@ -263,6 +264,38 @@ def _apply_require_streaming( applied.append("streaming") +def _apply_forbid_extra_input_keys( + run: BenchmarkRun, + spec: ScenarioSpec, + violations: list[ScenarioViolation], + applied: list[str], +) -> None: + """Reject ``--extra-inputs`` keys that would override per-turn settings. + + ``EndpointConfig.extra`` is merged into every request body after the + per-turn ``max_tokens``, so a key such as ``max_tokens`` or ``min_tokens`` + there would replace the dataset's output length for every request. + """ + if not spec.forbid_extra_input_keys: + return + extra = run.cfg.endpoint.extra + found = [key for key in spec.forbid_extra_input_keys if key in extra] + for key in found: + violations.append( + ScenarioViolation( + flag=f"extra_inputs.{key}", + current_value=extra[key], + required_value=None, + message=( + f"scenario {spec.name!r} takes {key!r} from the dataset and " + "forbids overriding it with --extra-inputs" + ), + ) + ) + if not found: + applied.append("forbid_extra_input_keys") + + def _apply_require_ignore_eos( run: BenchmarkRun, spec: ScenarioSpec, diff --git a/src/aiperf/dataset/loader/dag_jsonl.py b/src/aiperf/dataset/loader/dag_jsonl.py index f19ea3f0fc..a01d27ea8e 100644 --- a/src/aiperf/dataset/loader/dag_jsonl.py +++ b/src/aiperf/dataset/loader/dag_jsonl.py @@ -17,6 +17,7 @@ from aiperf.common.models import DatasetMetadata, TurnPrerequisite from aiperf.common.models.branch import ConversationBranchInfo from aiperf.common.models.dataset_models import Conversation, ThinkTimeSpec, Turn +from aiperf.common.scenario.registry import get_scenario from aiperf.common.validators.orchestrator_v1 import validate_for_orchestrator_v1 from aiperf.dataset.loader._dag_jsonl_helpers import ( DagLoadError, @@ -195,6 +196,7 @@ def load_dataset(self) -> dict[str, list[Conversation]]: self._parse_lines() self._desugar_forks() self._resolve_and_validate() + self._check_turn_max_tokens() self._roots = self._compute_roots() for sid, conv in self._conversations.items(): # Respect an author-declared context mode (e.g. @@ -393,6 +395,31 @@ def _check_pre_session_duplicates( seen.add(child) return list(children) + def _check_turn_max_tokens(self) -> None: + """Reject turns without ``max_tokens`` when the scenario requires them. + + Under ``require_ignore_eos`` a turn without ``max_tokens`` would decode + until the server's own limit, so the dataset must set every turn's + output length. + """ + scenario_name = self.run.cfg.scenario if self.run is not None else None + if scenario_name is None: + return + if not get_scenario(scenario_name).require_turn_max_tokens: + return + missing = [ + f"{sid}[{index}]" + for sid, conv in self._conversations.items() + for index, turn in enumerate(conv.turns) + if turn.max_tokens is None + ] + if missing: + raise DagLoadError( + f"{self._path}: scenario {scenario_name!r} requires max_tokens on " + f"every turn, but {len(missing)} turn(s) have none " + f"(first: {', '.join(missing[:5])})" + ) + def _build_turn(self, t: DagTurn) -> Turn: return Turn( raw_messages=list(t.messages), diff --git a/tests/unit/common/scenario/test_scenario_validator.py b/tests/unit/common/scenario/test_scenario_validator.py index 4a01d6c68a..b05434bb1b 100644 --- a/tests/unit/common/scenario/test_scenario_validator.py +++ b/tests/unit/common/scenario/test_scenario_validator.py @@ -652,3 +652,39 @@ def test_random_seed_auto_filled_through_resolver_chain() -> None: assert run.random_seed is None build_default_resolver_chain().resolve_all(run) assert run.random_seed is not None + + +_AGENTPERF_V2_DATASET = {"name": "main", "type": "public", "dataset": "agentperf_v2"} + + +def test_agentperf_v2_injects_ignore_eos() -> None: + run = _build_run( + scenario="agentperf-v2", dataset=dict(_AGENTPERF_V2_DATASET), streaming=True + ) + outcome = apply_scenario(run) + assert outcome.submission_valid is True + assert run.cfg.endpoint.extra["ignore_eos"] is True + + +@pytest.mark.parametrize( + "key, value", + [ + param("max_tokens", 4096, id="max_tokens"), + param("max_completion_tokens", 4096, id="max_completion_tokens"), + param("min_tokens", 16, id="min_tokens"), + param("stop", [""], id="stop"), + param("reasoning_effort", "low", id="reasoning_effort"), + param("chat_template_kwargs", {"enable_thinking": False}, id="chat_template_kwargs"), + param("temperature", 0.0, id="temperature"), + ], +) # fmt: skip +def test_agentperf_v2_rejects_output_shaping_extra_inputs(key: str, value: Any) -> None: + run = _build_run( + scenario="agentperf-v2", + dataset=dict(_AGENTPERF_V2_DATASET), + streaming=True, + extra={key: value}, + ) + with pytest.raises(ScenarioLockError) as exc: + apply_scenario(run) + assert f"extra_inputs.{key}" in str(exc.value) diff --git a/tests/unit/dataset/loader/test_dag_jsonl_turn_max_tokens.py b/tests/unit/dataset/loader/test_dag_jsonl_turn_max_tokens.py new file mode 100644 index 0000000000..967ef368f8 --- /dev/null +++ b/tests/unit/dataset/loader/test_dag_jsonl_turn_max_tokens.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Scenario-required per-turn max_tokens in the DAG JSONL loader.""" + +import json +from pathlib import Path + +import pytest +from pytest import param + +from aiperf.dataset.loader.dag_jsonl import DagJsonlLoader, DagLoadError +from tests.unit.dataset.loader.conftest import make_weka_run + + +def _write(tmp_path: Path, max_tokens: list[int | None]) -> Path: + turns = [] + for value in max_tokens: + turn: dict = {"messages": [{"role": "user", "content": "hi"}]} + if value is not None: + turn["max_tokens"] = value + turns.append(turn) + path = tmp_path / "dag.jsonl" + path.write_text(json.dumps({"session_id": "s1", "turns": turns}) + "\n") + return path + + +def _loader(path: Path, scenario: str | None) -> DagJsonlLoader: + run = make_weka_run() + run.cfg.scenario = scenario + return DagJsonlLoader(filename=str(path), run=run) + + +def test_agentperf_v2_accepts_turns_with_max_tokens(tmp_path: Path) -> None: + loader = _loader(_write(tmp_path, [120, 45]), "agentperf-v2") + convs = loader.convert_to_conversations(loader.load_dataset()) + assert [turn.max_tokens for turn in convs[0].turns] == [120, 45] + + +def test_agentperf_v2_rejects_turn_without_max_tokens(tmp_path: Path) -> None: + loader = _loader(_write(tmp_path, [120, None]), "agentperf-v2") + with pytest.raises( + DagLoadError, match=r"requires max_tokens on every turn.*s1\[1\]" + ): + loader.load_dataset() + + +@pytest.mark.parametrize( + "scenario", + [ + param(None, id="no_scenario"), + param("inferencex-agentx-mvp", id="other_scenario"), + ], +) # fmt: skip +def test_missing_max_tokens_allowed_without_requirement( + tmp_path: Path, scenario: str | None +) -> None: + loader = _loader(_write(tmp_path, [None, None]), scenario) + convs = loader.convert_to_conversations(loader.load_dataset()) + assert [turn.max_tokens for turn in convs[0].turns] == [None, None] From e989e8c4cfdd50b1c038dc2a5869a723ccae400d Mon Sep 17 00:00:00 2001 From: Bofeng Luo Date: Thu, 12 Nov 2026 15:40:46 -0800 Subject: [PATCH 2/4] docs: move the output-length note to its own paragraph Signed-off-by: Bofeng Luo --- docs/tutorials/agentperf-v2.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/agentperf-v2.md b/docs/tutorials/agentperf-v2.md index 6cc663107d..de1e10187e 100644 --- a/docs/tutorials/agentperf-v2.md +++ b/docs/tutorials/agentperf-v2.md @@ -40,13 +40,13 @@ to 12 hours. At high concurrency, cap aiperf's session cache so finished sessions cannot use up the client's memory: set `AIPERF_WORKER_SESSION_CACHE_MAX_ENTRIES` to about 8 × users ÷ workers. The default was fine at 16 users. +Use the recordings of the model your server is serving. Your server must report cached tokens in its usage figures for the +cache results to appear: for example, `--enable-cache-report` on SGLang. + Every request carries its turn's recorded output length as `max_tokens` and is sent with `ignore_eos`, so each reply is exactly as long as the recorded one and reply length cannot move the results. The dataset must set `max_tokens` on every turn; the loader rejects a turn without one. `--extra-inputs` cannot set output length, stop, reasoning or sampling options. -Use the recordings of the model your server is serving. Your server must report cached tokens in its usage figures for the -cache results to appear: for example, `--enable-cache-report` on SGLang. - ## What a run does The dataset's sessions sit end to end on one timeline, which is the loop. `N` users are spaced evenly along it, and each From c205e17d174a74002a87e54009030e1e513152dc Mon Sep 17 00:00:00 2001 From: Bofeng Luo Date: Thu, 12 Nov 2026 15:41:17 -0800 Subject: [PATCH 3/4] test: move agentperf-v2 output-length tests to their own module Signed-off-by: Bofeng Luo --- .../test_agentperf_v2_output_length.py | 70 +++++++++++++++++++ .../scenario/test_scenario_validator.py | 36 ---------- 2 files changed, 70 insertions(+), 36 deletions(-) create mode 100644 tests/unit/common/scenario/test_agentperf_v2_output_length.py diff --git a/tests/unit/common/scenario/test_agentperf_v2_output_length.py b/tests/unit/common/scenario/test_agentperf_v2_output_length.py new file mode 100644 index 0000000000..b9816966dc --- /dev/null +++ b/tests/unit/common/scenario/test_agentperf_v2_output_length.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""agentperf-v2 locks output length to the dataset's per-turn max_tokens.""" + +from pathlib import Path +from typing import Any + +import pytest +from pytest import param + +from aiperf.common.scenario import ScenarioLockError, apply_scenario +from aiperf.config.config import BenchmarkConfig +from aiperf.config.resolution.plan import BenchmarkRun + + +def _build_run(extra: dict[str, Any] | None = None) -> BenchmarkRun: + endpoint: dict[str, Any] = { + "urls": ["http://localhost:8000/v1/chat/completions"], + "type": "chat", + "streaming": True, + } + if extra is not None: + endpoint["extra"] = extra + cfg = BenchmarkConfig.model_validate( + { + "models": ["my-model"], + "endpoint": endpoint, + "datasets": [{"name": "main", "type": "public", "dataset": "agentperf_v2"}], + "phases": [ + { + "name": "profiling", + "type": "concurrency", + "concurrency": 8, + "duration": 1800, + } + ], + "scenario": "agentperf-v2", + } + ) + return BenchmarkRun( + benchmark_id="test-run", + cfg=cfg, + artifact_dir=Path("/tmp/aiperf-scenario-test"), + ) + + +def test_agentperf_v2_injects_ignore_eos() -> None: + run = _build_run() + outcome = apply_scenario(run) + assert outcome.submission_valid is True + assert run.cfg.endpoint.extra["ignore_eos"] is True + + +@pytest.mark.parametrize( + "key, value", + [ + param("max_tokens", 4096, id="max_tokens"), + param("max_completion_tokens", 4096, id="max_completion_tokens"), + param("min_tokens", 16, id="min_tokens"), + param("stop", [""], id="stop"), + param("reasoning_effort", "low", id="reasoning_effort"), + param("chat_template_kwargs", {"enable_thinking": False}, id="chat_template_kwargs"), + param("temperature", 0.0, id="temperature"), + ], +) # fmt: skip +def test_agentperf_v2_rejects_output_shaping_extra_inputs(key: str, value: Any) -> None: + run = _build_run(extra={key: value}) + with pytest.raises(ScenarioLockError) as exc: + apply_scenario(run) + assert f"extra_inputs.{key}" in str(exc.value) diff --git a/tests/unit/common/scenario/test_scenario_validator.py b/tests/unit/common/scenario/test_scenario_validator.py index b05434bb1b..4a01d6c68a 100644 --- a/tests/unit/common/scenario/test_scenario_validator.py +++ b/tests/unit/common/scenario/test_scenario_validator.py @@ -652,39 +652,3 @@ def test_random_seed_auto_filled_through_resolver_chain() -> None: assert run.random_seed is None build_default_resolver_chain().resolve_all(run) assert run.random_seed is not None - - -_AGENTPERF_V2_DATASET = {"name": "main", "type": "public", "dataset": "agentperf_v2"} - - -def test_agentperf_v2_injects_ignore_eos() -> None: - run = _build_run( - scenario="agentperf-v2", dataset=dict(_AGENTPERF_V2_DATASET), streaming=True - ) - outcome = apply_scenario(run) - assert outcome.submission_valid is True - assert run.cfg.endpoint.extra["ignore_eos"] is True - - -@pytest.mark.parametrize( - "key, value", - [ - param("max_tokens", 4096, id="max_tokens"), - param("max_completion_tokens", 4096, id="max_completion_tokens"), - param("min_tokens", 16, id="min_tokens"), - param("stop", [""], id="stop"), - param("reasoning_effort", "low", id="reasoning_effort"), - param("chat_template_kwargs", {"enable_thinking": False}, id="chat_template_kwargs"), - param("temperature", 0.0, id="temperature"), - ], -) # fmt: skip -def test_agentperf_v2_rejects_output_shaping_extra_inputs(key: str, value: Any) -> None: - run = _build_run( - scenario="agentperf-v2", - dataset=dict(_AGENTPERF_V2_DATASET), - streaming=True, - extra={key: value}, - ) - with pytest.raises(ScenarioLockError) as exc: - apply_scenario(run) - assert f"extra_inputs.{key}" in str(exc.value) From 0acfcfe1de3c818f3ccbd6075a0ad7535dfb1172 Mon Sep 17 00:00:00 2001 From: Bofeng Luo Date: Thu, 12 Nov 2026 15:48:24 -0800 Subject: [PATCH 4/4] fix(scenario): close output-length overrides in agentperf-v2 - Skip turns that send no request (orchestrator turns) in the per-turn max_tokens check, so orchestrator sessions load. - Reject request turns whose per-turn extra sets ignore_eos=false or a forbidden key, since per-turn extra is merged last. - New require_endpoint_types scenario field; agentperf-v2 accepts only the chat endpoint, so a payload template cannot replace max_tokens. - Extend the forbidden --extra-inputs keys (stop_regex, reasoning, structured outputs, payload_template, more sampling keys) and list them accurately in the tutorial. Signed-off-by: Bofeng Luo --- docs/tutorials/agentperf-v2.md | 4 +- src/aiperf/common/scenario/agentperf_v2.py | 17 ++++++++ src/aiperf/common/scenario/base.py | 14 ++++++- src/aiperf/common/scenario/validator.py | 27 ++++++++++++ src/aiperf/dataset/loader/dag_jsonl.py | 40 ++++++++++++------ .../test_agentperf_v2_output_length.py | 20 ++++++++- .../loader/test_dag_jsonl_turn_max_tokens.py | 42 ++++++++++++++++++- 7 files changed, 146 insertions(+), 18 deletions(-) diff --git a/docs/tutorials/agentperf-v2.md b/docs/tutorials/agentperf-v2.md index de1e10187e..c40c29de6c 100644 --- a/docs/tutorials/agentperf-v2.md +++ b/docs/tutorials/agentperf-v2.md @@ -45,7 +45,9 @@ cache results to appear: for example, `--enable-cache-report` on SGLang. Every request carries its turn's recorded output length as `max_tokens` and is sent with `ignore_eos`, so each reply is exactly as long as the recorded one and reply length cannot move the results. The dataset must set `max_tokens` on every -turn; the loader rejects a turn without one. `--extra-inputs` cannot set output length, stop, reasoning or sampling options. +turn; the loader rejects a turn without one. The scenario accepts only `--endpoint-type chat`, and rejects +`--extra-inputs` keys that set output length or stopping, reasoning, chat templates, constrained decoding or the main +sampling settings (the full list is `forbid_extra_input_keys` in the scenario). ## What a run does diff --git a/src/aiperf/common/scenario/agentperf_v2.py b/src/aiperf/common/scenario/agentperf_v2.py index 6eb97e1d23..3f6420c352 100644 --- a/src/aiperf/common/scenario/agentperf_v2.py +++ b/src/aiperf/common/scenario/agentperf_v2.py @@ -12,18 +12,35 @@ # move the score. require_ignore_eos=True, require_turn_max_tokens=True, + require_endpoint_types=("chat",), forbid_extra_input_keys=( + # Output length and stopping. "max_tokens", "max_completion_tokens", + "max_new_tokens", "min_tokens", "min_new_tokens", "stop", "stop_token_ids", + "stop_regex", + "include_stop_str_in_output", + # Reasoning, templates and constrained decoding. + "reasoning", "reasoning_effort", + "chat_template", "chat_template_kwargs", + "response_format", + "structured_outputs", + "payload_template", + "vllm_xargs", + # Sampling. "temperature", "top_p", "top_k", + "min_p", + "repetition_penalty", + "presence_penalty", + "frequency_penalty", ), require_streaming=True, forbid_ignore_trace_delays=True, diff --git a/src/aiperf/common/scenario/base.py b/src/aiperf/common/scenario/base.py index 30b5103af4..4972480d23 100644 --- a/src/aiperf/common/scenario/base.py +++ b/src/aiperf/common/scenario/base.py @@ -149,12 +149,22 @@ class ScenarioSpec(AIPerfBaseModel): "each trace's original inter-turn timing." ), ) + require_endpoint_types: tuple[str, ...] = Field( + default=(), + description=( + "Endpoint types the scenario accepts. Empty accepts any. Limiting " + "it keeps request bodies built by formatters that write each " + "turn's max_tokens, so a custom payload template cannot replace it." + ), + ) require_turn_max_tokens: bool = Field( default=False, description=( - "Reject a DAG dataset in which any turn lacks max_tokens. Pair with " + "Reject a DAG dataset in which any request turn lacks max_tokens, " + "or carries a forbidden extra key or ignore_eos=false. Pair with " "require_ignore_eos so every reply has exactly the length the " - "dataset sets for that turn (its recorded output length)." + "dataset sets for that turn (its recorded output length). This is " + "a load error, so --unsafe-override does not bypass it." ), ) forbid_extra_input_keys: tuple[str, ...] = Field( diff --git a/src/aiperf/common/scenario/validator.py b/src/aiperf/common/scenario/validator.py index 3bcfdacca0..7d3901b4f9 100644 --- a/src/aiperf/common/scenario/validator.py +++ b/src/aiperf/common/scenario/validator.py @@ -85,6 +85,7 @@ def apply_scenario(run: BenchmarkRun) -> ScenarioOutcome: _apply_require_streaming(run, spec, violations, applied) _apply_require_ignore_eos(run, spec, violations, applied) _apply_forbid_extra_input_keys(run, spec, violations, applied) + _apply_require_endpoint_types(run, spec, violations, applied) _apply_forbid_ignore_trace_delays(run, spec, violations, applied) _apply_use_think_time_only(run, spec, violations, applied) _apply_forbid_input_truncation(run, spec, violations, applied) @@ -296,6 +297,32 @@ def _apply_forbid_extra_input_keys( applied.append("forbid_extra_input_keys") +def _apply_require_endpoint_types( + run: BenchmarkRun, + spec: ScenarioSpec, + violations: list[ScenarioViolation], + applied: list[str], +) -> None: + """Reject endpoint types outside ``spec.require_endpoint_types``.""" + if not spec.require_endpoint_types: + return + endpoint_type = str(run.cfg.endpoint.type) + if endpoint_type in spec.require_endpoint_types: + applied.append("endpoint_type") + return + violations.append( + ScenarioViolation( + flag="--endpoint-type", + current_value=endpoint_type, + required_value=", ".join(spec.require_endpoint_types), + message=( + f"scenario {spec.name!r} accepts only endpoint types " + f"{', '.join(spec.require_endpoint_types)}" + ), + ) + ) + + def _apply_require_ignore_eos( run: BenchmarkRun, spec: ScenarioSpec, diff --git a/src/aiperf/dataset/loader/dag_jsonl.py b/src/aiperf/dataset/loader/dag_jsonl.py index a01d27ea8e..b270d71320 100644 --- a/src/aiperf/dataset/loader/dag_jsonl.py +++ b/src/aiperf/dataset/loader/dag_jsonl.py @@ -396,29 +396,45 @@ def _check_pre_session_duplicates( return list(children) def _check_turn_max_tokens(self) -> None: - """Reject turns without ``max_tokens`` when the scenario requires them. + """Reject request turns that would not get the dataset's output length. - Under ``require_ignore_eos`` a turn without ``max_tokens`` would decode - until the server's own limit, so the dataset must set every turn's - output length. + Under ``require_ignore_eos`` a request turn without ``max_tokens`` + would decode until the server's own limit, and a per-turn ``extra`` + key from the scenario's forbid list, or ``ignore_eos=false``, would + override the length. Turns that send no request are skipped. """ scenario_name = self.run.cfg.scenario if self.run is not None else None if scenario_name is None: return - if not get_scenario(scenario_name).require_turn_max_tokens: + spec = get_scenario(scenario_name) + if not spec.require_turn_max_tokens: return - missing = [ - f"{sid}[{index}]" - for sid, conv in self._conversations.items() - for index, turn in enumerate(conv.turns) - if turn.max_tokens is None - ] + missing: list[str] = [] + overridden: list[str] = [] + for sid, conv in self._conversations.items(): + for index, turn in enumerate(conv.turns): + if turn.no_request: + continue + if turn.max_tokens is None: + missing.append(f"{sid}[{index}]") + extra = turn.extra_body or {} + if extra.get("ignore_eos") is False or any( + key in extra for key in spec.forbid_extra_input_keys + ): + overridden.append(f"{sid}[{index}]") if missing: raise DagLoadError( f"{self._path}: scenario {scenario_name!r} requires max_tokens on " - f"every turn, but {len(missing)} turn(s) have none " + f"every request turn, but {len(missing)} turn(s) have none " f"(first: {', '.join(missing[:5])})" ) + if overridden: + raise DagLoadError( + f"{self._path}: scenario {scenario_name!r} takes output length " + "from max_tokens, but the extra of " + f"{len(overridden)} turn(s) overrides it " + f"(first: {', '.join(overridden[:5])})" + ) def _build_turn(self, t: DagTurn) -> Turn: return Turn( diff --git a/tests/unit/common/scenario/test_agentperf_v2_output_length.py b/tests/unit/common/scenario/test_agentperf_v2_output_length.py index b9816966dc..764e33e772 100644 --- a/tests/unit/common/scenario/test_agentperf_v2_output_length.py +++ b/tests/unit/common/scenario/test_agentperf_v2_output_length.py @@ -13,14 +13,20 @@ from aiperf.config.resolution.plan import BenchmarkRun -def _build_run(extra: dict[str, Any] | None = None) -> BenchmarkRun: +def _build_run( + extra: dict[str, Any] | None = None, endpoint_type: str = "chat" +) -> BenchmarkRun: endpoint: dict[str, Any] = { "urls": ["http://localhost:8000/v1/chat/completions"], - "type": "chat", + "type": endpoint_type, "streaming": True, } if extra is not None: endpoint["extra"] = extra + if endpoint_type == "template": + endpoint["template"] = { + "body": '{"messages": {{ messages | tojson }}, "max_tokens": 16}' + } cfg = BenchmarkConfig.model_validate( { "models": ["my-model"], @@ -61,6 +67,8 @@ def test_agentperf_v2_injects_ignore_eos() -> None: param("reasoning_effort", "low", id="reasoning_effort"), param("chat_template_kwargs", {"enable_thinking": False}, id="chat_template_kwargs"), param("temperature", 0.0, id="temperature"), + param("payload_template", "{}", id="payload_template"), + param("stop_regex", "x", id="stop_regex"), ], ) # fmt: skip def test_agentperf_v2_rejects_output_shaping_extra_inputs(key: str, value: Any) -> None: @@ -68,3 +76,11 @@ def test_agentperf_v2_rejects_output_shaping_extra_inputs(key: str, value: Any) with pytest.raises(ScenarioLockError) as exc: apply_scenario(run) assert f"extra_inputs.{key}" in str(exc.value) + + +@pytest.mark.parametrize("endpoint_type", ["template", "completions"]) +def test_agentperf_v2_accepts_only_chat_endpoint(endpoint_type: str) -> None: + run = _build_run(endpoint_type=endpoint_type) + with pytest.raises(ScenarioLockError) as exc: + apply_scenario(run) + assert "--endpoint-type" in str(exc.value) diff --git a/tests/unit/dataset/loader/test_dag_jsonl_turn_max_tokens.py b/tests/unit/dataset/loader/test_dag_jsonl_turn_max_tokens.py index 967ef368f8..f85b2d3b64 100644 --- a/tests/unit/dataset/loader/test_dag_jsonl_turn_max_tokens.py +++ b/tests/unit/dataset/loader/test_dag_jsonl_turn_max_tokens.py @@ -39,7 +39,7 @@ def test_agentperf_v2_accepts_turns_with_max_tokens(tmp_path: Path) -> None: def test_agentperf_v2_rejects_turn_without_max_tokens(tmp_path: Path) -> None: loader = _loader(_write(tmp_path, [120, None]), "agentperf-v2") with pytest.raises( - DagLoadError, match=r"requires max_tokens on every turn.*s1\[1\]" + DagLoadError, match=r"requires max_tokens on every request turn.*s1\[1\]" ): loader.load_dataset() @@ -57,3 +57,43 @@ def test_missing_max_tokens_allowed_without_requirement( loader = _loader(_write(tmp_path, [None, None]), scenario) convs = loader.convert_to_conversations(loader.load_dataset()) assert [turn.max_tokens for turn in convs[0].turns] == [None, None] + + +def test_agentperf_v2_skips_turns_without_requests(tmp_path: Path) -> None: + rows = [ + {"session_id": "orch", "orchestrator": True, "spawns": ["child"]}, + { + "session_id": "child", + "turns": [ + {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 8} + ], + }, + ] + path = tmp_path / "dag.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + loader = _loader(path, "agentperf-v2") + convs = loader.convert_to_conversations(loader.load_dataset()) + assert {conv.session_id for conv in convs} == {"orch", "child"} + + +@pytest.mark.parametrize( + "extra", + [ + param({"ignore_eos": False}, id="ignore_eos_false"), + param({"min_tokens": 4}, id="min_tokens"), + param({"stop": ["\\n"]}, id="stop"), + ], +) # fmt: skip +def test_agentperf_v2_rejects_turn_extra_overriding_length( + tmp_path: Path, extra: dict +) -> None: + turn = { + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 8, + "extra": extra, + } + path = tmp_path / "dag.jsonl" + path.write_text(json.dumps({"session_id": "s1", "turns": [turn]}) + "\n") + loader = _loader(path, "agentperf-v2") + with pytest.raises(DagLoadError, match=r"overrides it.*s1\[0\]"): + loader.load_dataset()