From 72af614c9469b8ae68c1b55dbde16488b68aceae Mon Sep 17 00:00:00 2001 From: luochen Date: Mon, 19 Oct 2026 07:35:38 +0000 Subject: [PATCH 1/4] Abort grid variants whose AITER_CONFIG file path is missing --- .../test_grid_runner_helpers_coverage_unit.py | 51 +++++++++++++++++++ .../actions/executors/_grid_runner.py | 35 ++++++++++++- 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py index b3b706b33f..44f07d1cf6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py @@ -60,6 +60,57 @@ def test_compatibility_filter_no_model_path_assumes_compatible() -> None: assert [v.name for v in kept] == ["moe"] and dropped == [] +# -- ungrounded_env_file_reason (env-value file grounding) ---------------- +def test_ungrounded_env_file_reason_drops_missing_config_path(tmp_path) -> None: + # A proposal may point a config env at an artifact no stage authored; the server would read it + # unconditionally and die at config load, so the variant must be dropped with a reason. + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(tmp_path / "nope.csv")}) + reason = gr.ungrounded_env_file_reason(v) + assert reason and "AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE" in reason + assert str(tmp_path / "nope.csv") in reason + + +def test_ungrounded_env_file_reason_passes_authored_file(tmp_path) -> None: + csv = tmp_path / "tuned.csv" + csv.write_text("gfx,M,N,K,splitK\ngfx950,64,5120,5120,2\n", encoding="utf-8") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(csv)}) + assert gr.ungrounded_env_file_reason(v) is None + + +def test_ungrounded_env_file_reason_accepts_colon_joined_existing_files(tmp_path) -> None: + # A set AITER_CONFIG_* is aiter's SET form: a `:`-joined list, each element loaded (csvs_aiter_will_load). + # Both files on disk is a valid variant and must not be dropped. + a = tmp_path / "t.csv" + a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") + b = tmp_path / "t2.csv" + b.write_text("gfx,M,N,K\ngfx950,4,5,6\n", encoding="utf-8") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{b}"}) + assert gr.ungrounded_env_file_reason(v) is None + + +def test_ungrounded_env_file_reason_drops_colon_joined_missing_element(tmp_path) -> None: + a = tmp_path / "t.csv" + a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") + missing = str(tmp_path / "never_authored.csv") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{missing}"}) + reason = gr.ungrounded_env_file_reason(v) + assert reason and missing in reason and str(a) not in reason.split("element(s)")[1].split(" do not exist")[0] + + + +def test_ungrounded_env_file_reason_noop_without_config_envs(tmp_path) -> None: + # Non-file envs (even under aiter-ish prefixes like the JSON-payload AITER_CONFIG) must stay untouched. + v = _variant( + "plain", + envs={ + "AITER_CONFIG": '{"path": "/tmp/never_scanned.json"}', + "SGLANG_USE_AITER": "1", + "AITER_CONFIG_GEMM_A8W8": "", + }, + ) + assert gr.ungrounded_env_file_reason(v) is None + + # -- unsupported_capability_reason (env-flag build probe) ----------------- def _clear_cap_cache() -> None: gr._CAP_PROBE_CACHE.clear() diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index dd132b124a..94ecac3895 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -287,6 +287,39 @@ def unsupported_capability_reason(variant: "GridVariant") -> str | None: return _probe_vllm_aiter_shared_expert_unsupported() +# Env vars whose value names a config file the serving stack reads unconditionally: aiter's tuned-GEMM loader skips +# its merge default and pd.read_csv's exactly the AITER_CONFIG_* path when one is set, so a missing file kills the +# server at config load instead of falling back to the default table. Values that are not file paths (e.g. geak's +# JSON-payload AITER_CONFIG) do not carry the trailing underscore and stay untouched. +_FILE_REFERENCING_ENV_PREFIXES: tuple[str, ...] = ("AITER_CONFIG_",) + + +def ungrounded_env_file_reason(variant: "GridVariant") -> str | None: + """Return a drop reason if an extra_envs value names a config file that does not exist, else ``None``. + + Grounds file-like env values the same way ``vet_patches`` grounds patch targets: a proposal may point a config + env at an artifact only a later (possibly rejected) authoring step would create, and the benchmark would boot a + server that dies at config load. The check is existence-only; anything already on disk (a promoted, durable + artifact) passes. A set value is a ``:``-joined list of paths (see ``csvs_aiter_will_load``), so every + non-empty element must exist, not only the first. + """ + for raw_key, raw_value in (getattr(variant, "extra_envs", None) or {}).items(): + key = str(raw_key or "").strip().upper() + if not key.startswith(_FILE_REFERENCING_ENV_PREFIXES): + continue + value = str(raw_value or "").strip() + if not value: + continue + missing = [p for p in (part.strip() for part in value.split(":")) if p and not Path(p).is_file()] + if missing: + return ( + f"{key} points at {value}, whose element(s) {':'.join(missing)} do not exist; the server would read it " + "unconditionally and die at config load. Point it at a file a stage authored " + "(e.g. a promoted tuned-GEMM CSV) or leave it unset to use the default config table." + ) + return None + + # Sanitization for LLM-supplied overrides (benchmark_script / result_dir): reject path separators / shell # metacharacters, raising ``ValueError`` instead of running an unsafe subprocess. _SCRIPT_NAME_RE = re.compile(r"^[A-Za-z0-9_.\-]+\.sh$") @@ -1074,7 +1107,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant server_log = slot / "server.log" # Capability fast-fail: drop a variant whose env flag the build cannot honour before booting a doomed server, # still recording the failure so the LLM learns not to re-pick it. - cap_reason = unsupported_capability_reason(variant) + cap_reason = unsupported_capability_reason(variant) or ungrounded_env_file_reason(variant) if cap_reason: log.warning( "grid_runner: variant %d/%d name=%s aborted: capability_unsupported: %s", From eb52f689c5979364d58cf83e53d89a585a3642f8 Mon Sep 17 00:00:00 2001 From: luochen Date: Mon, 19 Oct 2026 15:41:26 +0800 Subject: [PATCH 2/4] Treat an AITER_CONFIG path Path.is_file() cannot stat as missing; format the tests --- .../test_grid_runner_helpers_coverage_unit.py | 1064 ++-- .../actions/executors/_grid_runner.py | 4569 +++++++++-------- 2 files changed, 2826 insertions(+), 2807 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py index 44f07d1cf6..017bb4a456 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py @@ -1,527 +1,537 @@ -# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Supplemental coverage for _grid_runner pure helpers: compatibility filter model-class drop, runtime override env branches, report parsing, and per-variant yaml env injection.""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path - -import pytest -import yaml - -from hyperloom.orchestrator.actions.executors import _grid_runner as gr -from hyperloom.orchestrator.actions.executors import _benchmark_interpreter as bi - -# Patch compatibility-filter helpers in the ``_grid_variant_filter`` sibling, where apply_compatibility_filter -# resolves them (not via the re-export). -from hyperloom.orchestrator.actions.executors import _grid_variant_filter as vf - - -def _variant(name: str, args: str = "", envs: dict | None = None) -> gr.GridVariant: - return gr.GridVariant(name=name, extra_server_args=args, extra_envs=envs or {}) - - -def test_validate_magpie_python_override_requires_python(tmp_path): - py = tmp_path / "python3" - py.write_text("", encoding="utf-8") - bash = tmp_path / "bash" - bash.write_text("", encoding="utf-8") - assert gr._validate_magpie_python_override(str(py)) == str(py) - with pytest.raises(ValueError, match="Python interpreter"): - gr._validate_magpie_python_override(str(bash)) - assert gr._validate_magpie_python_override(sys.executable).endswith(Path(sys.executable).name) - - -# -- apply_compatibility_filter ------------------------------------------- -def test_compatibility_filter_drops_on_model_class(monkeypatch) -> None: - monkeypatch.setattr(vf, "_detect_model_class", lambda mp: (False, False)) - grid = [_variant("mla", "--enable-flashinfer-mla"), _variant("plain", "")] - kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="meta-llama-3-8b") - assert [v.name for v in kept] == ["plain"] - assert len(dropped) == 1 - assert dropped[0]["source"] == "compatibility_filter" - assert "MLA" in dropped[0]["reason"] - - -def test_compatibility_filter_keeps_a_flag_the_model_class_supports(monkeypatch) -> None: - # Whether the installed server accepts the flag is left to its own argparse at launch. - monkeypatch.setattr(vf, "_detect_model_class", lambda mp: (True, True)) - grid = [_variant("moe", "--enable-ep-moe")] - kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="deepseek-moe") - assert [v.name for v in kept] == ["moe"] and dropped == [] - - -def test_compatibility_filter_no_model_path_assumes_compatible() -> None: - grid = [_variant("moe", "--enable-ep-moe")] - kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="") - assert [v.name for v in kept] == ["moe"] and dropped == [] - - -# -- ungrounded_env_file_reason (env-value file grounding) ---------------- -def test_ungrounded_env_file_reason_drops_missing_config_path(tmp_path) -> None: - # A proposal may point a config env at an artifact no stage authored; the server would read it - # unconditionally and die at config load, so the variant must be dropped with a reason. - v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(tmp_path / "nope.csv")}) - reason = gr.ungrounded_env_file_reason(v) - assert reason and "AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE" in reason - assert str(tmp_path / "nope.csv") in reason - - -def test_ungrounded_env_file_reason_passes_authored_file(tmp_path) -> None: - csv = tmp_path / "tuned.csv" - csv.write_text("gfx,M,N,K,splitK\ngfx950,64,5120,5120,2\n", encoding="utf-8") - v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(csv)}) - assert gr.ungrounded_env_file_reason(v) is None - - -def test_ungrounded_env_file_reason_accepts_colon_joined_existing_files(tmp_path) -> None: - # A set AITER_CONFIG_* is aiter's SET form: a `:`-joined list, each element loaded (csvs_aiter_will_load). - # Both files on disk is a valid variant and must not be dropped. - a = tmp_path / "t.csv" - a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") - b = tmp_path / "t2.csv" - b.write_text("gfx,M,N,K\ngfx950,4,5,6\n", encoding="utf-8") - v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{b}"}) - assert gr.ungrounded_env_file_reason(v) is None - - -def test_ungrounded_env_file_reason_drops_colon_joined_missing_element(tmp_path) -> None: - a = tmp_path / "t.csv" - a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") - missing = str(tmp_path / "never_authored.csv") - v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{missing}"}) - reason = gr.ungrounded_env_file_reason(v) - assert reason and missing in reason and str(a) not in reason.split("element(s)")[1].split(" do not exist")[0] - - - -def test_ungrounded_env_file_reason_noop_without_config_envs(tmp_path) -> None: - # Non-file envs (even under aiter-ish prefixes like the JSON-payload AITER_CONFIG) must stay untouched. - v = _variant( - "plain", - envs={ - "AITER_CONFIG": '{"path": "/tmp/never_scanned.json"}', - "SGLANG_USE_AITER": "1", - "AITER_CONFIG_GEMM_A8W8": "", - }, - ) - assert gr.ungrounded_env_file_reason(v) is None - - -# -- unsupported_capability_reason (env-flag build probe) ----------------- -def _clear_cap_cache() -> None: - gr._CAP_PROBE_CACHE.clear() - - -def test_capability_reason_noop_when_flag_absent(monkeypatch) -> None: - monkeypatch.setenv("FRAMEWORK", "vllm") - _clear_cap_cache() - # Probe must NOT even run when the flag isn't set on the variant. - monkeypatch.setattr( - gr, - "_probe_vllm_aiter_shared_expert_unsupported", - lambda: (_ for _ in ()).throw(AssertionError("probe should not run")), - ) - assert gr.unsupported_capability_reason(_variant("plain")) is None - - -def test_capability_reason_noop_for_non_vllm_framework(monkeypatch) -> None: - monkeypatch.setenv("FRAMEWORK", "sglang") - _clear_cap_cache() - v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1"}) - assert gr.unsupported_capability_reason(v) is None - - -def test_capability_reason_drops_when_module_missing(monkeypatch) -> None: - monkeypatch.setenv("FRAMEWORK", "vllm") - _clear_cap_cache() - monkeypatch.setattr( - gr, - "_probe_vllm_aiter_shared_expert_unsupported", - lambda: "missing module(s): vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe", - ) - v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1"}) - reason = gr.unsupported_capability_reason(v) - assert reason is not None and "rocm_aiter_fused_moe" in reason - - -def test_capability_reason_falsey_flag_not_probed(monkeypatch) -> None: - monkeypatch.setenv("FRAMEWORK", "vllm") - _clear_cap_cache() - monkeypatch.setattr( - gr, - "_probe_vllm_aiter_shared_expert_unsupported", - lambda: (_ for _ in ()).throw(AssertionError("probe should not run")), - ) - v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0"}) - assert gr.unsupported_capability_reason(v) is None - - -def test_probe_caches_ok_and_unsupported(monkeypatch) -> None: - _clear_cap_cache() - calls = {"n": 0} - - class _Proc: - def __init__(self, out: str) -> None: - self.stdout = out - self.stderr = "" - - def fake_run(*_a, **_k): - calls["n"] += 1 - return _Proc(json.dumps({"status": "ok"})) - - monkeypatch.setattr(gr.subprocess, "run", fake_run) - assert gr._probe_vllm_aiter_shared_expert_unsupported() is None - # Second call must hit the cache (no second subprocess). - assert gr._probe_vllm_aiter_shared_expert_unsupported() is None - assert calls["n"] == 1 - - -def test_probe_unknown_not_cached(monkeypatch) -> None: - _clear_cap_cache() - - class _Proc: - stdout = json.dumps({"status": "unknown"}) - stderr = "" - - monkeypatch.setattr(gr.subprocess, "run", lambda *_a, **_k: _Proc()) - assert gr._probe_vllm_aiter_shared_expert_unsupported() is None - assert "vllm" not in gr._CAP_PROBE_CACHE - - -def test_probe_swallows_subprocess_error(monkeypatch) -> None: - _clear_cap_cache() - - def boom(*_a, **_k): - raise OSError("no python3") - - monkeypatch.setattr(gr.subprocess, "run", boom) - assert gr._probe_vllm_aiter_shared_expert_unsupported() is None - - -# -- _resolve_probe_python / probe interpreter selection ------------------ -def test_resolve_probe_python_prefers_magpie_interpreter(monkeypatch) -> None: - # The harness interpreter is used directly; no vllm-exe resolution is tried. - monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/srv/venv/bin/python") - monkeypatch.setattr( - gr.shutil, "which", lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("which() should not be called")) - ) - assert gr._resolve_probe_python() == "/srv/venv/bin/python" - - -def test_resolve_probe_python_falls_back_to_vllm_venv(monkeypatch) -> None: - # magpie_python is the canonical default -> pin the venv that backs ``vllm serve``. - monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/opt/venv/bin/python") - monkeypatch.setattr(gr.shutil, "which", lambda name: "/other/venv/bin/vllm" if name == "vllm" else None) - monkeypatch.setattr(gr.os.path, "exists", lambda p: p == "/other/venv/bin/python") - assert gr._resolve_probe_python() == "/other/venv/bin/python" - - -def test_resolve_probe_python_no_bare_python3_fallback(monkeypatch) -> None: - # With no resolvable vllm exe, fall back to the canonical magpie default — never a bare "python3". - monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/opt/venv/bin/python") - monkeypatch.setattr(gr.shutil, "which", lambda *_a, **_k: None) - assert gr._resolve_probe_python() == "/opt/venv/bin/python" - - -def test_probe_invokes_resolved_interpreter(monkeypatch) -> None: - # The capability probe must run under the resolved interpreter, not "python3". - _clear_cap_cache() - monkeypatch.setattr(gr, "_resolve_probe_python", lambda: "/srv/venv/bin/python") - seen: dict = {} - - class _Proc: - stdout = json.dumps({"status": "ok"}) - stderr = "" - - def fake_run(cmd, *_a, **_k): - seen["cmd"] = list(cmd) - return _Proc() - - monkeypatch.setattr(gr.subprocess, "run", fake_run) - gr._probe_vllm_aiter_shared_expert_unsupported() - assert seen["cmd"][0] == "/srv/venv/bin/python" - assert seen["cmd"][0] != "python3" - - -# -- apply_runtime_benchmark_overrides ------------------------------------ -def test_runtime_overrides_model_precision_and_gpu_no_framework_agent(monkeypatch) -> None: - monkeypatch.setenv("PRECISION", "fp8") - for k in ("ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): - monkeypatch.delenv(k, raising=False) - bench: dict = {} # no framework -> benchmark_script popped - gr.apply_runtime_benchmark_overrides( - bench, - model_path="/models/x", - gpu_type="mi355x", - ) - assert bench["model"] == "/models/x" - assert bench["precision"] == "fp8" - assert bench["runner_type"] == "mi355x" - assert "benchmark_script" not in bench - - -def test_runtime_overrides_framework_pins_generic_script(monkeypatch) -> None: - for k in ("PRECISION", "ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): - monkeypatch.delenv(k, raising=False) - bench = {"framework": "sglang"} - gr.apply_runtime_benchmark_overrides(bench, gpu_type="mi300x") - assert bench["benchmark_script"] == "sglang_mi300x.sh" - - -def test_runtime_overrides_env_ints_and_rocr_autofill(monkeypatch) -> None: - for k in ("PRECISION",): - monkeypatch.delenv(k, raising=False) - monkeypatch.setenv("ISL", "128") - monkeypatch.setenv("TP", "2") - monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising=False) - bench: dict = {} - envs = gr.apply_runtime_benchmark_overrides(bench) - assert envs["ISL"] == 128 - assert envs["TP"] == 2 - # TP>1 with no explicit ROCR -> auto-filled device list - assert envs["ROCR_VISIBLE_DEVICES"] == "0,1" - - -def test_runtime_overrides_explicit_benchmark_script_wins(monkeypatch) -> None: - for k in ("PRECISION", "ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): - monkeypatch.delenv(k, raising=False) - bench = {"framework": "vllm"} - gr.apply_runtime_benchmark_overrides( - bench, - gpu_type="mi300x", - benchmark_script="custom.sh", - ) - assert bench["benchmark_script"] == "custom.sh" - - -# -- _parse_report --------------------------------------------------------- -def test_parse_report_missing(tmp_path: Path) -> None: - assert gr._parse_report(tmp_path) is None - - -def test_parse_report_invalid_json(tmp_path: Path) -> None: - (tmp_path / "benchmark_report.json").write_text("{bad", encoding="utf-8") - assert gr._parse_report(tmp_path) is None - - -def test_parse_report_non_dict(tmp_path: Path) -> None: - (tmp_path / "benchmark_report.json").write_text("[1,2,3]", encoding="utf-8") - assert gr._parse_report(tmp_path) is None - - -def test_parse_report_valid(tmp_path: Path) -> None: - (tmp_path / "benchmark_report.json").write_text( - json.dumps({"output_throughput": 100.0}), - encoding="utf-8", - ) - assert gr._parse_report(tmp_path) == {"output_throughput": 100.0} - - -# -- _build_variant_yaml --------------------------------------------------- -def test_build_variant_yaml_injects_extra_envs(tmp_path: Path) -> None: - base = tmp_path / "base.yaml" - base.write_text( - yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), - encoding="utf-8", - ) - variant = _variant("v1", "--foo 1", envs={"USE_AITER": "1"}) - out = gr._build_variant_yaml( - base, - "", - variant, - output_subdir=tmp_path / "v1", - ) - cfg = yaml.safe_load(out.read_text(encoding="utf-8")) - envs = cfg["benchmark"]["envs"] - assert envs["USE_AITER"] == "1" - # variant + base server args merged into the framework's args env - arg_key = gr.server_args_env_name("sglang") - assert "--foo 1" in envs[arg_key] - - -def test_build_variant_yaml_dedupes_repeated_flags(tmp_path: Path) -> None: - """When base YAML + base_extra_args + variant all set the same flag, the materialized YAML must contain each flag - only once (last wins). - """ - base = tmp_path / "base.yaml" - base.write_text( - yaml.safe_dump( - { - "benchmark": { - "framework": "vllm", - "envs": {"EXTRA_VLLM_ARGS": "--attention-backend ROCM_ATTN"}, - }, - } - ), - encoding="utf-8", - ) - variant = _variant("v1", "--attention-backend ROCM_AITER_FA") - out = gr._build_variant_yaml( - base, - "", - variant, - output_subdir=tmp_path / "v1", - ) - cfg = yaml.safe_load(out.read_text(encoding="utf-8")) - args = cfg["benchmark"]["envs"]["EXTRA_VLLM_ARGS"] - assert args.count("--attention-backend") == 1, f"duplicate flag: {args}" - assert "ROCM_AITER_FA" in args, "last-wins should keep variant value" - - -def test_build_variant_yaml_prepends_legit_overlay(tmp_path: Path) -> None: - # SWSPLAT-42358: a legitimate overlay is a single existing directory; it is prepended to PYTHONPATH unchanged. - base = tmp_path / "base.yaml" - base.write_text( - yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), - encoding="utf-8", - ) - overlay = tmp_path / "overlay" - overlay.mkdir() - variant = _variant("v1") - variant.overlay_pythonpath = str(overlay) # type: ignore[attr-defined] - out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") - envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] - assert envs.get("PYTHONPATH", "").startswith(str(overlay)) - - -@pytest.mark.parametrize( - ("config_envs", "tail"), - [({}, ["/inh/a", "/inh/b"]), ({"PYTHONPATH": "/cfg"}, ["/cfg"])], - ids=["inherited", "config-authoritative"], -) -def test_build_variant_yaml_overlay_builds_on_the_launch_pythonpath(tmp_path, monkeypatch, config_envs, tail) -> None: - # The launch overlays benchmark.envs on os.environ, so a config without PYTHONPATH carries the inherited one. - # The overlay is relative so the path carries no drive colon on any host. - import os - - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/inh/a", "/inh/b"])) - (tmp_path / "overlay").mkdir() - base = tmp_path / "base.yaml" - base.write_text(yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": config_envs}}), encoding="utf-8") - variant = _variant("v1") - variant.overlay_pythonpath = "overlay" # type: ignore[attr-defined] - out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") - envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] - assert envs["PYTHONPATH"] == os.pathsep.join(["overlay", *tail]) - - -def test_build_variant_yaml_drops_unsafe_overlay(tmp_path: Path) -> None: - # SWSPLAT-42358: a ``:``-joined / traversal / non-existent overlay is dropped (would otherwise smuggle extra - # PYTHONPATH entries or escape). - base = tmp_path / "base.yaml" - base.write_text( - yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), - encoding="utf-8", - ) - overlay = tmp_path / "overlay" - overlay.mkdir() - for bad in (f"{overlay}:/etc", f"{overlay}/../evil", str(tmp_path / "missing")): - variant = _variant("v1") - variant.overlay_pythonpath = bad # type: ignore[attr-defined] - out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") - envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] - assert "PYTHONPATH" not in envs or bad not in envs.get("PYTHONPATH", ""), bad - - -def test_shell_safe_dedupe_preserves_json_and_dedupes_other_flags() -> None: - """JSON stays an opaque argv token while unrelated flags dedupe last-wins.""" - args = '--json-model-override-args {"rope_scaling":null} --context-length 8192 --context-length 4096' - out = gr._shell_safe_dedupe(args) - assert out == '--json-model-override-args {"rope_scaling":null} --context-length 4096' - assert json.loads(out.split("--json-model-override-args ", 1)[1].split(" ", 1)[0]) == {"rope_scaling": None} - - -def test_shell_safe_dedupe_leaves_multi_value_arg_untouched() -> None: - """Multi-token flags must not be reassembled as stray positional tokens.""" - args = "--cuda-graph-bs 1 2 4 --cuda-graph-bs 8 16" - out = gr._shell_safe_dedupe(args) - assert out == args - - -def test_shell_safe_dedupe_normalizes_equals_form() -> None: - """--flag=value and --flag value must dedupe to one (last wins).""" - out = gr._shell_safe_dedupe("--attention-backend=ROCM_ATTN --attention-backend ROCM_AITER_FA") - assert out.count("--attention-backend") == 1, out - assert "ROCM_AITER_FA" in out, "last-wins should keep the later value" - assert "ROCM_ATTN" not in out - - -def test_shell_safe_dedupe_simple_last_wins() -> None: - out = gr._shell_safe_dedupe("--tp 1 --tp 8 --mem-fraction-static 0.9") - assert out == "--tp 8 --mem-fraction-static 0.9" - - -def test_compose_server_args_replace_still_applies_remove_args() -> None: - out = gr.compose_server_args( - inherited_args="--bad-base 1", - base_extra_args="--also-bad 2", - variant_extra_args="--bad-base 3 --keep 4", - remove_args=["--bad-base"], - args_mode="replace", - ) - assert out == "--also-bad 2 --keep 4" - - -def test_compose_server_args_launches_with_the_harness_flags_the_baseline_ran_with() -> None: - out = gr.compose_server_args( - inherited_args="--no-enable-prefix-caching --block-size 128", - base_extra_args="--no-enable-prefix-caching", - variant_extra_args="--kv-cache-dtype fp8 --no-enable-prefix-caching", - ) - assert "--no-enable-prefix-caching" in out - assert "--block-size 128" in out - assert "--kv-cache-dtype fp8" in out - - -def test_compose_server_args_keeps_the_harness_flag_in_replace_mode() -> None: - out = gr.compose_server_args( - inherited_args="--ignored", - base_extra_args="--no-enable-prefix-caching --max-num-seqs 256", - variant_extra_args="--kv-cache-dtype fp8", - args_mode="replace", - ) - assert out == "--no-enable-prefix-caching --max-num-seqs 256 --kv-cache-dtype fp8" - - -def test_launch_keeps_what_the_persisted_recipe_strips() -> None: - from hyperloom.inference_optimizer.grid_server_args import strip_benchmark_harness_flags - - launched = gr.compose_server_args(base_extra_args="--no-enable-prefix-caching --max-num-seqs 256") - assert "--no-enable-prefix-caching" in launched - assert strip_benchmark_harness_flags(launched) == "--max-num-seqs 256" - - -def test_remove_server_args_accepts_multi_flag_string() -> None: - out = gr.remove_server_args( - "--flag-a --flag-b --flag-c 3 --keep 4", - "--flag-a --flag-b --flag-c", - ) - assert out == "--keep 4" - - -def test_remove_server_args_keeps_a_sibling_json_value_parseable() -> None: - """Removing one flag must not corrupt a JSON-valued sibling.""" - args = ( - '--online_quant_config {"global_quant_config":"ptpc_fp8",' - '"exclude_layer":["*.mlp.gate","*expert*"]} ' - "--no-enable-prefix-caching --tp 8" - ) - out = gr.remove_server_args(args, ["--no-enable-prefix-caching"]) - tokens = out.split(" ") - assert tokens[0] == "--online_quant_config" - assert json.loads(tokens[1]) == { - "global_quant_config": "ptpc_fp8", - "exclude_layer": ["*.mlp.gate", "*expert*"], - } - assert tokens[2:] == ["--tp", "8"] - - -def test_remove_server_args_unbalanced_brace_drops_only_its_own_flag() -> None: - """A stray ``}`` is not a JSON blob and must not take the tail with it.""" - out = gr.remove_server_args("--foo a} --tp 8 --max-num-seqs 64", ["--foo"]) - assert out == "--tp 8 --max-num-seqs 64" +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Supplemental coverage for _grid_runner pure helpers: compatibility filter model-class drop, runtime override env branches, report parsing, and per-variant yaml env injection.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +import yaml + +from hyperloom.orchestrator.actions.executors import _grid_runner as gr +from hyperloom.orchestrator.actions.executors import _benchmark_interpreter as bi + +# Patch compatibility-filter helpers in the ``_grid_variant_filter`` sibling, where apply_compatibility_filter +# resolves them (not via the re-export). +from hyperloom.orchestrator.actions.executors import _grid_variant_filter as vf + + +def _variant(name: str, args: str = "", envs: dict | None = None) -> gr.GridVariant: + return gr.GridVariant(name=name, extra_server_args=args, extra_envs=envs or {}) + + +def test_validate_magpie_python_override_requires_python(tmp_path): + py = tmp_path / "python3" + py.write_text("", encoding="utf-8") + bash = tmp_path / "bash" + bash.write_text("", encoding="utf-8") + assert gr._validate_magpie_python_override(str(py)) == str(py) + with pytest.raises(ValueError, match="Python interpreter"): + gr._validate_magpie_python_override(str(bash)) + assert gr._validate_magpie_python_override(sys.executable).endswith(Path(sys.executable).name) + + +# -- apply_compatibility_filter ------------------------------------------- +def test_compatibility_filter_drops_on_model_class(monkeypatch) -> None: + monkeypatch.setattr(vf, "_detect_model_class", lambda mp: (False, False)) + grid = [_variant("mla", "--enable-flashinfer-mla"), _variant("plain", "")] + kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="meta-llama-3-8b") + assert [v.name for v in kept] == ["plain"] + assert len(dropped) == 1 + assert dropped[0]["source"] == "compatibility_filter" + assert "MLA" in dropped[0]["reason"] + + +def test_compatibility_filter_keeps_a_flag_the_model_class_supports(monkeypatch) -> None: + # Whether the installed server accepts the flag is left to its own argparse at launch. + monkeypatch.setattr(vf, "_detect_model_class", lambda mp: (True, True)) + grid = [_variant("moe", "--enable-ep-moe")] + kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="deepseek-moe") + assert [v.name for v in kept] == ["moe"] and dropped == [] + + +def test_compatibility_filter_no_model_path_assumes_compatible() -> None: + grid = [_variant("moe", "--enable-ep-moe")] + kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="") + assert [v.name for v in kept] == ["moe"] and dropped == [] + + +# -- ungrounded_env_file_reason (env-value file grounding) ---------------- +def test_ungrounded_env_file_reason_drops_missing_config_path(tmp_path) -> None: + # A proposal may point a config env at an artifact no stage authored; the server would read it + # unconditionally and die at config load, so the variant must be dropped with a reason. + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(tmp_path / "nope.csv")}) + reason = gr.ungrounded_env_file_reason(v) + assert reason and "AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE" in reason + assert str(tmp_path / "nope.csv") in reason + + +def test_ungrounded_env_file_reason_passes_authored_file(tmp_path) -> None: + csv = tmp_path / "tuned.csv" + csv.write_text("gfx,M,N,K,splitK\ngfx950,64,5120,5120,2\n", encoding="utf-8") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(csv)}) + assert gr.ungrounded_env_file_reason(v) is None + + +def test_ungrounded_env_file_reason_accepts_colon_joined_existing_files(tmp_path) -> None: + # A set AITER_CONFIG_* is aiter's SET form: a `:`-joined list, each element loaded (csvs_aiter_will_load). + # Both files on disk is a valid variant and must not be dropped. + a = tmp_path / "t.csv" + a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") + b = tmp_path / "t2.csv" + b.write_text("gfx,M,N,K\ngfx950,4,5,6\n", encoding="utf-8") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{b}"}) + assert gr.ungrounded_env_file_reason(v) is None + + +def test_ungrounded_env_file_reason_drops_colon_joined_missing_element(tmp_path) -> None: + a = tmp_path / "t.csv" + a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") + missing = str(tmp_path / "never_authored.csv") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{missing}"}) + reason = gr.ungrounded_env_file_reason(v) + assert reason and missing in reason and str(a) not in reason.split("element(s)")[1].split(" do not exist")[0] + + +def test_ungrounded_env_file_reason_treats_an_unreadable_path_as_missing(monkeypatch) -> None: + # An unreadable parent or an overlong name makes Path.is_file() raise; the variant is dropped, the grid runs on. + def raise_eacces(self): + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(gr.Path, "is_file", raise_eacces) + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": "/root/private/tuned.csv"}) + reason = gr.ungrounded_env_file_reason(v) + assert reason and "/root/private/tuned.csv" in reason + + +def test_ungrounded_env_file_reason_noop_without_config_envs(tmp_path) -> None: + # Non-file envs (even under aiter-ish prefixes like the JSON-payload AITER_CONFIG) must stay untouched. + v = _variant( + "plain", + envs={ + "AITER_CONFIG": '{"path": "/tmp/never_scanned.json"}', + "SGLANG_USE_AITER": "1", + "AITER_CONFIG_GEMM_A8W8": "", + }, + ) + assert gr.ungrounded_env_file_reason(v) is None + + +# -- unsupported_capability_reason (env-flag build probe) ----------------- +def _clear_cap_cache() -> None: + gr._CAP_PROBE_CACHE.clear() + + +def test_capability_reason_noop_when_flag_absent(monkeypatch) -> None: + monkeypatch.setenv("FRAMEWORK", "vllm") + _clear_cap_cache() + # Probe must NOT even run when the flag isn't set on the variant. + monkeypatch.setattr( + gr, + "_probe_vllm_aiter_shared_expert_unsupported", + lambda: (_ for _ in ()).throw(AssertionError("probe should not run")), + ) + assert gr.unsupported_capability_reason(_variant("plain")) is None + + +def test_capability_reason_noop_for_non_vllm_framework(monkeypatch) -> None: + monkeypatch.setenv("FRAMEWORK", "sglang") + _clear_cap_cache() + v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1"}) + assert gr.unsupported_capability_reason(v) is None + + +def test_capability_reason_drops_when_module_missing(monkeypatch) -> None: + monkeypatch.setenv("FRAMEWORK", "vllm") + _clear_cap_cache() + monkeypatch.setattr( + gr, + "_probe_vllm_aiter_shared_expert_unsupported", + lambda: "missing module(s): vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe", + ) + v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1"}) + reason = gr.unsupported_capability_reason(v) + assert reason is not None and "rocm_aiter_fused_moe" in reason + + +def test_capability_reason_falsey_flag_not_probed(monkeypatch) -> None: + monkeypatch.setenv("FRAMEWORK", "vllm") + _clear_cap_cache() + monkeypatch.setattr( + gr, + "_probe_vllm_aiter_shared_expert_unsupported", + lambda: (_ for _ in ()).throw(AssertionError("probe should not run")), + ) + v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0"}) + assert gr.unsupported_capability_reason(v) is None + + +def test_probe_caches_ok_and_unsupported(monkeypatch) -> None: + _clear_cap_cache() + calls = {"n": 0} + + class _Proc: + def __init__(self, out: str) -> None: + self.stdout = out + self.stderr = "" + + def fake_run(*_a, **_k): + calls["n"] += 1 + return _Proc(json.dumps({"status": "ok"})) + + monkeypatch.setattr(gr.subprocess, "run", fake_run) + assert gr._probe_vllm_aiter_shared_expert_unsupported() is None + # Second call must hit the cache (no second subprocess). + assert gr._probe_vllm_aiter_shared_expert_unsupported() is None + assert calls["n"] == 1 + + +def test_probe_unknown_not_cached(monkeypatch) -> None: + _clear_cap_cache() + + class _Proc: + stdout = json.dumps({"status": "unknown"}) + stderr = "" + + monkeypatch.setattr(gr.subprocess, "run", lambda *_a, **_k: _Proc()) + assert gr._probe_vllm_aiter_shared_expert_unsupported() is None + assert "vllm" not in gr._CAP_PROBE_CACHE + + +def test_probe_swallows_subprocess_error(monkeypatch) -> None: + _clear_cap_cache() + + def boom(*_a, **_k): + raise OSError("no python3") + + monkeypatch.setattr(gr.subprocess, "run", boom) + assert gr._probe_vllm_aiter_shared_expert_unsupported() is None + + +# -- _resolve_probe_python / probe interpreter selection ------------------ +def test_resolve_probe_python_prefers_magpie_interpreter(monkeypatch) -> None: + # The harness interpreter is used directly; no vllm-exe resolution is tried. + monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/srv/venv/bin/python") + monkeypatch.setattr( + gr.shutil, "which", lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("which() should not be called")) + ) + assert gr._resolve_probe_python() == "/srv/venv/bin/python" + + +def test_resolve_probe_python_falls_back_to_vllm_venv(monkeypatch) -> None: + # magpie_python is the canonical default -> pin the venv that backs ``vllm serve``. + monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/opt/venv/bin/python") + monkeypatch.setattr(gr.shutil, "which", lambda name: "/other/venv/bin/vllm" if name == "vllm" else None) + monkeypatch.setattr(gr.os.path, "exists", lambda p: p == "/other/venv/bin/python") + assert gr._resolve_probe_python() == "/other/venv/bin/python" + + +def test_resolve_probe_python_no_bare_python3_fallback(monkeypatch) -> None: + # With no resolvable vllm exe, fall back to the canonical magpie default — never a bare "python3". + monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/opt/venv/bin/python") + monkeypatch.setattr(gr.shutil, "which", lambda *_a, **_k: None) + assert gr._resolve_probe_python() == "/opt/venv/bin/python" + + +def test_probe_invokes_resolved_interpreter(monkeypatch) -> None: + # The capability probe must run under the resolved interpreter, not "python3". + _clear_cap_cache() + monkeypatch.setattr(gr, "_resolve_probe_python", lambda: "/srv/venv/bin/python") + seen: dict = {} + + class _Proc: + stdout = json.dumps({"status": "ok"}) + stderr = "" + + def fake_run(cmd, *_a, **_k): + seen["cmd"] = list(cmd) + return _Proc() + + monkeypatch.setattr(gr.subprocess, "run", fake_run) + gr._probe_vllm_aiter_shared_expert_unsupported() + assert seen["cmd"][0] == "/srv/venv/bin/python" + assert seen["cmd"][0] != "python3" + + +# -- apply_runtime_benchmark_overrides ------------------------------------ +def test_runtime_overrides_model_precision_and_gpu_no_framework_agent(monkeypatch) -> None: + monkeypatch.setenv("PRECISION", "fp8") + for k in ("ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): + monkeypatch.delenv(k, raising=False) + bench: dict = {} # no framework -> benchmark_script popped + gr.apply_runtime_benchmark_overrides( + bench, + model_path="/models/x", + gpu_type="mi355x", + ) + assert bench["model"] == "/models/x" + assert bench["precision"] == "fp8" + assert bench["runner_type"] == "mi355x" + assert "benchmark_script" not in bench + + +def test_runtime_overrides_framework_pins_generic_script(monkeypatch) -> None: + for k in ("PRECISION", "ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): + monkeypatch.delenv(k, raising=False) + bench = {"framework": "sglang"} + gr.apply_runtime_benchmark_overrides(bench, gpu_type="mi300x") + assert bench["benchmark_script"] == "sglang_mi300x.sh" + + +def test_runtime_overrides_env_ints_and_rocr_autofill(monkeypatch) -> None: + for k in ("PRECISION",): + monkeypatch.delenv(k, raising=False) + monkeypatch.setenv("ISL", "128") + monkeypatch.setenv("TP", "2") + monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising=False) + bench: dict = {} + envs = gr.apply_runtime_benchmark_overrides(bench) + assert envs["ISL"] == 128 + assert envs["TP"] == 2 + # TP>1 with no explicit ROCR -> auto-filled device list + assert envs["ROCR_VISIBLE_DEVICES"] == "0,1" + + +def test_runtime_overrides_explicit_benchmark_script_wins(monkeypatch) -> None: + for k in ("PRECISION", "ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): + monkeypatch.delenv(k, raising=False) + bench = {"framework": "vllm"} + gr.apply_runtime_benchmark_overrides( + bench, + gpu_type="mi300x", + benchmark_script="custom.sh", + ) + assert bench["benchmark_script"] == "custom.sh" + + +# -- _parse_report --------------------------------------------------------- +def test_parse_report_missing(tmp_path: Path) -> None: + assert gr._parse_report(tmp_path) is None + + +def test_parse_report_invalid_json(tmp_path: Path) -> None: + (tmp_path / "benchmark_report.json").write_text("{bad", encoding="utf-8") + assert gr._parse_report(tmp_path) is None + + +def test_parse_report_non_dict(tmp_path: Path) -> None: + (tmp_path / "benchmark_report.json").write_text("[1,2,3]", encoding="utf-8") + assert gr._parse_report(tmp_path) is None + + +def test_parse_report_valid(tmp_path: Path) -> None: + (tmp_path / "benchmark_report.json").write_text( + json.dumps({"output_throughput": 100.0}), + encoding="utf-8", + ) + assert gr._parse_report(tmp_path) == {"output_throughput": 100.0} + + +# -- _build_variant_yaml --------------------------------------------------- +def test_build_variant_yaml_injects_extra_envs(tmp_path: Path) -> None: + base = tmp_path / "base.yaml" + base.write_text( + yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), + encoding="utf-8", + ) + variant = _variant("v1", "--foo 1", envs={"USE_AITER": "1"}) + out = gr._build_variant_yaml( + base, + "", + variant, + output_subdir=tmp_path / "v1", + ) + cfg = yaml.safe_load(out.read_text(encoding="utf-8")) + envs = cfg["benchmark"]["envs"] + assert envs["USE_AITER"] == "1" + # variant + base server args merged into the framework's args env + arg_key = gr.server_args_env_name("sglang") + assert "--foo 1" in envs[arg_key] + + +def test_build_variant_yaml_dedupes_repeated_flags(tmp_path: Path) -> None: + """When base YAML + base_extra_args + variant all set the same flag, the materialized YAML must contain each flag + only once (last wins). + """ + base = tmp_path / "base.yaml" + base.write_text( + yaml.safe_dump( + { + "benchmark": { + "framework": "vllm", + "envs": {"EXTRA_VLLM_ARGS": "--attention-backend ROCM_ATTN"}, + }, + } + ), + encoding="utf-8", + ) + variant = _variant("v1", "--attention-backend ROCM_AITER_FA") + out = gr._build_variant_yaml( + base, + "", + variant, + output_subdir=tmp_path / "v1", + ) + cfg = yaml.safe_load(out.read_text(encoding="utf-8")) + args = cfg["benchmark"]["envs"]["EXTRA_VLLM_ARGS"] + assert args.count("--attention-backend") == 1, f"duplicate flag: {args}" + assert "ROCM_AITER_FA" in args, "last-wins should keep variant value" + + +def test_build_variant_yaml_prepends_legit_overlay(tmp_path: Path) -> None: + # SWSPLAT-42358: a legitimate overlay is a single existing directory; it is prepended to PYTHONPATH unchanged. + base = tmp_path / "base.yaml" + base.write_text( + yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), + encoding="utf-8", + ) + overlay = tmp_path / "overlay" + overlay.mkdir() + variant = _variant("v1") + variant.overlay_pythonpath = str(overlay) # type: ignore[attr-defined] + out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") + envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] + assert envs.get("PYTHONPATH", "").startswith(str(overlay)) + + +@pytest.mark.parametrize( + ("config_envs", "tail"), + [({}, ["/inh/a", "/inh/b"]), ({"PYTHONPATH": "/cfg"}, ["/cfg"])], + ids=["inherited", "config-authoritative"], +) +def test_build_variant_yaml_overlay_builds_on_the_launch_pythonpath(tmp_path, monkeypatch, config_envs, tail) -> None: + # The launch overlays benchmark.envs on os.environ, so a config without PYTHONPATH carries the inherited one. + # The overlay is relative so the path carries no drive colon on any host. + import os + + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/inh/a", "/inh/b"])) + (tmp_path / "overlay").mkdir() + base = tmp_path / "base.yaml" + base.write_text(yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": config_envs}}), encoding="utf-8") + variant = _variant("v1") + variant.overlay_pythonpath = "overlay" # type: ignore[attr-defined] + out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") + envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] + assert envs["PYTHONPATH"] == os.pathsep.join(["overlay", *tail]) + + +def test_build_variant_yaml_drops_unsafe_overlay(tmp_path: Path) -> None: + # SWSPLAT-42358: a ``:``-joined / traversal / non-existent overlay is dropped (would otherwise smuggle extra + # PYTHONPATH entries or escape). + base = tmp_path / "base.yaml" + base.write_text( + yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), + encoding="utf-8", + ) + overlay = tmp_path / "overlay" + overlay.mkdir() + for bad in (f"{overlay}:/etc", f"{overlay}/../evil", str(tmp_path / "missing")): + variant = _variant("v1") + variant.overlay_pythonpath = bad # type: ignore[attr-defined] + out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") + envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] + assert "PYTHONPATH" not in envs or bad not in envs.get("PYTHONPATH", ""), bad + + +def test_shell_safe_dedupe_preserves_json_and_dedupes_other_flags() -> None: + """JSON stays an opaque argv token while unrelated flags dedupe last-wins.""" + args = '--json-model-override-args {"rope_scaling":null} --context-length 8192 --context-length 4096' + out = gr._shell_safe_dedupe(args) + assert out == '--json-model-override-args {"rope_scaling":null} --context-length 4096' + assert json.loads(out.split("--json-model-override-args ", 1)[1].split(" ", 1)[0]) == {"rope_scaling": None} + + +def test_shell_safe_dedupe_leaves_multi_value_arg_untouched() -> None: + """Multi-token flags must not be reassembled as stray positional tokens.""" + args = "--cuda-graph-bs 1 2 4 --cuda-graph-bs 8 16" + out = gr._shell_safe_dedupe(args) + assert out == args + + +def test_shell_safe_dedupe_normalizes_equals_form() -> None: + """--flag=value and --flag value must dedupe to one (last wins).""" + out = gr._shell_safe_dedupe("--attention-backend=ROCM_ATTN --attention-backend ROCM_AITER_FA") + assert out.count("--attention-backend") == 1, out + assert "ROCM_AITER_FA" in out, "last-wins should keep the later value" + assert "ROCM_ATTN" not in out + + +def test_shell_safe_dedupe_simple_last_wins() -> None: + out = gr._shell_safe_dedupe("--tp 1 --tp 8 --mem-fraction-static 0.9") + assert out == "--tp 8 --mem-fraction-static 0.9" + + +def test_compose_server_args_replace_still_applies_remove_args() -> None: + out = gr.compose_server_args( + inherited_args="--bad-base 1", + base_extra_args="--also-bad 2", + variant_extra_args="--bad-base 3 --keep 4", + remove_args=["--bad-base"], + args_mode="replace", + ) + assert out == "--also-bad 2 --keep 4" + + +def test_compose_server_args_launches_with_the_harness_flags_the_baseline_ran_with() -> None: + out = gr.compose_server_args( + inherited_args="--no-enable-prefix-caching --block-size 128", + base_extra_args="--no-enable-prefix-caching", + variant_extra_args="--kv-cache-dtype fp8 --no-enable-prefix-caching", + ) + assert "--no-enable-prefix-caching" in out + assert "--block-size 128" in out + assert "--kv-cache-dtype fp8" in out + + +def test_compose_server_args_keeps_the_harness_flag_in_replace_mode() -> None: + out = gr.compose_server_args( + inherited_args="--ignored", + base_extra_args="--no-enable-prefix-caching --max-num-seqs 256", + variant_extra_args="--kv-cache-dtype fp8", + args_mode="replace", + ) + assert out == "--no-enable-prefix-caching --max-num-seqs 256 --kv-cache-dtype fp8" + + +def test_launch_keeps_what_the_persisted_recipe_strips() -> None: + from hyperloom.inference_optimizer.grid_server_args import strip_benchmark_harness_flags + + launched = gr.compose_server_args(base_extra_args="--no-enable-prefix-caching --max-num-seqs 256") + assert "--no-enable-prefix-caching" in launched + assert strip_benchmark_harness_flags(launched) == "--max-num-seqs 256" + + +def test_remove_server_args_accepts_multi_flag_string() -> None: + out = gr.remove_server_args( + "--flag-a --flag-b --flag-c 3 --keep 4", + "--flag-a --flag-b --flag-c", + ) + assert out == "--keep 4" + + +def test_remove_server_args_keeps_a_sibling_json_value_parseable() -> None: + """Removing one flag must not corrupt a JSON-valued sibling.""" + args = ( + '--online_quant_config {"global_quant_config":"ptpc_fp8",' + '"exclude_layer":["*.mlp.gate","*expert*"]} ' + "--no-enable-prefix-caching --tp 8" + ) + out = gr.remove_server_args(args, ["--no-enable-prefix-caching"]) + tokens = out.split(" ") + assert tokens[0] == "--online_quant_config" + assert json.loads(tokens[1]) == { + "global_quant_config": "ptpc_fp8", + "exclude_layer": ["*.mlp.gate", "*expert*"], + } + assert tokens[2:] == ["--tp", "8"] + + +def test_remove_server_args_unbalanced_brace_drops_only_its_own_flag() -> None: + """A stray ``}`` is not a JSON blob and must not take the tail with it.""" + out = gr.remove_server_args("--foo a} --tp 8 --max-num-seqs 64", ["--foo"]) + assert out == "--tp 8 --max-num-seqs 64" diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 94ecac3895..aef86b46d2 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1,2280 +1,2289 @@ -# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Shared helper for the ``explore`` executor's grid runs.""" - -from __future__ import annotations - -import asyncio -import json -import logging -import os -import re -import shutil -import subprocess -import time -from pathlib import Path -from typing import Any, Callable - -import yaml - -from hyperloom.common.coerce import to_str_list -from hyperloom.common.env import env_flag, is_truthy -from hyperloom.common.perf_metric import constraint_fields -from hyperloom.common.env_safety import ( - BLOCKED_CHILD_ENV_NAMES, - BLOCKED_EXTERNAL_ENV_NAMES, - _ENV_KEY_RE, - is_python_package_root, - redact_secret_values, - scrub_benchmark_process_env, -) - -from ...phases import machine_state as _phase_state -from hyperloom.inference_optimizer.trace.task_progress import heartbeat_while_output_flows, report_progress -from ..stop_attribution import ( - ORCHESTRATOR_CANCELLED_CLASS, - SESSION_TIME_EXHAUSTED_CLASS, - STOPPED_BY_THE_RUN, - StoppedByTheRun, -) -from ._benchmark_runtime import apply_runtime_benchmark_overrides as apply_runtime_benchmark_overrides -from ._benchmark_interpreter import ( - _resolve_magpie_python as _resolve_magpie_python, - _resolve_probe_python as _resolve_probe_python, -) -from ._accuracy_gate import materialized_run_eval_disabled -from ._recipe_script import launcher_overwritten_envs -from ._subprocess_kill import ( - AGENTX_PREFLIGHT_ERROR_CLASS, - AGENTX_PREFLIGHT_RETURNCODE, - DETOKENIZER_STALL_RETURNCODE, - EVAL_PROBE_UNPATCHABLE_RETURNCODE, - ORCHESTRATOR_CANCELLED_RETURNCODE, - SERVER_DEAD_RETURNCODE, - SESSION_TIME_EXHAUSTED_RETURNCODE, - resolve_benchmark_timeouts, - run_with_session_kill, - server_log_death_excerpt, - session_deadline_to_remaining_sec, - sync_agentx_phase_wait, -) -from .benchmark_result import ( - extract_benchmark_measurement, - harvest_leaked_artifacts, - select_run_workspace, - served_complete_protocol, - snapshot_workspaces, -) -from ._gpu_metrics import write_gpu_metrics_from_report -from .benchmark_backend import build_benchmark_command -from ._inferencex_patcher import ( - ensure_benchmark_lib_eval_start_patched, - ensure_eval_probe_patched, - ensure_eval_unbound_outputs_patched, - eval_probe_targets_exist, -) -from ._launch_evidence import build_launch_evidence, persist_launch_evidence -from ...measurement.runtime_findings import persist_runtime_findings, scan_server_log -from ._server_argv import seal_server_argv - -# Re-exported from sibling modules to keep the module namespace intact. -from ._grid_base import ( - DEFAULT_KEEP_THRESHOLD_PCT as DEFAULT_KEEP_THRESHOLD_PCT, - GridVariant as GridVariant, - coerce_extra_envs as coerce_extra_envs, - VariantResult as VariantResult, -) -from hyperloom.inference_optimizer.grid_server_args import ( - server_args_env_name as server_args_env_name, - merge_server_args as merge_server_args, - compose_server_args as compose_server_args, - remove_server_args as remove_server_args, - compact_json_server_args as compact_json_server_args, - _SPACE_VALUE_FLAGS as _SPACE_VALUE_FLAGS, - _MULTI_VALUE_FLAGS as _MULTI_VALUE_FLAGS, - _VLLM_SINGLE_VALUE_FLAGS as _VLLM_SINGLE_VALUE_FLAGS, - dedup_vllm_server_args as dedup_vllm_server_args, - _shell_safe_dedupe as _shell_safe_dedupe, - DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC as DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC, - SGLANG_WATCHDOG_TIMEOUT_ENV as SGLANG_WATCHDOG_TIMEOUT_ENV, - _SGLANG_WATCHDOG_FLAG as _SGLANG_WATCHDOG_FLAG, - _SGLANG_WATCHDOG_RE as _SGLANG_WATCHDOG_RE, - resolve_sglang_watchdog_timeout as resolve_sglang_watchdog_timeout, - inject_sglang_watchdog_timeout as inject_sglang_watchdog_timeout, - DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS as DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS, - DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS as DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS, - SGLANG_CONTEXT_HEADROOM_ENV as SGLANG_CONTEXT_HEADROOM_ENV, - SGLANG_CONTEXT_FLOOR_ENV as SGLANG_CONTEXT_FLOOR_ENV, - _SGLANG_CONTEXT_LENGTH_FLAG as _SGLANG_CONTEXT_LENGTH_FLAG, - _SGLANG_CONTEXT_LENGTH_RE as _SGLANG_CONTEXT_LENGTH_RE, - _SGLANG_ATTN_BACKEND_FLAG as _SGLANG_ATTN_BACKEND_FLAG, - _SGLANG_ATTN_BACKEND_RE as _SGLANG_ATTN_BACKEND_RE, - _SGLANG_DUAL_CHUNK_BACKEND as _SGLANG_DUAL_CHUNK_BACKEND, - _resolve_nonneg_int_env as _resolve_nonneg_int_env, - resolve_sglang_context_cap as resolve_sglang_context_cap, - inject_sglang_context_length as inject_sglang_context_length, - _resolve_dual_chunk_backend as _resolve_dual_chunk_backend, - inject_sglang_attention_backend as inject_sglang_attention_backend, - _SGLANG_MOE_RUNNER_BACKEND_FLAG as _SGLANG_MOE_RUNNER_BACKEND_FLAG, - _SGLANG_MOE_RUNNER_BACKEND_RE as _SGLANG_MOE_RUNNER_BACKEND_RE, - moe_runner_requires_aiter as moe_runner_requires_aiter, -) -from ._grid_variant_filter import ( - resolve_skip_spec as resolve_skip_spec, - _parse_skip_spec as _parse_skip_spec, - _RE_CUDA_GRAPH_MAX_BS as _RE_CUDA_GRAPH_MAX_BS, - _MN_PARAMS_PRIORITY as _MN_PARAMS_PRIORITY, - _MN_BACKENDS_PRIORITY as _MN_BACKENDS_PRIORITY, - _mn_priority_index as _mn_priority_index, - reorder_grid_for_multi_node as reorder_grid_for_multi_node, - apply_multi_node_invalid_variants as apply_multi_node_invalid_variants, - apply_aiter_moe_pin_filter as apply_aiter_moe_pin_filter, - _COMPATIBILITY_FLAG_RULES as _COMPATIBILITY_FLAG_RULES, - _XDIT_ENV_BLACKLIST as _XDIT_ENV_BLACKLIST, - _XDIT_ENV_COMBO_BLACKLIST as _XDIT_ENV_COMBO_BLACKLIST, - xdit_blacklist_reason as xdit_blacklist_reason, - _detect_model_class as _detect_model_class, - apply_compatibility_filter as apply_compatibility_filter, - apply_user_skip_list as apply_user_skip_list, -) - - -log = logging.getLogger(__name__) - - -def _validate_magpie_python_override(value: str) -> str: - """Validate caller-supplied Magpie interpreter overrides before argv[0].""" - raw = str(value or "").strip() - if not raw: - return "" - if any(ch in raw for ch in ("\x00", "\n", "\r")): - raise ValueError("magpie_python contains control characters") - resolved = Path(raw) - if not resolved.is_absolute(): - found = shutil.which(raw) - if not found: - raise ValueError(f"magpie_python is not executable on PATH: {raw}") - resolved = Path(found) - if not resolved.is_file(): - raise ValueError(f"magpie_python is not a file: {raw}") - if not resolved.name.lower().startswith("python"): - raise ValueError(f"magpie_python must point to a Python interpreter, got: {raw}") - return str(resolved) - - -class _VariantAborted(Exception): - """A variant ended before it reached its benchmark, carrying the result that records why.""" - - def __init__(self, result: VariantResult) -> None: - super().__init__(result.error_class or "variant_aborted") - self.result = result - - -def _resolve_session_dir() -> Path: - """Resolve the active session_dir for executors that need an output root.""" - from hyperloom.inference_optimizer.session.paths import session_dir as _sd - - return _sd() - - -class SessionDirField: - """An executor's ``session_dir``, resolved on read rather than at construction. - - The executors here are module-level singletons built at import, which is - before the CLI pins the session. Resolving in ``__init__`` freezes them on - the workspace root that concurrent sessions share. - """ - - def __set_name__(self, owner: type, name: str) -> None: - self._slot = f"_{name}" - - def __get__(self, obj: Any, objtype: type | None = None) -> Any: - if obj is None: - return self - explicit = getattr(obj, self._slot, None) - return explicit if explicit is not None else _resolve_session_dir() - - def __set__(self, obj: Any, value: Path | str | None) -> None: - setattr(obj, self._slot, Path(value) if value else None) - - -# SKIP_VARIANTS: comma/whitespace patterns matched (exact or fnmatch) against ``GridVariant.name``. - - -# Env-flag capability probe: a serving env flag can be defined in the build yet still crash the server at engine init -# because the code path it activates imports a module the build did not package. -_UNSET = object() - -# Cached probe result keyed by framework. -_CAP_PROBE_CACHE: dict[str, str | None] = {} - -# Subprocess probe: locate the installed vLLM package via ``find_spec`` without importing it, read the aiter -# shared-expert router source, extract the ``fused_moe.*`` modules it imports, and verify each resolves to a real file -# in THIS build. -_AITER_SHARED_EXPERT_PROBE_SCRIPT = ( - "import importlib.util as u, os, re, json\n" - "def go():\n" - " try:\n" - " spec = u.find_spec('vllm')\n" - " except Exception:\n" - " return {'status': 'unknown'}\n" - " if not spec or not spec.origin:\n" - " return {'status': 'unknown'}\n" - " vdir = os.path.dirname(spec.origin)\n" - " router = os.path.join(vdir, 'model_executor', 'layers', 'fused_moe',\n" - " 'router', 'aiter_shared_routed_fused_moe_router.py')\n" - " if not os.path.exists(router):\n" - " return {'status': 'unknown'}\n" - " try:\n" - " text = open(router, encoding='utf-8').read()\n" - " except Exception:\n" - " return {'status': 'unknown'}\n" - " mods = re.findall(r'from\\s+(vllm\\.model_executor\\.layers\\.fused_moe\\.[A-Za-z0-9_.]+)\\s+import', text)\n" - " missing = []\n" - " for m in sorted(set(mods)):\n" - " rel = m[len('vllm.'):].replace('.', os.sep)\n" - " if not (os.path.exists(os.path.join(vdir, rel + '.py'))\n" - " or os.path.exists(os.path.join(vdir, rel, '__init__.py'))):\n" - " missing.append(m)\n" - " return {'status': 'unsupported', 'missing': missing} if missing else {'status': 'ok'}\n" - "print(json.dumps(go()))\n" -) - - -def _probe_vllm_aiter_shared_expert_unsupported() -> str | None: - """Return a drop reason if the installed vLLM build can't honour the aiter shared-expert fusion flag, else ``None``.""" - cached = _CAP_PROBE_CACHE.get("vllm", _UNSET) - if cached is not _UNSET: - return cached # type: ignore[return-value] - try: - proc = subprocess.run( - [_resolve_probe_python(), "-c", _AITER_SHARED_EXPERT_PROBE_SCRIPT], - capture_output=True, - text=True, - timeout=30, - ) - lines = [ln for ln in (proc.stdout or "").splitlines() if ln.strip()] - payload = json.loads(lines[-1]) if lines else {} - except Exception: # noqa: BLE001 - return None - status = payload.get("status") - if status == "ok": - _CAP_PROBE_CACHE["vllm"] = None - return None - if status == "unsupported": - missing = ", ".join(payload.get("missing") or []) or "(unknown module)" - reason = ( - "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS enabled but the installed " - "vLLM build's aiter shared-expert router imports missing module(s): " - f"{missing}. Flag unusable on this build (server crashes at engine " - "init); upgrade vLLM to a build that ships the module." - ) - _CAP_PROBE_CACHE["vllm"] = reason - return reason - return None # unknown => do not drop or cache - - -def unsupported_capability_reason(variant: "GridVariant") -> str | None: - """Return a drop reason if a variant sets an env flag the installed framework build cannot honour, else ``None``.""" - fw = (os.environ.get("FRAMEWORK", "") or "sglang").strip().lower() - if fw != "vllm": - return None - envs = {str(k): str(v) for k, v in (getattr(variant, "extra_envs", None) or {}).items()} - val = envs.get("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS") - if val is None or not is_truthy(val, default=True): - return None - return _probe_vllm_aiter_shared_expert_unsupported() - - -# Env vars whose value names a config file the serving stack reads unconditionally: aiter's tuned-GEMM loader skips -# its merge default and pd.read_csv's exactly the AITER_CONFIG_* path when one is set, so a missing file kills the -# server at config load instead of falling back to the default table. Values that are not file paths (e.g. geak's -# JSON-payload AITER_CONFIG) do not carry the trailing underscore and stay untouched. -_FILE_REFERENCING_ENV_PREFIXES: tuple[str, ...] = ("AITER_CONFIG_",) - - -def ungrounded_env_file_reason(variant: "GridVariant") -> str | None: - """Return a drop reason if an extra_envs value names a config file that does not exist, else ``None``. - - Grounds file-like env values the same way ``vet_patches`` grounds patch targets: a proposal may point a config - env at an artifact only a later (possibly rejected) authoring step would create, and the benchmark would boot a - server that dies at config load. The check is existence-only; anything already on disk (a promoted, durable - artifact) passes. A set value is a ``:``-joined list of paths (see ``csvs_aiter_will_load``), so every - non-empty element must exist, not only the first. - """ - for raw_key, raw_value in (getattr(variant, "extra_envs", None) or {}).items(): - key = str(raw_key or "").strip().upper() - if not key.startswith(_FILE_REFERENCING_ENV_PREFIXES): - continue - value = str(raw_value or "").strip() - if not value: - continue - missing = [p for p in (part.strip() for part in value.split(":")) if p and not Path(p).is_file()] - if missing: - return ( - f"{key} points at {value}, whose element(s) {':'.join(missing)} do not exist; the server would read it " - "unconditionally and die at config load. Point it at a file a stage authored " - "(e.g. a promoted tuned-GEMM CSV) or leave it unset to use the default config table." - ) - return None - - -# Sanitization for LLM-supplied overrides (benchmark_script / result_dir): reject path separators / shell -# metacharacters, raising ``ValueError`` instead of running an unsafe subprocess. -_SCRIPT_NAME_RE = re.compile(r"^[A-Za-z0-9_.\-]+\.sh$") -_RESULT_DIR_FORBID_RE = re.compile(r"[\s\"'`$;&|<>(){}\[\]\\*?!]") - - -def sanitize_script_name(value: Any) -> str | None: - """Return ``value`` if it's a safe Magpie benchmark script file name.""" - if value is None: - return None - text = str(value).strip() - if not text: - return None - if not _SCRIPT_NAME_RE.match(text): - raise ValueError( - f"benchmark_script={text!r} rejected: must be a bare *.sh " - "file name (no path separators, no shell metacharacters)" - ) - return text - - -def sanitize_result_dir(value: Any) -> str | None: - """Return ``value`` if it's a safe absolute (or workspace-relative) dir.""" - if value is None: - return None - text = str(value).strip() - if not text: - return None - if _RESULT_DIR_FORBID_RE.search(text): - raise ValueError( - f"result_dir={text!r} rejected: contains whitespace or shell " - "metacharacters; pass an absolute or workspace-relative path" - ) - return text - - -def _is_safe_path_entry(entry: str) -> bool: - """A single path entry is safe iff it has no ``:``, no ``..``, no control chars.""" - return ( - bool(entry) - and ":" not in entry - and ".." not in Path(entry).parts - and not any(c in entry for c in ("\n", "\r", "\x00")) - ) - - -# Reserved keys carried by dedicated override fields; never accepted via runtime_env. -_RUNTIME_ENV_RESERVED: frozenset[str] = frozenset({"PATH", "PYTHONPATH", "LD_LIBRARY_PATH"}) - - -def apply_runtime_override(envs: dict[str, str], override: dict[str, Any]) -> None: - """Inject an attempt runtime override into the materialized YAML envs dict.""" - if not override: - return - from ._workload_envs import prepend_launch_path - - path_prefix = str(override.get("path_prefix") or "").strip() - if path_prefix: - prepend_launch_path(envs, "PATH", path_prefix) - entrypoint_bin = str(override.get("entrypoint_bin_dir") or "").strip() - if entrypoint_bin: - if _is_safe_path_entry(entrypoint_bin): - prepend_launch_path(envs, "PATH", entrypoint_bin) - else: - log.warning("apply_runtime_override: dropping unsafe entrypoint_bin_dir %r", entrypoint_bin) - pp_prefix = str(override.get("pythonpath_prefix") or "").strip() - if pp_prefix: - if _is_safe_path_entry(pp_prefix): - prepend_launch_path(envs, "PYTHONPATH", pp_prefix) - else: - log.warning("apply_runtime_override: dropping unsafe pythonpath_prefix %r", pp_prefix) - # Multi-entry prefixes: prepend in order so the first entry wins, ahead of any single-dir pythonpath_prefix and - # the inherited value. - for entry in reversed(_coerce_prefix_list(override.get("pythonpath_prefixes"))): - if _is_safe_path_entry(entry): - prepend_launch_path(envs, "PYTHONPATH", entry) - else: - log.warning("apply_runtime_override: dropping unsafe pythonpath entry %r", entry) - # Native loader path: prepend attempt entries while preserving any inherited entries (e.g. /opt/rocm/lib). - for entry in reversed(_coerce_prefix_list(override.get("ld_library_path_prefix"))): - if _is_safe_path_entry(entry): - prepend_launch_path(envs, "LD_LIBRARY_PATH", entry) - else: - log.warning("apply_runtime_override: dropping unsafe ld_library_path entry %r", entry) - _runtime_env = override.get("runtime_env") - if isinstance(_runtime_env, dict): - for raw_k, raw_v in _runtime_env.items(): - key = str(raw_k) - if not _ENV_KEY_RE.match(key) or key in BLOCKED_CHILD_ENV_NAMES or key in _RUNTIME_ENV_RESERVED: - log.warning("apply_runtime_override: dropping unsafe runtime_env key %r", key) - continue - envs[key] = str(raw_v) - for key in ("framework_bin", "framework_python", "framework_venv_root"): - val = str(override.get(key) or "").strip() - if val: - envs[f"HYPERLOOM_{key.upper()}"] = val - # runtime_python_exe takes priority over framework_python as the launch interpreter. - rpe = str(override.get("runtime_python_exe") or "").strip() - if rpe: - envs["HYPERLOOM_FRAMEWORK_PYTHON"] = rpe - - -def _coerce_prefix_list(value: Any) -> list[str]: - """Normalize a runtime-prefix field (list/tuple/str) to a list of entries.""" - if value is None: - return [] - if isinstance(value, str): - return [value.strip()] if value.strip() else [] - if isinstance(value, (list, tuple)): - return [str(v).strip() for v in value if str(v).strip()] - return [] - - -def _build_variant_yaml( - base_yaml_path: Path, - base_extra_args: str, - variant: GridVariant, - *, - output_subdir: Path, - model_path: str | None = None, - gpu_type: str | None = None, - benchmark_script: str | None = None, - server_lifecycle: dict[str, Any] | None = None, - base_args_mode: str = "append", - base_extra_envs: dict[str, str] | None = None, - base_remove_args: list[str] | None = None, - base_unset_envs: list[str] | None = None, -) -> Path: - """Materialize a per-variant Magpie YAML on disk.""" - with base_yaml_path.open(encoding="utf-8") as f: - cfg = yaml.safe_load(f) - bench = cfg.setdefault("benchmark", {}) - replacing = str(base_args_mode).strip().lower() == "replace" - envs = apply_runtime_benchmark_overrides( - bench, - model_path=model_path, - gpu_type=gpu_type, - benchmark_script=benchmark_script, - conc=variant_conc(variant), - ) - extra_args_env = server_args_env_name(bench.get("framework")) - - variant_remove = to_str_list(getattr(variant, "remove_args", [])) - # A replacing base drops the inherited string wholesale, so only the - # variant's own removals still name flags that survive to be stripped. - effective_remove = ( - variant_remove if replacing else list(dict.fromkeys(to_str_list(base_remove_args) + variant_remove)) - ) - combined = compose_server_args( - inherited_args="" if replacing else str(envs.get(extra_args_env, "")), - base_extra_args=base_extra_args, - variant_extra_args=variant.extra_server_args, - remove_args=effective_remove, - args_mode=getattr(variant, "args_mode", "append"), - ) - # A grid variant never injects a MoE runner backend itself, but it does inherit one -- from the baseline recipe it - # was seeded with, or from an explicitly authored variant. - if combined and _SGLANG_MOE_RUNNER_BACKEND_RE.search(combined): - from ._workload_envs import _remove_moe_runner_backend_arg - - if extra_args_env == "EXTRA_SGLANG_ARGS" and moe_runner_requires_aiter(combined, model_path): - log.warning( - "grid: dropping inherited --moe-runner-backend for variant %s: " - "this checkpoint's MoE quant scheme is only implemented on the " - "aiter runner and would crash on the first forward pass.", - variant.name, - ) - combined = _remove_moe_runner_backend_arg(combined) - if combined: - envs[extra_args_env] = _shell_safe_dedupe(combined) - elif extra_args_env in envs or variant.args_mode == "replace" or base_args_mode == "replace": - envs[extra_args_env] = "" - # Composed base-then-variant, so a variant unsetting a key the base sets - # removes it: the last layer to name a key is the one that decides it. - for k in to_str_list(base_unset_envs): - if k.strip().upper() in BLOCKED_EXTERNAL_ENV_NAMES: - continue - envs.pop(k, None) - for k, v in (base_extra_envs or {}).items(): - envs[str(k)] = str(v) - for k in getattr(variant, "unset_envs", []) or []: - # Unsetting a pin retargets the benchmark rather than toggling a knob. - if str(k).strip().upper() in BLOCKED_EXTERNAL_ENV_NAMES: - log.warning("grid: refusing to unset pinned env %s for variant %s", k, variant.name) - continue - envs.pop(str(k), None) - for k, v in variant.extra_envs.items(): - envs[str(k)] = str(v) - from ._workload_envs import pin_mlperf_round_concurrency, prepend_launch_path - - pin_mlperf_round_concurrency(envs) - # The launcher re-exports these unconditionally, so a value carried here is - # one the run never used. - for k in launcher_overwritten_envs(bench) & envs.keys(): - log.warning("grid: dropping %s for variant %s; the recipe overwrites it", k, variant.name) - envs.pop(k, None) - # The three AgentX bounds took this rung's CONC through ``variant_conc`` above, not through this merge: raising - # the client's grace alone would make the round wait inside a cap that did not move with it. - _overlay = str(getattr(variant, "overlay_pythonpath", "") or "").strip() - if _overlay: - # Structural containment on the overlay dir before it is prepended to PYTHONPATH: a legitimate authored-kernel - # overlay is a single existing directory (never a ``:``-joined list, never a ``..`` traversal or control - # char). - _overlay_ok = ( - ":" not in _overlay - and ".." not in Path(_overlay).parts - and not any(c in _overlay for c in ("\n", "\r", "\x00")) - and Path(_overlay).is_dir() - ) - if _overlay_ok: - prepend_launch_path(envs, "PYTHONPATH", _overlay) - else: - log.warning( - "grid: dropping unsafe overlay_pythonpath %r (not a single " - "existing directory / contains separator or traversal)", - _overlay, - ) - - # Attempt runtime override: inject path_prefix/pythonpath_prefix/framework_bin etc. into benchmark.envs so the - # server subprocess resolves the attempt runtime. - _rt_override = getattr(variant, "runtime_override", None) or {} - if _rt_override: - apply_runtime_override(envs, _rt_override) - - # PATH guard: the xdit wrapper needs both `/venv/bin` (the `xdit` console script) and `/opt/rocm/bin` (`hipcc`); - # force-prepend both so an LLM-supplied PATH can't drop one. - if str(bench.get("framework", "")).strip().lower() == "xdit": - _cur_path = str(envs.get("PATH", "") or "") - _parts = [p for p in _cur_path.split(":") if p] - for _essential in ("/opt/rocm/bin", "/venv/bin"): - if _essential not in _parts: - _parts.insert(0, _essential) - envs["PATH"] = ":".join(_parts) - - if server_lifecycle is not None: - from ._server_lifecycle import inject_lifecycle - - inject_lifecycle( - bench, - cleanup=bool(server_lifecycle.get("cleanup", True)), - pid_dir=server_lifecycle["pid_dir"], - port=int(server_lifecycle["port"]), - ) - - # The final write to the argument env; nothing below may touch it. - seal_server_argv(envs, bench.get("framework")) - output_subdir.mkdir(parents=True, exist_ok=True) - out_path = output_subdir / "config.yaml" - with out_path.open("w", encoding="utf-8") as f: - yaml.safe_dump(cfg, f, sort_keys=False) - return out_path - - -def _parse_report(workspace: Path) -> dict[str, Any] | None: - """Load ``benchmark_report.json`` from a benchmark workspace.""" - report = workspace / "benchmark_report.json" - if not report.exists(): - return None - try: - with report.open(encoding="utf-8") as f: - data = json.load(f) - except (OSError, json.JSONDecodeError): - return None - return data if isinstance(data, dict) else None - - -# How long to keep re-reading ``benchmark_report.json`` when the report does not yet parse into a valid measurement. -REPORT_SETTLE_SECONDS = 30.0 -REPORT_SETTLE_POLL_SECONDS = 1.0 - - -async def _settled_measurement( - workspace: Path, - *, - subprocess_started_unix: float | None, - settle_seconds: float = REPORT_SETTLE_SECONDS, - poll_seconds: float = REPORT_SETTLE_POLL_SECONDS, -) -> tuple[dict[str, Any] | None, dict[str, Any]]: - """Read the benchmark report, re-reading briefly while it is still settling.""" - deadline = time.monotonic() + max(0.0, float(settle_seconds)) - attempts = 0 - while True: - report = _parse_report(workspace) - measurement = extract_benchmark_measurement( - report, - workspace=workspace, - subprocess_started_unix=subprocess_started_unix, - ) - attempts += 1 - if measurement.get("valid_measurement") or time.monotonic() >= deadline: - if attempts > 1 and measurement.get("valid_measurement"): - log.info( - "grid_runner: benchmark_report became valid after %d read(s); " - "the report was still being written when the subprocess was reaped", - attempts, - ) - # Harvest already tried this, but it runs before the report is guaranteed to exist: a report Magpie - # finishes writing after the subprocess is reaped would otherwise leave the round with no GPU artifact at - # all. Here the report is in hand, so write it from that rather than reading the file a second time. - if report is not None: - write_gpu_metrics_from_report(workspace, report, source="benchmark_report.json") - return report, measurement - await asyncio.sleep(max(0.01, float(poll_seconds))) - - -def _run_grid_warmup_enabled() -> bool: - """Whether ``run_grid`` should discard a cold warmup round when possible.""" - return env_flag("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", default=not os.environ.get("PYTEST_CURRENT_TEST")) - - -def _prepend_magpie_pythonpath(magpie_dir: str, current_pythonpath: str) -> str: - """Prepend Magpie's import root to PYTHONPATH, skipping package-root dirs.""" - if not magpie_dir or is_python_package_root(magpie_dir): - return current_pythonpath - return f"{magpie_dir}:{current_pythonpath}" if current_pythonpath else magpie_dir - - -def sync_benchmark_timeout(config_path: Path, timeout_sec: float) -> None: - """Give Magpie and bypass the same cap as the enclosing benchmark process.""" - cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) - bench = cfg["benchmark"] - bench["timeout_seconds"] = timeout_sec - if bench.get("server_lifecycle"): - bench["server_lifecycle"]["server_ready_timeout_s"] = timeout_sec - envs = bench.setdefault("envs", {}) - envs["PYTHONUNBUFFERED"] = "1" - sync_agentx_phase_wait(envs, timeout_sec) - config_path.write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8") - - -def _benchmark_server_log(config_path: Path, output_dir: Path) -> str | None: - """Scriptable benchmarks have no server whose readiness can arm silence.""" - cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) - framework = str(cfg["benchmark"].get("framework", "")).lower() - return None if framework in {"custom", "xdit"} else str(output_dir / "server.log") - - -def _run_magpie( - *, - magpie_python: str, - config_path: Path, - output_dir: Path, - timeout_sec: int, - cwd: str, - result_dir: str | None = None, - silence_timeout_sec: float | None = None, - server_already_ready: bool = False, - serving_lease: Any = None, - on_output: Callable[[], None] | None = None, - session_deadline_sec: float | None = None, - unset_envs: list[str] | None = None, -) -> tuple[int, str, str]: - """Blocking subprocess wrapper. Returns (rc, stdout, stderr).""" - sync_benchmark_timeout(config_path, timeout_sec) - server_log_path = _benchmark_server_log(config_path, output_dir) - env = scrub_benchmark_process_env(os.environ.copy()) - env["PYTHONUNBUFFERED"] = "1" - from ._workload_envs import resolve_reference_launch - - _args, _envs, reference_controls = resolve_reference_launch() - for name in [*(reference_controls.get("unset_envs") or []), *(unset_envs or [])]: - if name.strip().upper() not in BLOCKED_EXTERNAL_ENV_NAMES: - env.pop(name, None) - env["PATH"] = f"/opt/venv/bin:{env.get('PATH', '')}" - magpie_dir = os.environ.get("MAGPIE_PATH") or "" - if magpie_dir: - env["PYTHONPATH"] = _prepend_magpie_pythonpath(magpie_dir, env.get("PYTHONPATH", "")) - - # Multi-node: tell Magpie to skip its local-server launch and point benchmark_serving at the head pod's ClusterIP. - from ._multi_node_env import magpie_remote_env - - env.update(magpie_remote_env()) - - # Pin Magpie's InferenceX resolution to ``$INFERENCEX_PATH`` (its highest-precedence rung) so it loads the patched - # checkout, not a stale copy. - inferencex_path = os.environ.get("INFERENCEX_PATH", "").strip() - if inferencex_path: - env["MAGPIE_INFERENCEX_PATH"] = inferencex_path - # Baseline patches its own checkout, but explore / sweep never pass through that hook: re-assert here so a - # resumed session or a re-cloned checkout still emits the eval-start marker. - ensure_benchmark_lib_eval_start_patched(Path(inferencex_path)) - - # The generation bounds + pathology probe are asserted whether or not ``$INFERENCEX_PATH`` is set: unset falls - # back to the same env discovery the baseline arm uses ($MAGPIE_PATH/InferenceX). - probe_root = Path(inferencex_path) if inferencex_path else None - # Best-effort, unlike the probe below: a missing guard only costs the reason a failed round reports. - ensure_eval_unbound_outputs_patched(probe_root) - if not ensure_eval_probe_patched(probe_root) and not materialized_run_eval_disabled(config_path): - eval_bounds_msg = ( - "eval generation bounds + pathology probe are not installed " - "(utils/evals/patches/lm_eval_sitecustomize.py, inferencex=" - f"{inferencex_path or ''}, INFERENCEX_PATH=" - f"{os.environ.get('INFERENCEX_PATH', '') or ''}); this " - "variant runs eval, so it would be scored against a baseline that " - "truncated at a different point" - ) - if eval_probe_targets_exist(probe_root): - log.error("grid_runner: %s; failing this benchmark", eval_bounds_msg) - return (EVAL_PROBE_UNPATCHABLE_RETURNCODE, "", eval_bounds_msg) - log.warning("grid_runner: %s", eval_bounds_msg) - # AgentX: deploy the aiperf client into InferenceX ``benchmarks/`` + preflight aiperf right before Magpie runs it, - # via the shared helper (also used by the baseline/profile shell-out). - from ._workload_envs import prepare_agentx_runtime - - _agx_err = prepare_agentx_runtime(env=env, inferencex_path=inferencex_path, config_path=config_path) - if _agx_err: - log.error("%s; failing this benchmark", _agx_err) - return (AGENTX_PREFLIGHT_RETURNCODE, "", _agx_err) - # RESULT_DIR default; leaks are picked up by the salvage path. - env["RESULT_DIR"] = result_dir or str(output_dir) - # InferenceX ``run_lm_eval`` cleans ``$EVAL_RESULT_DIR`` after processing lm-eval output. - env["EVAL_RESULT_DIR"] = str(Path(env["RESULT_DIR"]) / "eval_output") - # Pin SERVER_LOG / GPU_METRICS_CSV per-task so logs land alongside ``benchmark_report.json``. - env["SERVER_LOG"] = str(output_dir / "server.log") - env["GPU_METRICS_CSV"] = str(output_dir / "gpu_metrics.csv") - - # Ray-managed GPU execution (§12 T1): route the round through the serving lease's actor, which holds ``num_gpus`` - # across every round sharing this server. - if serving_lease is not None: - from ._ray_backend import strip_visible_devices_from_config - - ray_config_path = strip_visible_devices_from_config(config_path) - cmd = build_benchmark_command( - python_exe=magpie_python, - config_path=ray_config_path, - output_dir=output_dir, - ) - # ``on_output`` cannot follow the round here: the benchmark runs inside a Ray actor in another process - # (potentially on another node) and only its final ``(rc, stdout, stderr)`` comes back, so there is nothing - # local to call per line. - return serving_lease.run_session_kill( - cmd, - env=env, - cwd=cwd, - timeout=timeout_sec, - silence_timeout_sec=silence_timeout_sec, - server_log_path=server_log_path, - server_already_ready=server_already_ready, - # Converted here, at the last moment before the process boundary: the actor cannot read this process's - # monotonic clock. - session_remaining_sec=session_deadline_to_remaining_sec(session_deadline_sec), - ) - - cmd = build_benchmark_command( - python_exe=magpie_python, - config_path=config_path, - output_dir=output_dir, - ) - # The launch puts Magpie in its own POSIX session and tears down the whole - # descendant tree on every exit path. - proc = run_with_session_kill( - cmd, - env=env, - cwd=cwd, - timeout=timeout_sec, - silence_timeout_sec=silence_timeout_sec, - server_log_path=server_log_path, - server_already_ready=server_already_ready, - on_output=on_output, - session_deadline_sec=session_deadline_sec, - ) - return proc.returncode, proc.stdout or "", proc.stderr or "" - - -def _num_gpus_for_config(config_path: Path) -> float: - """Read the tensor-parallel size (``TP``) from a materialized benchmark YAML.""" - try: - with Path(config_path).open(encoding="utf-8") as fp: - cfg = yaml.safe_load(fp) or {} - envs = (cfg.get("benchmark") or {}).get("envs") or {} - return float(int(envs.get("TP", 1) or 1)) - except Exception: # noqa: BLE001 — best-effort; default to 1 GPU - return 1.0 - - -def _mn_restart_env( - reference_envs: dict[str, str], - variant: Any, - unset_envs: list[str], -) -> dict[str, str]: - """Merge the reference server envs under one variant's own overrides.""" - merged = {k: v for k, v in reference_envs.items() if k not in unset_envs} - merged.update({str(k): str(v) for k, v in (variant.extra_envs or {}).items()}) - return merged - - -def _resolve_mn_effective_server_args( - cfg_path: Path, - base_yaml_path: Path, - variant: Any, - *, - base_extra_args: str, - base_args_mode: str, -) -> str: - """Resolve the multi-node server args for a variant restart; prefer the materialized variant YAML, falling back to a recompose from the base YAML.""" - try: - with cfg_path.open(encoding="utf-8") as _f: - _variant_cfg = yaml.safe_load(_f) or {} - _variant_bench = _variant_cfg.get("benchmark") or {} - _variant_envs = _variant_bench.get("envs") or {} - _variant_framework_env = server_args_env_name(_variant_bench.get("framework")) - return str(_variant_envs.get(_variant_framework_env) or "") - except Exception: - log.debug( - "grid_runner: failed to read materialized variant args from %s", - cfg_path, - exc_info=True, - ) - try: - with base_yaml_path.open(encoding="utf-8") as _f: - _base_cfg = yaml.safe_load(_f) or {} - _base_bench = _base_cfg.get("benchmark") or {} - _base_envs = _base_bench.get("envs") or {} - _base_framework_env = server_args_env_name(_base_bench.get("framework")) - _fallback_inherited_args = str(_base_envs.get(_base_framework_env) or "") - except Exception: # noqa: BLE001 - best-effort parity fallback - _fallback_inherited_args = "" - return _shell_safe_dedupe( - compose_server_args( - inherited_args="" if str(base_args_mode).strip().lower() == "replace" else _fallback_inherited_args, - base_extra_args=base_extra_args, - variant_extra_args=variant.extra_server_args, - remove_args=getattr(variant, "remove_args", []), - args_mode=getattr(variant, "args_mode", "append"), - ) - ) - - -def _variant_progress_note( - grid: list[GridVariant], - results: list[VariantResult], - idx: int, -) -> dict[str, Any]: - """Build the progress note for the variant at ``idx`` from that variant's own row.""" - landed = results[idx] if idx < len(results) else None - return { - "unit": "variant", - "label": grid[idx].name, - "index": idx + 1, - "total": len(grid), - "status": getattr(landed, "status", None), - "output_throughput": getattr(landed, "output_throughput", None), - } - - -# The returncode side of :mod:`...stop_attribution`: the same two causes, keyed by the sentinel a subprocess comes -# back with. -_STOPPED_BY_THE_RUN: dict[int, StoppedByTheRun] = { - SESSION_TIME_EXHAUSTED_RETURNCODE: STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], - ORCHESTRATOR_CANCELLED_RETURNCODE: STOPPED_BY_THE_RUN[ORCHESTRATOR_CANCELLED_CLASS], -} - - -def stopped_by_the_run(returncode: int | None) -> StoppedByTheRun | None: - """Return how to record a round the run itself stopped, if it did.""" - if returncode is None: - return None - return _STOPPED_BY_THE_RUN.get(int(returncode)) - - -def variant_conc(variant: Any) -> int | None: - """The concurrency a variant will run at, or ``None`` when it names none.""" - raw = (getattr(variant, "extra_envs", None) or {}).get("CONC") - try: - conc = int(raw) - except (TypeError, ValueError): - return None - return conc if conc > 0 else None - - -def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: - """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call.""" - if shared_state is None: - return (None, None) - deadline_fn = getattr(shared_state, "grid_session_deadline_sec", None) - deadline = deadline_fn() if callable(deadline_fn) else None - variant_sec = _phase_state.one_more_measurement_sec(shared_state) - if variant_sec is None: - variant_sec = _phase_state.measured_seconds(shared_state, "baseline_runtime_sec") - return (deadline, variant_sec) - - -async def run_grid( - *, - base_yaml_path: Path, - base_extra_args: str, - grid: list[GridVariant], - output_root: Path, - magpie_python: str | None = None, - keep_going_on_failure: bool = True, - model_path: str | None = None, - gpu_type: str | None = None, - benchmark_script: str | None = None, - result_dir: str | None = None, - server_lifecycle: dict[str, Any] | None = None, - base_args_mode: str = "append", - base_extra_envs: dict[str, str] | None = None, - base_remove_args: list[str] | None = None, - base_unset_envs: list[str] | None = None, - warmup_before_measure: bool | None = None, - server_already_ready: bool = False, - ready_server_log: str | None = None, - serving_lease: Any = None, - session_deadline_sec: float | None = None, - variant_expected_sec: float | None = None, - deadline_stop: StoppedByTheRun = STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], - lifecycle_boot_only: bool = False, -) -> list[VariantResult]: - """Execute variants; ``deadline_stop`` names the owner of the supplied deadline. - - ``ready_server_log`` is the log of the caller's ready server, measured when no variant log exists. - """ - silence_timeout_sec, benchmark_timeout_sec = resolve_benchmark_timeouts() - if not magpie_python: - # Backend-aware: bypass uses a plain python3, not Magpie's venv. - from .benchmark_backend import resolve_benchmark_interpreter - - magpie_python = resolve_benchmark_interpreter() - else: - magpie_python = _validate_magpie_python_override(magpie_python) - if warmup_before_measure is None: - warmup_before_measure = _run_grid_warmup_enabled() - auto_warmup_requested = bool(warmup_before_measure and server_lifecycle is None) - results: list[VariantResult] = [] - # This function names the working directory, so it creates it: callers and the per-variant config writer both - # happen to create it first today, and neither is a contract. - output_root.mkdir(parents=True, exist_ok=True) - cwd = str(output_root) - - # Reap orphaned aiter JIT build locks before booting any server. - try: - from ._aiter_jit import sweep_stale_aiter_locks_if_dead - - _lock_sweep = sweep_stale_aiter_locks_if_dead() - if _lock_sweep.get("deleted"): - log.warning( - "grid_runner: reaped %d orphaned aiter JIT lock(s) under %s before server launch (compiler_alive=%s)", - _lock_sweep.get("deleted"), - _lock_sweep.get("dir"), - _lock_sweep.get("compiler_alive"), - ) - except Exception as exc: # noqa: BLE001 — best-effort hygiene, never blocks the grid - log.debug("grid_runner: aiter lock sweep swallowed: %r", exc) - - # Multi-node: the reference recipe's server envs do not reach the variant restart through - # restart_server_for_round, which never reads them. - from ._multi_node_env import is_multi_node as _mn_is_multi_node - - _mn_ref_envs: dict[str, str] = {} - if _mn_is_multi_node(): - from ._workload_envs import resolve_reference_base - - _, _mn_ref_envs = resolve_reference_base() - - # Reported on entry, not on completion: ``_report_finished_variant`` only runs once a result has been appended, so - # a first variant that hangs — or a branch that raises before reaching it — would emit nothing at all, which is - # exactly the silence the heartbeat exists to break. - async def _unit_started(idx: int, label: str) -> None: - """Report that a unit of variant ``idx`` is about to start.""" - await report_progress( - unit="variant_step", - label=f"{grid[idx].name}:{label}", - index=idx + 1, - total=len(grid), - status="started", - ) - - async def _reported_magpie(idx: int, label: str, **kwargs: Any) -> tuple[int, str, str]: - """Run one Magpie pass, announced on entry and kept alive by its output.""" - await _unit_started(idx, label) - async with heartbeat_while_output_flows( - unit="variant_step", - label=f"{grid[idx].name}:{label}", - index=idx + 1, - total=len(grid), - ) as activity: - return await asyncio.to_thread( - _run_magpie, on_output=activity.note, unset_envs=grid[idx].unset_envs, **kwargs - ) - - # Variant boundary: a progress heartbeat so a grid that runs for hours is distinguishable from one that hung on - # its first variant. - async def _report_finished_variant(idx: int) -> None: - """Report the variant that just landed.""" - await report_progress(**_variant_progress_note(grid, results, idx)) - - def _record_round_stop( - stopped: StoppedByTheRun, - *, - idx: int, - variant: GridVariant, - slot: Path, - round_label: str, - returncode: int | None, - started_unix: float, - server_log: Path, - server_survivors: list[int], - ) -> bool: - """Record a round the run stopped and say whether the grid is over. - - ``server_survivors`` are the servers the variant booted that outlived its teardown. - """ - if returncode == SESSION_TIME_EXHAUSTED_RETURNCODE: - stopped = deadline_stop - runtime_sec = round(max(0.0, time.time() - started_unix), 2) - log.warning( - "grid_runner: variant %d/%d name=%s %s round reaped after %.1fs: %s; recorded as skipped, not failed", - idx + 1, - len(grid), - variant.name, - round_label, - runtime_sec, - stopped.interrupted, - ) - teardown = "tree reaped" - if server_survivors: - teardown += f", but server pid(s) {', '.join(map(str, server_survivors))} survived SIGKILL" - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class=stopped.error_class, - error_summary=f"{stopped.interrupted}; {teardown}", - extra_args=variant.extra_server_args, - ) - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="skipped", - returncode=returncode, - runtime_sec=runtime_sec, - error=stopped.interrupted, - error_class=stopped.error_class, - server_log_path=_measurement_server_log_path(server_log, slot=slot), - note=variant.note, - nonfatal_warnings=[f"server_survived_teardown:{pid}" for pid in server_survivors], - ) - ) - if stopped.ends_the_batch: - results.extend(_not_run_skip_result(rest, stopped) for rest in grid[idx + 1 :]) - return True - return not keep_going_on_failure - - # How many full benchmark passes one variant costs. - _mn_warmup_rounds = 0 - if variant_expected_sec is not None: - from ._multi_node_env import ( - is_multi_node as _mn_is_multi_node, - mn_bench_warmup_enabled as _mn_bench_warmup_enabled, - ) - - _mn_warmup_rounds = 1 if (_mn_is_multi_node() and _mn_bench_warmup_enabled()) else 0 - variant_rounds = 1 + (1 if auto_warmup_requested else 0) + _mn_warmup_rounds - - def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant_rounds) -> bool: - """Skip variant ``idx`` and every one after it, or say the budget still fits.""" - if session_deadline_sec is None: - return False - remaining_sec = session_deadline_sec - time.monotonic() - required_sec = float(variant_expected_sec) * rounds_left if variant_expected_sec is not None else None - if remaining_sec > 0 and (required_sec is None or remaining_sec >= required_sec): - return False - log.warning( - "grid_runner: %.0fs left cannot admit this variant's %d remaining round(s) " - "(expected_sec=%s, spent on: %s); skipping %d remaining variant(s)", - max(0.0, remaining_sec), - rounds_left, - required_sec, - spent_on, - len(grid) - idx, - ) - results.extend(_not_run_skip_result(variant, deadline_stop) for variant in grid[idx:]) - return True - - for i, variant in enumerate(grid): - try: - # Session-budget stop: skip the remaining variants once the wall-clock deadline is reached or the remaining - # budget cannot fit another variant, so a timeout halts the grid instead of draining it (and the last variant - # cannot overrun the close window). - if _skip_rest_for_budget(i, spent_on="the variants before this one"): - break - await _unit_started(i, "variant") - slot = output_root / f"variant_{i:02d}_{_safe(variant.name)}" - server_log = slot / "server.log" - # Capability fast-fail: drop a variant whose env flag the build cannot honour before booting a doomed server, - # still recording the failure so the LLM learns not to re-pick it. - cap_reason = unsupported_capability_reason(variant) or ungrounded_env_file_reason(variant) - if cap_reason: - log.warning( - "grid_runner: variant %d/%d name=%s aborted: capability_unsupported: %s", - i + 1, - len(grid), - variant.name, - cap_reason, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="capability_unsupported", - error_summary=cap_reason, - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=cap_reason, - error_class="capability_unsupported", - note=variant.note, - ) - ) - lifecycle: dict[str, Any] = {"eligible": False} - auto_warmup = False - try: - cfg_path = _build_variant_yaml( - base_yaml_path, - base_extra_args, - variant, - output_subdir=slot, - model_path=model_path, - gpu_type=gpu_type, - benchmark_script=benchmark_script, - server_lifecycle=server_lifecycle, - base_args_mode=base_args_mode, - base_extra_envs=base_extra_envs, - base_remove_args=base_remove_args, - base_unset_envs=base_unset_envs, - ) - except Exception as exc: # noqa: BLE001 - build_error = "yaml_build_error" - log.warning( - "grid_runner: variant %d/%d name=%s aborted: %s: %r", - i + 1, - len(grid), - variant.name, - build_error, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class=build_error, - error_summary=repr(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"{build_error}: {exc!r}", - error_class=build_error, - note=variant.note, - ) - ) - - _mn_effective_args = _resolve_mn_effective_server_args( - cfg_path, - base_yaml_path, - variant, - base_extra_args=base_extra_args, - base_args_mode=base_args_mode, - ) - - if auto_warmup_requested: - try: - from ._server_lifecycle import resolve_lifecycle_params - - lifecycle = resolve_lifecycle_params(cfg_path) - auto_warmup = bool(lifecycle.get("eligible")) - if auto_warmup: - cfg_path = _build_variant_yaml( - base_yaml_path, - base_extra_args, - variant, - output_subdir=slot, - model_path=model_path, - gpu_type=gpu_type, - benchmark_script=benchmark_script, - server_lifecycle={ - "cleanup": True, - "pid_dir": str(slot), - "port": int(lifecycle.get("port") or 0), - }, - base_args_mode=base_args_mode, - base_extra_envs=base_extra_envs, - base_remove_args=base_remove_args, - base_unset_envs=base_unset_envs, - ) - else: - log.info( - "grid_runner: warmup-before-measure not eligible (%s); running single measured round.", - lifecycle.get("reason") or "unknown", - ) - except Exception as exc: # noqa: BLE001 - log.warning( - "grid_runner: warmup-before-measure eligibility/materialization failed (%r); running single measured round.", - exc, - ) - auto_warmup = False - - warmup_tput: float | None = None - if auto_warmup: - warmup_slot = slot / "warmup_round" - warmup_server_log = warmup_slot / "server.log" - warmup_lifecycle = { - "cleanup": False, - "pid_dir": str(slot), - "port": int(lifecycle.get("port") or 0), - } - try: - warmup_cfg_path = _build_variant_yaml( - base_yaml_path, - base_extra_args, - variant, - output_subdir=warmup_slot, - model_path=model_path, - gpu_type=gpu_type, - benchmark_script=benchmark_script, - server_lifecycle=warmup_lifecycle, - base_args_mode=base_args_mode, - base_extra_envs=base_extra_envs, - base_remove_args=base_remove_args, - base_unset_envs=base_unset_envs, - ) - except Exception as exc: # noqa: BLE001 - log.warning( - "grid_runner: variant %d/%d name=%s aborted: warmup_yaml_build_error: %r", - i + 1, - len(grid), - variant.name, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="warmup_yaml_build_error", - error_summary=repr(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"warmup_yaml_build_error: {exc!r}", - error_class="warmup_yaml_build_error", - note=variant.note, - ) - ) - - warmup_workspaces_before = snapshot_workspaces(warmup_slot) - warmup_started_unix = time.time() - # Held in a local because the abort line below has to name the cap the round was actually granted: the - # declared one is a hang backstop, and a round killed at the reserved cap logged as a two-hour timeout - # reads as a variant that hangs rather than a budget that ran out. - warmup_cap_sec = benchmark_timeout_sec - try: - warmup_rc, warmup_stdout, warmup_stderr = await _reported_magpie( - i, - "warmup", - magpie_python=magpie_python, - config_path=warmup_cfg_path, - output_dir=warmup_slot, - timeout_sec=warmup_cap_sec, - silence_timeout_sec=silence_timeout_sec, - cwd=cwd, - result_dir=result_dir, - serving_lease=serving_lease, - session_deadline_sec=session_deadline_sec, - ) - except subprocess.TimeoutExpired as exc: - _teardown_variant_server(slot, lifecycle) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: warmup timeout (timeout_sec=%d): %s", - i + 1, - len(grid), - variant.name, - warmup_cap_sec, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="warmup_magpie_timeout", - error_summary=str(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"warmup_timeout: {exc}", - error_class="warmup_magpie_timeout", - server_log_path=_existing_log_path(warmup_server_log), - note=variant.note, - runtime_sec=round(max(0.0, time.time() - warmup_started_unix), 2), - nonfatal_warnings=["run_grid_warmup_round_failed"], - ) - ) - - warmup_stopped = stopped_by_the_run(warmup_rc) - if warmup_stopped is not None: - warmup_survivors = _teardown_variant_server(slot, lifecycle) - grid_is_over = _record_round_stop( - warmup_stopped, - idx=i, - variant=variant, - slot=slot, - round_label="warmup", - returncode=warmup_rc, - started_unix=warmup_started_unix, - server_log=warmup_server_log, - server_survivors=warmup_survivors, - ) - await _report_finished_variant(i) - if grid_is_over: - break - continue - - warmup_run_ws = select_run_workspace(warmup_slot, known_before=warmup_workspaces_before) - warmup_workspace = warmup_run_ws if warmup_run_ws is not None else warmup_slot - warmup_harvested = harvest_leaked_artifacts( - warmup_workspace, - subprocess_started_unix=warmup_started_unix, - ) - if warmup_run_ws is not None: - _, warmup_measurement = await _settled_measurement( - warmup_workspace, - subprocess_started_unix=warmup_started_unix, - settle_seconds=REPORT_SETTLE_SECONDS if warmup_rc == 0 else 0.0, - ) - else: - warmup_measurement = extract_benchmark_measurement( - None, - workspace=warmup_workspace, - subprocess_started_unix=warmup_started_unix, - ) - if warmup_rc != 0 or not warmup_measurement.get("valid_measurement"): - _teardown_variant_server(slot, lifecycle) - warmup_error = ( - server_log_death_excerpt(str(warmup_server_log)) - or redact_secret_values((warmup_stderr or warmup_stdout)[-2000:]) - if warmup_rc != 0 - else "warmup benchmark_report missing valid throughput/completed requests" - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: warmup_round_failed (rc=%s): %s", - i + 1, - len(grid), - variant.name, - warmup_rc, - warmup_error[:200], - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="warmup_round_failed", - error_summary=warmup_error, - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - workspace=str(warmup_workspace) if warmup_run_ws is not None else None, - report_path=( - str(warmup_workspace / "benchmark_report.json") - if (warmup_workspace / "benchmark_report.json").exists() - else None - ), - raw_result_path=warmup_measurement.get("raw_result_path"), - reported_success=warmup_measurement.get("reported_success"), - returncode=warmup_rc, - error=warmup_error, - error_class="warmup_round_failed", - server_log_path=_existing_log_path(warmup_server_log), - note=variant.note, - runtime_sec=round(max(0.0, time.time() - warmup_started_unix), 2), - nonfatal_warnings=[ - "run_grid_warmup_round_failed", - *[f"harvested_leaked_artifact:{src}" for src, _ in warmup_harvested], - ], - ) - ) - warmup_tput = warmup_measurement.get("output_throughput") - log.info( - "grid_runner: variant %s warmup tput=%.1f tok/s discarded; measuring hot round next", - variant.name, - warmup_tput or 0.0, - ) - - from ._multi_node_env import log_mn_banner - - log_mn_banner( - "grid_runner", - log, - variant=f"{i + 1}/{len(grid)}:{variant.name}", - ) - log.info( - "grid_runner: variant %d/%d name=%s args=%s", - i + 1, - len(grid), - variant.name, - variant.extra_server_args, - ) - - # Multi-node only: restart sglang/vllm with this variant's flags so each row runs against a fresh server. - from ._multi_node_server_lifecycle import ( - ServerRestartFailed, - restart_server_for_round, - ) - - try: - # PD knobs auto-resolved from $PD_* env; PD config stays constant across variants within one run. - _variant_unset = [str(k) for k in getattr(variant, "unset_envs", []) or [] if str(k).strip()] - _restart_env = _mn_restart_env(_mn_ref_envs, variant, _variant_unset) - await restart_server_for_round( - extra_server_args=_mn_effective_args, - # Reference recipe envs under this variant's own overrides (e.g. MORI_* MoE-dispatch tuning) so - # server-side env knobs proposed by specialists take effect on the restarted sglang without dropping - # the envs the baseline was measured with. - extra_env=_restart_env, - unset_env=_variant_unset, - model_path=model_path, - ep=int(os.environ.get("EP") or 0) or None, - ) - except ServerRestartFailed as exc: - log.warning( - "grid_runner: variant %d/%d name=%s aborted: mn_server_restart_failed: %s", - i + 1, - len(grid), - variant.name, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="mn_server_restart_failed", - error_summary=str(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"mn_server_restart_failed: {exc}", - error_class="mn_server_restart_failed", - note=variant.note, - ) - ) - - # The restart above is a launch of its own and is under no cap, so the budget admitted at the top of the loop - # may no longer be there. - if _skip_rest_for_budget( - i, - spent_on="this variant's server restart", - rounds_left=1 + _mn_warmup_rounds, - ): - if auto_warmup: - _teardown_variant_server(slot, lifecycle) - break - - # Multi-node client warmup: one discarded benchmark pass against the just-restarted, persistent remote server - # to warm JIT / steady-state before the measured pass. - from ._multi_node_env import ( - is_multi_node as _mn_imn, - mn_bench_warmup_enabled as _mn_warm, - ) - - if _mn_imn() and _mn_warm(): - _mn_warm_slot = slot / "mn_warmup" - _mn_warm_started_unix = time.time() - # The measurement is discarded, but the returncode is not: a warmup the run stopped is the same stop as - # one in the measured round, and discarding it launches the measured round after the cancel. - _mn_warm_rc: int | None = None - _mn_warm_rc, _, _ = await _reported_magpie( - i, - "mn_warmup", - magpie_python=magpie_python, - config_path=cfg_path, - output_dir=_mn_warm_slot, - timeout_sec=benchmark_timeout_sec, - silence_timeout_sec=silence_timeout_sec, - server_already_ready=True, - cwd=cwd, - result_dir=None, - serving_lease=serving_lease, - session_deadline_sec=session_deadline_sec, - ) - log.info( - "grid_runner: MN warmup pass done (discarded) %d/%d name=%s rc=%s", - i + 1, - len(grid), - variant.name, - _mn_warm_rc, - ) - _mn_warm_stopped = stopped_by_the_run(_mn_warm_rc) - if _mn_warm_stopped is not None: - _mn_warm_survivors = _teardown_variant_server(slot, lifecycle if auto_warmup else None) - grid_is_over = _record_round_stop( - _mn_warm_stopped, - idx=i, - variant=variant, - slot=slot, - round_label="mn_warmup", - returncode=_mn_warm_rc, - started_unix=_mn_warm_started_unix, - server_log=_mn_warm_slot / "server.log", - server_survivors=_mn_warm_survivors, - ) - await _report_finished_variant(i) - if grid_is_over: - break - continue - - # Snapshot wall-clock before launch so the salvage path can mtime-gate leak destinations per-variant. - slot_workspaces_before = snapshot_workspaces(slot) - variant_started_unix = time.time() - measure_cap_sec = benchmark_timeout_sec - try: - rc, stdout, stderr = await _reported_magpie( - i, - "benchmark", - magpie_python=magpie_python, - config_path=cfg_path, - output_dir=slot, - timeout_sec=measure_cap_sec, - cwd=cwd, - result_dir=result_dir, - silence_timeout_sec=silence_timeout_sec, - server_already_ready=(server_already_ready or auto_warmup or _mn_imn()), - serving_lease=serving_lease, - session_deadline_sec=session_deadline_sec, - ) - except subprocess.TimeoutExpired as exc: - # Harvest pre-timeout leaks. - to_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot - to_harvested = harvest_leaked_artifacts( - to_destination, - subprocess_started_unix=variant_started_unix, - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: magpie timeout (timeout_sec=%d): %s", - i + 1, - len(grid), - variant.name, - measure_cap_sec, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="magpie_timeout", - error_summary=str(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"timeout: {exc}", - error_class="magpie_timeout", - server_log_path=_existing_log_path(server_log), - note=variant.note, - runtime_sec=round( - max(0.0, time.time() - variant_started_unix), - 2, - ), - nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in to_harvested], - ) - ) - finally: - round_survivors = _teardown_variant_server(slot, lifecycle if auto_warmup else None) - - # Eval bounds could not be installed for a variant that runs eval, so nothing launched. - if rc == EVAL_PROBE_UNPATCHABLE_RETURNCODE: - variant_runtime_sec = round(max(0.0, time.time() - variant_started_unix), 2) - log.error( - "grid_runner: variant %d/%d name=%s aborted: eval_probe_unpatchable: %s", - i + 1, - len(grid), - variant.name, - stderr, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="eval_probe_unpatchable", - error_summary=stderr, - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - returncode=rc, - runtime_sec=variant_runtime_sec, - error=stderr, - error_class="eval_probe_unpatchable", - note=variant.note, - ) - ) - - # Server-liveness watchdog fired: engine/worker bootstrap died but the parent hung. - if rc == SERVER_DEAD_RETURNCODE: - variant_runtime_sec = round( - max(0.0, time.time() - variant_started_unix), - 2, - ) - sd_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot - sd_harvested = harvest_leaked_artifacts( - sd_destination, - subprocess_started_unix=variant_started_unix, - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: " - "server_init_dead (engine/worker bootstrap failed; parent " - "hung) after %.1fs", - i + 1, - len(grid), - variant.name, - variant_runtime_sec, - ) - death_excerpt = server_log_death_excerpt(str(server_log)) or ( - "server engine/worker init failed; parent process hung and was reaped by the liveness watchdog" - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="server_init_dead", - error_summary=death_excerpt, - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - returncode=rc, - runtime_sec=variant_runtime_sec, - error=death_excerpt, - error_class="server_init_dead", - server_log_path=_existing_log_path(server_log), - note=variant.note, - nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in sd_harvested], - ) - ) - - # Detokenizer-stall watchdog fired: the server came up healthy but then went silent (hung engine / wedged - # detokenizer). - if rc == DETOKENIZER_STALL_RETURNCODE: - variant_runtime_sec = round( - max(0.0, time.time() - variant_started_unix), - 2, - ) - ds_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot - ds_harvested = harvest_leaked_artifacts( - ds_destination, - subprocess_started_unix=variant_started_unix, - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: detokenizer_stall " - "(server ready but log went silent) after %.1fs", - i + 1, - len(grid), - variant.name, - variant_runtime_sec, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="detokenizer_stall", - error_summary=( - "server reported ready but emitted no log output (hung " - "engine / detokenizer stall); reaped by the " - "detokenizer-stall watchdog" - ), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - returncode=rc, - runtime_sec=variant_runtime_sec, - error="detokenizer_stall: server ready but log went silent", - error_class="detokenizer_stall", - server_log_path=_existing_log_path(server_log), - note=variant.note, - nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in ds_harvested], - ) - ) - - stopped = stopped_by_the_run(rc) - if stopped is not None: - grid_is_over = _record_round_stop( - stopped, - idx=i, - variant=variant, - slot=slot, - round_label="measured", - returncode=rc, - started_unix=variant_started_unix, - server_log=server_log, - server_survivors=round_survivors, - ) - await _report_finished_variant(i) - if grid_is_over: - break - continue - - workspace = select_run_workspace(slot, known_before=slot_workspaces_before) - # Always-on artifact harvest so each slot keeps its server.log / gpu_metrics / profile relay for Robustness - # RCA. - harvest_destination = workspace if workspace is not None else slot - harvested = harvest_leaked_artifacts( - harvest_destination, - subprocess_started_unix=variant_started_unix, - ) - if harvested: - log.info( - "_grid_runner: variant=%s harvested %d leaked artifact(s): %s", - variant.name, - len(harvested), - ", ".join(src.name for src, _ in harvested), - ) - if workspace is None: - harvest_tags = [f"harvested_leaked_artifact:{src}" for src, _ in harvested] - # An AgentX preflight abort never reaches Magpie, so of course no workspace exists -- but calling it - # ``no_benchmark_workspace`` erases the one fact that decides what to do next. - agentx_preflight_abort = rc == AGENTX_PREFLIGHT_RETURNCODE - if agentx_preflight_abort: - # Same class the baseline path uses for the same abort, so both routes classify it identically. - no_ws_error_class = AGENTX_PREFLIGHT_ERROR_CLASS - no_ws_error_summary = ( - redact_secret_values((stderr or "").strip()[-2000:]) or "AgentX preflight aborted the round" - ) - else: - no_ws_error_class = "no_benchmark_workspace" - no_ws_error_summary = server_log_death_excerpt(str(server_log)) or ( - redact_secret_values((stderr or stdout)[-2000:]) - if rc != 0 - else "no benchmark_* workspace produced" - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: %s (rc=%s)", - i + 1, - len(grid), - variant.name, - no_ws_error_class, - rc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class=no_ws_error_class, - error_summary=no_ws_error_summary, - extra_args=variant.extra_server_args, - ) - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - returncode=rc, - error=no_ws_error_summary, - error_class=no_ws_error_class, - server_log_path=_existing_log_path(server_log), - nonfatal_warnings=harvest_tags, - note=variant.note, - ) - ) - await _report_finished_variant(i) - if agentx_preflight_abort: - # Environment, not variant. - log.error( - "grid_runner: variant %d/%d name=%s aborted on the AgentX preflight; " - "abandoning the remaining %d point(s) -- the client is missing for the " - "whole grid, not for this variant: %s", - i + 1, - len(grid), - variant.name, - len(grid) - (i + 1), - no_ws_error_summary, - ) - # Recorded rather than dropped: a grid that just returns fewer points than it declared reads as a grid - # that was that size. - results.extend( - VariantResult( - name=rest.name, - extra_server_args=rest.extra_server_args, - extra_envs=dict(rest.extra_envs), - status="skipped", - error="AgentX preflight aborted the grid before this variant ran", - error_class=AGENTX_PREFLIGHT_ERROR_CLASS, - note=rest.note, - ) - for rest in grid[i + 1 :] - ) - break - if rc != 0 and not keep_going_on_failure: - break - continue - report_path = workspace / "benchmark_report.json" - # The report is written during shutdown and the reader runs the moment the subprocess is reaped, so the read is - # worth waiting on however the process exited: a non-zero exit can still be a round that served its whole - # protocol, and that verdict is taken from this one read. - report, measurement = await _settled_measurement( - workspace, - subprocess_started_unix=variant_started_unix, - settle_seconds=REPORT_SETTLE_SECONDS, - ) - nonzero_kept_error: str | None = None - warnings = list(measurement.pop("nonfatal_warnings", []) or []) - for leak_src, _ in harvested: - warnings.append(f"harvested_leaked_artifact:{leak_src}") - if warmup_tput is not None: - warnings.append("run_grid_warmup_discarded_first") - warnings.append(f"warmup_round_tput:{float(warmup_tput):.1f}") - - if not measurement.get("valid_measurement"): - if lifecycle_boot_only and rc == 0: - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="succeeded", - workspace=str(workspace), - report_path=str(report_path) if report_path.exists() else None, - returncode=rc, - nonfatal_warnings=warnings, - note="server_lifecycle_boot_only", - ) - ) - await _report_finished_variant(i) - continue - death_excerpt = server_log_death_excerpt(str(server_log)) - if rc != 0: - error = death_excerpt or redact_secret_values((stderr or stdout)[-2000:]) - # The bypass/scriptable path runs the customer body in a child whose stderr is redirected to - # benchmark_stderr.log, not the parent pipe — so `stderr`/`stdout` here are often empty and the real - # diagnostic (e.g. argparse "unrecognized arguments") is only on disk. - if not error.strip(): - error = redact_secret_values(_on_disk_stderr_tail(workspace, slot)) - # Last resort: report.errors when the pipe and on-disk logs are all empty. - if not error.strip(): - error = redact_secret_values(_report_errors_summary(report)) - invalid_class = "magpie_nonzero_invalid_measurement" - elif not report: - error = death_excerpt or "benchmark_report missing" - invalid_class = "benchmark_report_missing" - else: - error = death_excerpt or "benchmark_report missing valid throughput/completed requests" - invalid_class = "benchmark_report_invalid_metric" - log.warning( - "grid_runner: variant %d/%d name=%s aborted: %s (rc=%s): %s", - i + 1, - len(grid), - variant.name, - invalid_class, - rc, - error[:200], - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class=invalid_class, - error_summary=error, - extra_args=variant.extra_server_args, - ) - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - workspace=str(workspace), - report_path=str(report_path) if report_path.exists() else None, - raw_result_path=measurement.get("raw_result_path"), - reported_success=measurement.get("reported_success"), - returncode=rc, - nonfatal_warnings=warnings, - error=error, - error_class=invalid_class, - server_log_path=_existing_log_path(server_log), - note=variant.note, - ) - ) - await _report_finished_variant(i) - if rc != 0 and not keep_going_on_failure: - break - continue - - if rc != 0: - nonzero_error = redact_secret_values((stderr or stdout)[-2000:]) - if served_complete_protocol(measurement): - # Every request the client recorded as requested was served, so - # the exit code came from something the round had already - # finished with. The cause stays on the result: a reader - # comparing this point to its neighbours is owed the reason it - # is not a clean zero. - warnings.append(f"nonzero_rc_after_complete_protocol:{rc}") - nonzero_kept_error = nonzero_error - log.warning( - "grid_runner: variant %s exited %d after serving its whole protocol " - "(%s of %s requests); keeping the measurement: %s", - variant.name, - rc, - measurement.get("completed_requests"), - measurement.get("requested_requests"), - nonzero_error, - ) - else: - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="magpie_nonzero_after_valid_measurement", - error_summary=nonzero_error, - extra_args=variant.extra_server_args, - ) - log.warning( - "grid_runner: variant %s aborted: magpie_nonzero_after_valid_measurement " - "(rc=%d, served %s of %s requests)", - variant.name, - rc, - measurement.get("completed_requests"), - measurement.get("requested_requests"), - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - workspace=str(workspace), - report_path=str(report_path) if report_path.exists() else None, - raw_result_path=measurement.get("raw_result_path"), - reported_success=measurement.get("reported_success"), - returncode=rc, - nonfatal_warnings=warnings, - error=nonzero_error, - error_class="magpie_nonzero_after_valid_measurement", - server_log_path=None, - note=variant.note, - ) - ) - - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="succeeded", - output_throughput=measurement.get("output_throughput"), - request_throughput=measurement.get("request_throughput"), - total_token_throughput=measurement.get("total_token_throughput"), - completed_requests=measurement.get("completed_requests"), - duration_seconds=measurement.get("duration_seconds"), - ttft_mean_ms=measurement.get("ttft_mean_ms"), - **constraint_fields(measurement), - tpot_mean_ms=measurement.get("tpot_mean_ms"), - input_throughput=measurement.get("input_throughput"), - tpot_p90_ms=measurement.get("tpot_p90_ms"), - intvty_p90=measurement.get("e2e_norm_intvty_p90"), - intvty_p50=measurement.get("e2e_norm_intvty_p50"), - request_error_rate=measurement.get("request_error_rate"), - workspace=str(workspace), - report_path=str(report_path) if report_path.exists() else None, - raw_result_path=measurement.get("raw_result_path"), - reported_success=measurement.get("reported_success"), - returncode=rc, - nonfatal_warnings=warnings, - error=nonzero_kept_error, - server_log_path=None, - note=variant.note, - runtime_sec=round( - max(0.0, time.time() - variant_started_unix), - 2, - ), - ) - ) - log.info( - "grid_runner: variant %s tput=%.1f tok/s", - variant.name, - results[-1].output_throughput or 0.0, - ) - await _report_finished_variant(i) - except _VariantAborted as abort: - results.append(abort.result) - await _report_finished_variant(i) - if not keep_going_on_failure: - break - await asyncio.to_thread( - _attach_grid_launch_evidence, - results, - grid=grid, - output_root=output_root, - caller_reused_ready_server=server_already_ready, - ready_server_log=ready_server_log, - ) - return results - - -def _teardown_variant_server(slot: Path, lifecycle: dict[str, Any] | None) -> list[int]: - """Stop every server this variant booted for its own rounds; return the pids still alive after SIGKILL. - - ``lifecycle`` names the persistent server the rounds shared, ``None`` when they shared none. - """ - from ._server_lifecycle import stop_agentx_servers, teardown_lifecycle_server - - if lifecycle is not None: - teardown_lifecycle_server( - pid_dir=slot, - framework=str(lifecycle.get("framework") or ""), - port=int(lifecycle.get("port") or 0), - ) - return stop_agentx_servers(slot) - - -def _not_run_skip_result(variant: GridVariant, stopped: StoppedByTheRun) -> VariantResult: - """Build the ``skipped`` result for a variant the run never got to.""" - return VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="skipped", - error=stopped.never_started, - error_class=stopped.error_class, - note=variant.note, - ) - - -def _existing_log_path(path: Path) -> str | None: - """Return ``path`` as a string when it exists, else ``None``.""" - return str(path) if path.exists() else None - - -def _measurement_server_log_path( - server_log: Path, - workspace: Path | None = None, - *, - slot: Path | None = None, -) -> str | None: - """Return the server log attributable to this variant's measurement.""" - owning_slot = slot or server_log.parent - direct = _existing_log_path(server_log) - if not direct and workspace is not None: - direct = _existing_log_path(workspace / "server.log") - if direct: - return direct - warmup_dir = owning_slot / "warmup_round" - try: - candidates = [path for path in warmup_dir.glob("*/server.log") if path.is_file()] - except OSError: - candidates = [] - if not candidates: - return None - try: - return str(max(candidates, key=lambda path: path.stat().st_mtime)) - except OSError: - return None - - -def _attach_grid_launch_evidence( - results: list[VariantResult], - *, - grid: list[GridVariant], - output_root: Path, - caller_reused_ready_server: bool, - ready_server_log: str | None = None, -) -> None: - """Persist declared and observed launch evidence for each grid result.""" - from ._workload_envs import VLLM_SOURCE_ROOT_ENVS - - for idx, result in enumerate(results): - if idx >= len(grid): - break - # A skipped variant never launched a measurement. - if result.status == "skipped" or result.error_class in { - "capability_unsupported", - "yaml_build_error", - "warmup_yaml_build_error", - }: - continue - slot = output_root / f"variant_{idx:02d}_{_safe(grid[idx].name)}" - config_path = slot / "config.yaml" - workspace = Path(result.workspace) if result.workspace else None - primary_log = Path(result.server_log_path) if result.server_log_path else slot / "server.log" - actual_log = _measurement_server_log_path(primary_log, workspace, slot=slot) - if not actual_log and ready_server_log: - actual_log = _existing_log_path(Path(ready_server_log)) - result.server_log_path = actual_log - evidence = build_launch_evidence( - config_path=config_path, - actual_server_log=actual_log, - framework=os.environ.get("FRAMEWORK", ""), - slot=slot, - caller_reused_ready_server=caller_reused_ready_server, - ) - result.launch_evidence = evidence - result.launch_evidence_path = persist_launch_evidence(evidence, slot=slot) - declared_env = evidence["requested_server_env"].keys() - set(VLLM_SOURCE_ROOT_ENVS) - report = scan_server_log(actual_log, evidence["framework"], declared_env=declared_env) - persist_runtime_findings(report, slot=slot) - - -def _safe(name: str) -> str: - """Filesystem-safe slug for variant directory names.""" - return "".join(c if c.isalnum() or c in "-_." else "_" for c in name)[:60] - - -def _write_variant_abort_marker( - slot: Path, - *, - variant_name: str, - error_class: str, - error_summary: str, - extra_args: str = "", -) -> None: - """Write ``abort_reason.json`` into the variant slot directory.""" - try: - slot.mkdir(parents=True, exist_ok=True) - marker = { - "variant": variant_name, - "error_class": error_class, - "error": (error_summary or "")[:2000], - "extra_args": extra_args, - "aborted_at_utc": time.strftime( - "%Y-%m-%dT%H:%M:%SZ", - time.gmtime(), - ), - } - (slot / "abort_reason.json").write_text( - json.dumps(marker, indent=2, sort_keys=True), - encoding="utf-8", - ) - except OSError as exc: - log.warning( - "_grid_runner: failed to write abort_reason.json at %s: %s", - slot, - exc, - ) - - -def _report_errors_summary(report: dict[str, Any] | None, limit: int = 2000) -> str: - """Join ``benchmark_report.json`` ``errors`` into a single diagnostic.""" - if not isinstance(report, dict): - return "" - errors = report.get("errors") - if not isinstance(errors, list): - return "" - text = "; ".join(str(item).strip() for item in errors if str(item).strip()) - return text[-limit:] if text else "" - - -def _on_disk_stderr_tail(*dirs: Path, limit: int = 2000) -> str: - """Return the tail of the first non-empty on-disk benchmark log.""" - for d in dirs: - if not d: - continue - for name in ("benchmark_stderr.log", "benchmark_stdout.log"): - try: - p = d / name - if not p.is_file(): - continue - text = p.read_text(encoding="utf-8", errors="replace").strip() - if text: - return f"[{name}] {text[-limit:]}" - except OSError: - continue - return "" - - -__all__ = [ - "DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC", - "GridVariant", - "ORCHESTRATOR_CANCELLED_CLASS", - "SESSION_TIME_EXHAUSTED_CLASS", - "StoppedByTheRun", - "SGLANG_WATCHDOG_TIMEOUT_ENV", - "VariantResult", - "apply_multi_node_invalid_variants", - "apply_aiter_moe_pin_filter", - "apply_runtime_benchmark_overrides", - "reorder_grid_for_multi_node", - "inject_sglang_attention_backend", - "inject_sglang_context_length", - "inject_sglang_watchdog_timeout", - "merge_server_args", - "remove_server_args", - "resolve_sglang_watchdog_timeout", - "run_grid", - "sanitize_result_dir", - "sanitize_script_name", - "server_args_env_name", - "session_grid_bounds", - "stopped_by_the_run", - # Re-exported from the sibling modules to keep the namespace intact. - "_resolve_magpie_python", - "_resolve_probe_python", - "coerce_extra_envs", - "compact_json_server_args", - "_SPACE_VALUE_FLAGS", - "_MULTI_VALUE_FLAGS", - "_VLLM_SINGLE_VALUE_FLAGS", - "dedup_vllm_server_args", - "_SGLANG_WATCHDOG_FLAG", - "_SGLANG_WATCHDOG_RE", - "DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS", - "DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS", - "SGLANG_CONTEXT_HEADROOM_ENV", - "SGLANG_CONTEXT_FLOOR_ENV", - "_SGLANG_CONTEXT_LENGTH_FLAG", - "_SGLANG_CONTEXT_LENGTH_RE", - "_SGLANG_ATTN_BACKEND_FLAG", - "_SGLANG_ATTN_BACKEND_RE", - "_SGLANG_DUAL_CHUNK_BACKEND", - "_resolve_nonneg_int_env", - "resolve_sglang_context_cap", - "_resolve_dual_chunk_backend", - "_SGLANG_MOE_RUNNER_BACKEND_FLAG", - "_SGLANG_MOE_RUNNER_BACKEND_RE", - "moe_runner_requires_aiter", - "resolve_skip_spec", - "_parse_skip_spec", - "_RE_CUDA_GRAPH_MAX_BS", - "_MN_PARAMS_PRIORITY", - "_MN_BACKENDS_PRIORITY", - "_mn_priority_index", - "_COMPATIBILITY_FLAG_RULES", - "_XDIT_ENV_BLACKLIST", - "_XDIT_ENV_COMBO_BLACKLIST", - "xdit_blacklist_reason", - "_detect_model_class", - "apply_compatibility_filter", - "apply_user_skip_list", -] +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Shared helper for the ``explore`` executor's grid runs.""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import re +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any, Callable + +import yaml + +from hyperloom.common.coerce import to_str_list +from hyperloom.common.env import env_flag, is_truthy +from hyperloom.common.perf_metric import constraint_fields +from hyperloom.common.env_safety import ( + BLOCKED_CHILD_ENV_NAMES, + BLOCKED_EXTERNAL_ENV_NAMES, + _ENV_KEY_RE, + is_python_package_root, + redact_secret_values, + scrub_benchmark_process_env, +) + +from ...phases import machine_state as _phase_state +from hyperloom.inference_optimizer.trace.task_progress import heartbeat_while_output_flows, report_progress +from ..stop_attribution import ( + ORCHESTRATOR_CANCELLED_CLASS, + SESSION_TIME_EXHAUSTED_CLASS, + STOPPED_BY_THE_RUN, + StoppedByTheRun, +) +from ._benchmark_runtime import apply_runtime_benchmark_overrides as apply_runtime_benchmark_overrides +from ._benchmark_interpreter import ( + _resolve_magpie_python as _resolve_magpie_python, + _resolve_probe_python as _resolve_probe_python, +) +from ._accuracy_gate import materialized_run_eval_disabled +from ._recipe_script import launcher_overwritten_envs +from ._subprocess_kill import ( + AGENTX_PREFLIGHT_ERROR_CLASS, + AGENTX_PREFLIGHT_RETURNCODE, + DETOKENIZER_STALL_RETURNCODE, + EVAL_PROBE_UNPATCHABLE_RETURNCODE, + ORCHESTRATOR_CANCELLED_RETURNCODE, + SERVER_DEAD_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, + resolve_benchmark_timeouts, + run_with_session_kill, + server_log_death_excerpt, + session_deadline_to_remaining_sec, + sync_agentx_phase_wait, +) +from .benchmark_result import ( + extract_benchmark_measurement, + harvest_leaked_artifacts, + select_run_workspace, + served_complete_protocol, + snapshot_workspaces, +) +from ._gpu_metrics import write_gpu_metrics_from_report +from .benchmark_backend import build_benchmark_command +from ._inferencex_patcher import ( + ensure_benchmark_lib_eval_start_patched, + ensure_eval_probe_patched, + ensure_eval_unbound_outputs_patched, + eval_probe_targets_exist, +) +from ._launch_evidence import build_launch_evidence, persist_launch_evidence +from ...measurement.runtime_findings import persist_runtime_findings, scan_server_log +from ._server_argv import seal_server_argv + +# Re-exported from sibling modules to keep the module namespace intact. +from ._grid_base import ( + DEFAULT_KEEP_THRESHOLD_PCT as DEFAULT_KEEP_THRESHOLD_PCT, + GridVariant as GridVariant, + coerce_extra_envs as coerce_extra_envs, + VariantResult as VariantResult, +) +from hyperloom.inference_optimizer.grid_server_args import ( + server_args_env_name as server_args_env_name, + merge_server_args as merge_server_args, + compose_server_args as compose_server_args, + remove_server_args as remove_server_args, + compact_json_server_args as compact_json_server_args, + _SPACE_VALUE_FLAGS as _SPACE_VALUE_FLAGS, + _MULTI_VALUE_FLAGS as _MULTI_VALUE_FLAGS, + _VLLM_SINGLE_VALUE_FLAGS as _VLLM_SINGLE_VALUE_FLAGS, + dedup_vllm_server_args as dedup_vllm_server_args, + _shell_safe_dedupe as _shell_safe_dedupe, + DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC as DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC, + SGLANG_WATCHDOG_TIMEOUT_ENV as SGLANG_WATCHDOG_TIMEOUT_ENV, + _SGLANG_WATCHDOG_FLAG as _SGLANG_WATCHDOG_FLAG, + _SGLANG_WATCHDOG_RE as _SGLANG_WATCHDOG_RE, + resolve_sglang_watchdog_timeout as resolve_sglang_watchdog_timeout, + inject_sglang_watchdog_timeout as inject_sglang_watchdog_timeout, + DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS as DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS, + DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS as DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS, + SGLANG_CONTEXT_HEADROOM_ENV as SGLANG_CONTEXT_HEADROOM_ENV, + SGLANG_CONTEXT_FLOOR_ENV as SGLANG_CONTEXT_FLOOR_ENV, + _SGLANG_CONTEXT_LENGTH_FLAG as _SGLANG_CONTEXT_LENGTH_FLAG, + _SGLANG_CONTEXT_LENGTH_RE as _SGLANG_CONTEXT_LENGTH_RE, + _SGLANG_ATTN_BACKEND_FLAG as _SGLANG_ATTN_BACKEND_FLAG, + _SGLANG_ATTN_BACKEND_RE as _SGLANG_ATTN_BACKEND_RE, + _SGLANG_DUAL_CHUNK_BACKEND as _SGLANG_DUAL_CHUNK_BACKEND, + _resolve_nonneg_int_env as _resolve_nonneg_int_env, + resolve_sglang_context_cap as resolve_sglang_context_cap, + inject_sglang_context_length as inject_sglang_context_length, + _resolve_dual_chunk_backend as _resolve_dual_chunk_backend, + inject_sglang_attention_backend as inject_sglang_attention_backend, + _SGLANG_MOE_RUNNER_BACKEND_FLAG as _SGLANG_MOE_RUNNER_BACKEND_FLAG, + _SGLANG_MOE_RUNNER_BACKEND_RE as _SGLANG_MOE_RUNNER_BACKEND_RE, + moe_runner_requires_aiter as moe_runner_requires_aiter, +) +from ._grid_variant_filter import ( + resolve_skip_spec as resolve_skip_spec, + _parse_skip_spec as _parse_skip_spec, + _RE_CUDA_GRAPH_MAX_BS as _RE_CUDA_GRAPH_MAX_BS, + _MN_PARAMS_PRIORITY as _MN_PARAMS_PRIORITY, + _MN_BACKENDS_PRIORITY as _MN_BACKENDS_PRIORITY, + _mn_priority_index as _mn_priority_index, + reorder_grid_for_multi_node as reorder_grid_for_multi_node, + apply_multi_node_invalid_variants as apply_multi_node_invalid_variants, + apply_aiter_moe_pin_filter as apply_aiter_moe_pin_filter, + _COMPATIBILITY_FLAG_RULES as _COMPATIBILITY_FLAG_RULES, + _XDIT_ENV_BLACKLIST as _XDIT_ENV_BLACKLIST, + _XDIT_ENV_COMBO_BLACKLIST as _XDIT_ENV_COMBO_BLACKLIST, + xdit_blacklist_reason as xdit_blacklist_reason, + _detect_model_class as _detect_model_class, + apply_compatibility_filter as apply_compatibility_filter, + apply_user_skip_list as apply_user_skip_list, +) + + +log = logging.getLogger(__name__) + + +def _validate_magpie_python_override(value: str) -> str: + """Validate caller-supplied Magpie interpreter overrides before argv[0].""" + raw = str(value or "").strip() + if not raw: + return "" + if any(ch in raw for ch in ("\x00", "\n", "\r")): + raise ValueError("magpie_python contains control characters") + resolved = Path(raw) + if not resolved.is_absolute(): + found = shutil.which(raw) + if not found: + raise ValueError(f"magpie_python is not executable on PATH: {raw}") + resolved = Path(found) + if not resolved.is_file(): + raise ValueError(f"magpie_python is not a file: {raw}") + if not resolved.name.lower().startswith("python"): + raise ValueError(f"magpie_python must point to a Python interpreter, got: {raw}") + return str(resolved) + + +class _VariantAborted(Exception): + """A variant ended before it reached its benchmark, carrying the result that records why.""" + + def __init__(self, result: VariantResult) -> None: + super().__init__(result.error_class or "variant_aborted") + self.result = result + + +def _resolve_session_dir() -> Path: + """Resolve the active session_dir for executors that need an output root.""" + from hyperloom.inference_optimizer.session.paths import session_dir as _sd + + return _sd() + + +class SessionDirField: + """An executor's ``session_dir``, resolved on read rather than at construction. + + The executors here are module-level singletons built at import, which is + before the CLI pins the session. Resolving in ``__init__`` freezes them on + the workspace root that concurrent sessions share. + """ + + def __set_name__(self, owner: type, name: str) -> None: + self._slot = f"_{name}" + + def __get__(self, obj: Any, objtype: type | None = None) -> Any: + if obj is None: + return self + explicit = getattr(obj, self._slot, None) + return explicit if explicit is not None else _resolve_session_dir() + + def __set__(self, obj: Any, value: Path | str | None) -> None: + setattr(obj, self._slot, Path(value) if value else None) + + +# SKIP_VARIANTS: comma/whitespace patterns matched (exact or fnmatch) against ``GridVariant.name``. + + +# Env-flag capability probe: a serving env flag can be defined in the build yet still crash the server at engine init +# because the code path it activates imports a module the build did not package. +_UNSET = object() + +# Cached probe result keyed by framework. +_CAP_PROBE_CACHE: dict[str, str | None] = {} + +# Subprocess probe: locate the installed vLLM package via ``find_spec`` without importing it, read the aiter +# shared-expert router source, extract the ``fused_moe.*`` modules it imports, and verify each resolves to a real file +# in THIS build. +_AITER_SHARED_EXPERT_PROBE_SCRIPT = ( + "import importlib.util as u, os, re, json\n" + "def go():\n" + " try:\n" + " spec = u.find_spec('vllm')\n" + " except Exception:\n" + " return {'status': 'unknown'}\n" + " if not spec or not spec.origin:\n" + " return {'status': 'unknown'}\n" + " vdir = os.path.dirname(spec.origin)\n" + " router = os.path.join(vdir, 'model_executor', 'layers', 'fused_moe',\n" + " 'router', 'aiter_shared_routed_fused_moe_router.py')\n" + " if not os.path.exists(router):\n" + " return {'status': 'unknown'}\n" + " try:\n" + " text = open(router, encoding='utf-8').read()\n" + " except Exception:\n" + " return {'status': 'unknown'}\n" + " mods = re.findall(r'from\\s+(vllm\\.model_executor\\.layers\\.fused_moe\\.[A-Za-z0-9_.]+)\\s+import', text)\n" + " missing = []\n" + " for m in sorted(set(mods)):\n" + " rel = m[len('vllm.'):].replace('.', os.sep)\n" + " if not (os.path.exists(os.path.join(vdir, rel + '.py'))\n" + " or os.path.exists(os.path.join(vdir, rel, '__init__.py'))):\n" + " missing.append(m)\n" + " return {'status': 'unsupported', 'missing': missing} if missing else {'status': 'ok'}\n" + "print(json.dumps(go()))\n" +) + + +def _probe_vllm_aiter_shared_expert_unsupported() -> str | None: + """Return a drop reason if the installed vLLM build can't honour the aiter shared-expert fusion flag, else ``None``.""" + cached = _CAP_PROBE_CACHE.get("vllm", _UNSET) + if cached is not _UNSET: + return cached # type: ignore[return-value] + try: + proc = subprocess.run( + [_resolve_probe_python(), "-c", _AITER_SHARED_EXPERT_PROBE_SCRIPT], + capture_output=True, + text=True, + timeout=30, + ) + lines = [ln for ln in (proc.stdout or "").splitlines() if ln.strip()] + payload = json.loads(lines[-1]) if lines else {} + except Exception: # noqa: BLE001 + return None + status = payload.get("status") + if status == "ok": + _CAP_PROBE_CACHE["vllm"] = None + return None + if status == "unsupported": + missing = ", ".join(payload.get("missing") or []) or "(unknown module)" + reason = ( + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS enabled but the installed " + "vLLM build's aiter shared-expert router imports missing module(s): " + f"{missing}. Flag unusable on this build (server crashes at engine " + "init); upgrade vLLM to a build that ships the module." + ) + _CAP_PROBE_CACHE["vllm"] = reason + return reason + return None # unknown => do not drop or cache + + +def unsupported_capability_reason(variant: "GridVariant") -> str | None: + """Return a drop reason if a variant sets an env flag the installed framework build cannot honour, else ``None``.""" + fw = (os.environ.get("FRAMEWORK", "") or "sglang").strip().lower() + if fw != "vllm": + return None + envs = {str(k): str(v) for k, v in (getattr(variant, "extra_envs", None) or {}).items()} + val = envs.get("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS") + if val is None or not is_truthy(val, default=True): + return None + return _probe_vllm_aiter_shared_expert_unsupported() + + +# Env vars whose value names a config file the serving stack reads unconditionally: aiter's tuned-GEMM loader skips +# its merge default and pd.read_csv's exactly the AITER_CONFIG_* path when one is set, so a missing file kills the +# server at config load instead of falling back to the default table. Values that are not file paths (e.g. geak's +# JSON-payload AITER_CONFIG) do not carry the trailing underscore and stay untouched. +_FILE_REFERENCING_ENV_PREFIXES: tuple[str, ...] = ("AITER_CONFIG_",) + + +def _is_file(path: str) -> bool: + # Path.is_file() raises for an unreadable parent or an overlong name (before Python 3.14); such a path is not a + # file the server can read either, and one variant's proposal must not crash the whole grid. + try: + return Path(path).is_file() + except OSError: + return False + + +def ungrounded_env_file_reason(variant: "GridVariant") -> str | None: + """Return a drop reason if an extra_envs value names a config file that does not exist, else ``None``. + + Grounds file-like env values the same way ``vet_patches`` grounds patch targets: a proposal may point a config + env at an artifact only a later (possibly rejected) authoring step would create, and the benchmark would boot a + server that dies at config load. The check is existence-only; anything already on disk (a promoted, durable + artifact) passes. A set value is a ``:``-joined list of paths (see ``csvs_aiter_will_load``), so every + non-empty element must exist, not only the first. + """ + for raw_key, raw_value in (getattr(variant, "extra_envs", None) or {}).items(): + key = str(raw_key or "").strip().upper() + if not key.startswith(_FILE_REFERENCING_ENV_PREFIXES): + continue + value = str(raw_value or "").strip() + if not value: + continue + missing = [p for p in (part.strip() for part in value.split(":")) if p and not _is_file(p)] + if missing: + return ( + f"{key} points at {value}, whose element(s) {':'.join(missing)} do not exist; the server would read it " + "unconditionally and die at config load. Point it at a file a stage authored " + "(e.g. a promoted tuned-GEMM CSV) or leave it unset to use the default config table." + ) + return None + + +# Sanitization for LLM-supplied overrides (benchmark_script / result_dir): reject path separators / shell +# metacharacters, raising ``ValueError`` instead of running an unsafe subprocess. +_SCRIPT_NAME_RE = re.compile(r"^[A-Za-z0-9_.\-]+\.sh$") +_RESULT_DIR_FORBID_RE = re.compile(r"[\s\"'`$;&|<>(){}\[\]\\*?!]") + + +def sanitize_script_name(value: Any) -> str | None: + """Return ``value`` if it's a safe Magpie benchmark script file name.""" + if value is None: + return None + text = str(value).strip() + if not text: + return None + if not _SCRIPT_NAME_RE.match(text): + raise ValueError( + f"benchmark_script={text!r} rejected: must be a bare *.sh " + "file name (no path separators, no shell metacharacters)" + ) + return text + + +def sanitize_result_dir(value: Any) -> str | None: + """Return ``value`` if it's a safe absolute (or workspace-relative) dir.""" + if value is None: + return None + text = str(value).strip() + if not text: + return None + if _RESULT_DIR_FORBID_RE.search(text): + raise ValueError( + f"result_dir={text!r} rejected: contains whitespace or shell " + "metacharacters; pass an absolute or workspace-relative path" + ) + return text + + +def _is_safe_path_entry(entry: str) -> bool: + """A single path entry is safe iff it has no ``:``, no ``..``, no control chars.""" + return ( + bool(entry) + and ":" not in entry + and ".." not in Path(entry).parts + and not any(c in entry for c in ("\n", "\r", "\x00")) + ) + + +# Reserved keys carried by dedicated override fields; never accepted via runtime_env. +_RUNTIME_ENV_RESERVED: frozenset[str] = frozenset({"PATH", "PYTHONPATH", "LD_LIBRARY_PATH"}) + + +def apply_runtime_override(envs: dict[str, str], override: dict[str, Any]) -> None: + """Inject an attempt runtime override into the materialized YAML envs dict.""" + if not override: + return + from ._workload_envs import prepend_launch_path + + path_prefix = str(override.get("path_prefix") or "").strip() + if path_prefix: + prepend_launch_path(envs, "PATH", path_prefix) + entrypoint_bin = str(override.get("entrypoint_bin_dir") or "").strip() + if entrypoint_bin: + if _is_safe_path_entry(entrypoint_bin): + prepend_launch_path(envs, "PATH", entrypoint_bin) + else: + log.warning("apply_runtime_override: dropping unsafe entrypoint_bin_dir %r", entrypoint_bin) + pp_prefix = str(override.get("pythonpath_prefix") or "").strip() + if pp_prefix: + if _is_safe_path_entry(pp_prefix): + prepend_launch_path(envs, "PYTHONPATH", pp_prefix) + else: + log.warning("apply_runtime_override: dropping unsafe pythonpath_prefix %r", pp_prefix) + # Multi-entry prefixes: prepend in order so the first entry wins, ahead of any single-dir pythonpath_prefix and + # the inherited value. + for entry in reversed(_coerce_prefix_list(override.get("pythonpath_prefixes"))): + if _is_safe_path_entry(entry): + prepend_launch_path(envs, "PYTHONPATH", entry) + else: + log.warning("apply_runtime_override: dropping unsafe pythonpath entry %r", entry) + # Native loader path: prepend attempt entries while preserving any inherited entries (e.g. /opt/rocm/lib). + for entry in reversed(_coerce_prefix_list(override.get("ld_library_path_prefix"))): + if _is_safe_path_entry(entry): + prepend_launch_path(envs, "LD_LIBRARY_PATH", entry) + else: + log.warning("apply_runtime_override: dropping unsafe ld_library_path entry %r", entry) + _runtime_env = override.get("runtime_env") + if isinstance(_runtime_env, dict): + for raw_k, raw_v in _runtime_env.items(): + key = str(raw_k) + if not _ENV_KEY_RE.match(key) or key in BLOCKED_CHILD_ENV_NAMES or key in _RUNTIME_ENV_RESERVED: + log.warning("apply_runtime_override: dropping unsafe runtime_env key %r", key) + continue + envs[key] = str(raw_v) + for key in ("framework_bin", "framework_python", "framework_venv_root"): + val = str(override.get(key) or "").strip() + if val: + envs[f"HYPERLOOM_{key.upper()}"] = val + # runtime_python_exe takes priority over framework_python as the launch interpreter. + rpe = str(override.get("runtime_python_exe") or "").strip() + if rpe: + envs["HYPERLOOM_FRAMEWORK_PYTHON"] = rpe + + +def _coerce_prefix_list(value: Any) -> list[str]: + """Normalize a runtime-prefix field (list/tuple/str) to a list of entries.""" + if value is None: + return [] + if isinstance(value, str): + return [value.strip()] if value.strip() else [] + if isinstance(value, (list, tuple)): + return [str(v).strip() for v in value if str(v).strip()] + return [] + + +def _build_variant_yaml( + base_yaml_path: Path, + base_extra_args: str, + variant: GridVariant, + *, + output_subdir: Path, + model_path: str | None = None, + gpu_type: str | None = None, + benchmark_script: str | None = None, + server_lifecycle: dict[str, Any] | None = None, + base_args_mode: str = "append", + base_extra_envs: dict[str, str] | None = None, + base_remove_args: list[str] | None = None, + base_unset_envs: list[str] | None = None, +) -> Path: + """Materialize a per-variant Magpie YAML on disk.""" + with base_yaml_path.open(encoding="utf-8") as f: + cfg = yaml.safe_load(f) + bench = cfg.setdefault("benchmark", {}) + replacing = str(base_args_mode).strip().lower() == "replace" + envs = apply_runtime_benchmark_overrides( + bench, + model_path=model_path, + gpu_type=gpu_type, + benchmark_script=benchmark_script, + conc=variant_conc(variant), + ) + extra_args_env = server_args_env_name(bench.get("framework")) + + variant_remove = to_str_list(getattr(variant, "remove_args", [])) + # A replacing base drops the inherited string wholesale, so only the + # variant's own removals still name flags that survive to be stripped. + effective_remove = ( + variant_remove if replacing else list(dict.fromkeys(to_str_list(base_remove_args) + variant_remove)) + ) + combined = compose_server_args( + inherited_args="" if replacing else str(envs.get(extra_args_env, "")), + base_extra_args=base_extra_args, + variant_extra_args=variant.extra_server_args, + remove_args=effective_remove, + args_mode=getattr(variant, "args_mode", "append"), + ) + # A grid variant never injects a MoE runner backend itself, but it does inherit one -- from the baseline recipe it + # was seeded with, or from an explicitly authored variant. + if combined and _SGLANG_MOE_RUNNER_BACKEND_RE.search(combined): + from ._workload_envs import _remove_moe_runner_backend_arg + + if extra_args_env == "EXTRA_SGLANG_ARGS" and moe_runner_requires_aiter(combined, model_path): + log.warning( + "grid: dropping inherited --moe-runner-backend for variant %s: " + "this checkpoint's MoE quant scheme is only implemented on the " + "aiter runner and would crash on the first forward pass.", + variant.name, + ) + combined = _remove_moe_runner_backend_arg(combined) + if combined: + envs[extra_args_env] = _shell_safe_dedupe(combined) + elif extra_args_env in envs or variant.args_mode == "replace" or base_args_mode == "replace": + envs[extra_args_env] = "" + # Composed base-then-variant, so a variant unsetting a key the base sets + # removes it: the last layer to name a key is the one that decides it. + for k in to_str_list(base_unset_envs): + if k.strip().upper() in BLOCKED_EXTERNAL_ENV_NAMES: + continue + envs.pop(k, None) + for k, v in (base_extra_envs or {}).items(): + envs[str(k)] = str(v) + for k in getattr(variant, "unset_envs", []) or []: + # Unsetting a pin retargets the benchmark rather than toggling a knob. + if str(k).strip().upper() in BLOCKED_EXTERNAL_ENV_NAMES: + log.warning("grid: refusing to unset pinned env %s for variant %s", k, variant.name) + continue + envs.pop(str(k), None) + for k, v in variant.extra_envs.items(): + envs[str(k)] = str(v) + from ._workload_envs import pin_mlperf_round_concurrency, prepend_launch_path + + pin_mlperf_round_concurrency(envs) + # The launcher re-exports these unconditionally, so a value carried here is + # one the run never used. + for k in launcher_overwritten_envs(bench) & envs.keys(): + log.warning("grid: dropping %s for variant %s; the recipe overwrites it", k, variant.name) + envs.pop(k, None) + # The three AgentX bounds took this rung's CONC through ``variant_conc`` above, not through this merge: raising + # the client's grace alone would make the round wait inside a cap that did not move with it. + _overlay = str(getattr(variant, "overlay_pythonpath", "") or "").strip() + if _overlay: + # Structural containment on the overlay dir before it is prepended to PYTHONPATH: a legitimate authored-kernel + # overlay is a single existing directory (never a ``:``-joined list, never a ``..`` traversal or control + # char). + _overlay_ok = ( + ":" not in _overlay + and ".." not in Path(_overlay).parts + and not any(c in _overlay for c in ("\n", "\r", "\x00")) + and Path(_overlay).is_dir() + ) + if _overlay_ok: + prepend_launch_path(envs, "PYTHONPATH", _overlay) + else: + log.warning( + "grid: dropping unsafe overlay_pythonpath %r (not a single " + "existing directory / contains separator or traversal)", + _overlay, + ) + + # Attempt runtime override: inject path_prefix/pythonpath_prefix/framework_bin etc. into benchmark.envs so the + # server subprocess resolves the attempt runtime. + _rt_override = getattr(variant, "runtime_override", None) or {} + if _rt_override: + apply_runtime_override(envs, _rt_override) + + # PATH guard: the xdit wrapper needs both `/venv/bin` (the `xdit` console script) and `/opt/rocm/bin` (`hipcc`); + # force-prepend both so an LLM-supplied PATH can't drop one. + if str(bench.get("framework", "")).strip().lower() == "xdit": + _cur_path = str(envs.get("PATH", "") or "") + _parts = [p for p in _cur_path.split(":") if p] + for _essential in ("/opt/rocm/bin", "/venv/bin"): + if _essential not in _parts: + _parts.insert(0, _essential) + envs["PATH"] = ":".join(_parts) + + if server_lifecycle is not None: + from ._server_lifecycle import inject_lifecycle + + inject_lifecycle( + bench, + cleanup=bool(server_lifecycle.get("cleanup", True)), + pid_dir=server_lifecycle["pid_dir"], + port=int(server_lifecycle["port"]), + ) + + # The final write to the argument env; nothing below may touch it. + seal_server_argv(envs, bench.get("framework")) + output_subdir.mkdir(parents=True, exist_ok=True) + out_path = output_subdir / "config.yaml" + with out_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(cfg, f, sort_keys=False) + return out_path + + +def _parse_report(workspace: Path) -> dict[str, Any] | None: + """Load ``benchmark_report.json`` from a benchmark workspace.""" + report = workspace / "benchmark_report.json" + if not report.exists(): + return None + try: + with report.open(encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + return None + return data if isinstance(data, dict) else None + + +# How long to keep re-reading ``benchmark_report.json`` when the report does not yet parse into a valid measurement. +REPORT_SETTLE_SECONDS = 30.0 +REPORT_SETTLE_POLL_SECONDS = 1.0 + + +async def _settled_measurement( + workspace: Path, + *, + subprocess_started_unix: float | None, + settle_seconds: float = REPORT_SETTLE_SECONDS, + poll_seconds: float = REPORT_SETTLE_POLL_SECONDS, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + """Read the benchmark report, re-reading briefly while it is still settling.""" + deadline = time.monotonic() + max(0.0, float(settle_seconds)) + attempts = 0 + while True: + report = _parse_report(workspace) + measurement = extract_benchmark_measurement( + report, + workspace=workspace, + subprocess_started_unix=subprocess_started_unix, + ) + attempts += 1 + if measurement.get("valid_measurement") or time.monotonic() >= deadline: + if attempts > 1 and measurement.get("valid_measurement"): + log.info( + "grid_runner: benchmark_report became valid after %d read(s); " + "the report was still being written when the subprocess was reaped", + attempts, + ) + # Harvest already tried this, but it runs before the report is guaranteed to exist: a report Magpie + # finishes writing after the subprocess is reaped would otherwise leave the round with no GPU artifact at + # all. Here the report is in hand, so write it from that rather than reading the file a second time. + if report is not None: + write_gpu_metrics_from_report(workspace, report, source="benchmark_report.json") + return report, measurement + await asyncio.sleep(max(0.01, float(poll_seconds))) + + +def _run_grid_warmup_enabled() -> bool: + """Whether ``run_grid`` should discard a cold warmup round when possible.""" + return env_flag("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", default=not os.environ.get("PYTEST_CURRENT_TEST")) + + +def _prepend_magpie_pythonpath(magpie_dir: str, current_pythonpath: str) -> str: + """Prepend Magpie's import root to PYTHONPATH, skipping package-root dirs.""" + if not magpie_dir or is_python_package_root(magpie_dir): + return current_pythonpath + return f"{magpie_dir}:{current_pythonpath}" if current_pythonpath else magpie_dir + + +def sync_benchmark_timeout(config_path: Path, timeout_sec: float) -> None: + """Give Magpie and bypass the same cap as the enclosing benchmark process.""" + cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) + bench = cfg["benchmark"] + bench["timeout_seconds"] = timeout_sec + if bench.get("server_lifecycle"): + bench["server_lifecycle"]["server_ready_timeout_s"] = timeout_sec + envs = bench.setdefault("envs", {}) + envs["PYTHONUNBUFFERED"] = "1" + sync_agentx_phase_wait(envs, timeout_sec) + config_path.write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8") + + +def _benchmark_server_log(config_path: Path, output_dir: Path) -> str | None: + """Scriptable benchmarks have no server whose readiness can arm silence.""" + cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) + framework = str(cfg["benchmark"].get("framework", "")).lower() + return None if framework in {"custom", "xdit"} else str(output_dir / "server.log") + + +def _run_magpie( + *, + magpie_python: str, + config_path: Path, + output_dir: Path, + timeout_sec: int, + cwd: str, + result_dir: str | None = None, + silence_timeout_sec: float | None = None, + server_already_ready: bool = False, + serving_lease: Any = None, + on_output: Callable[[], None] | None = None, + session_deadline_sec: float | None = None, + unset_envs: list[str] | None = None, +) -> tuple[int, str, str]: + """Blocking subprocess wrapper. Returns (rc, stdout, stderr).""" + sync_benchmark_timeout(config_path, timeout_sec) + server_log_path = _benchmark_server_log(config_path, output_dir) + env = scrub_benchmark_process_env(os.environ.copy()) + env["PYTHONUNBUFFERED"] = "1" + from ._workload_envs import resolve_reference_launch + + _args, _envs, reference_controls = resolve_reference_launch() + for name in [*(reference_controls.get("unset_envs") or []), *(unset_envs or [])]: + if name.strip().upper() not in BLOCKED_EXTERNAL_ENV_NAMES: + env.pop(name, None) + env["PATH"] = f"/opt/venv/bin:{env.get('PATH', '')}" + magpie_dir = os.environ.get("MAGPIE_PATH") or "" + if magpie_dir: + env["PYTHONPATH"] = _prepend_magpie_pythonpath(magpie_dir, env.get("PYTHONPATH", "")) + + # Multi-node: tell Magpie to skip its local-server launch and point benchmark_serving at the head pod's ClusterIP. + from ._multi_node_env import magpie_remote_env + + env.update(magpie_remote_env()) + + # Pin Magpie's InferenceX resolution to ``$INFERENCEX_PATH`` (its highest-precedence rung) so it loads the patched + # checkout, not a stale copy. + inferencex_path = os.environ.get("INFERENCEX_PATH", "").strip() + if inferencex_path: + env["MAGPIE_INFERENCEX_PATH"] = inferencex_path + # Baseline patches its own checkout, but explore / sweep never pass through that hook: re-assert here so a + # resumed session or a re-cloned checkout still emits the eval-start marker. + ensure_benchmark_lib_eval_start_patched(Path(inferencex_path)) + + # The generation bounds + pathology probe are asserted whether or not ``$INFERENCEX_PATH`` is set: unset falls + # back to the same env discovery the baseline arm uses ($MAGPIE_PATH/InferenceX). + probe_root = Path(inferencex_path) if inferencex_path else None + # Best-effort, unlike the probe below: a missing guard only costs the reason a failed round reports. + ensure_eval_unbound_outputs_patched(probe_root) + if not ensure_eval_probe_patched(probe_root) and not materialized_run_eval_disabled(config_path): + eval_bounds_msg = ( + "eval generation bounds + pathology probe are not installed " + "(utils/evals/patches/lm_eval_sitecustomize.py, inferencex=" + f"{inferencex_path or ''}, INFERENCEX_PATH=" + f"{os.environ.get('INFERENCEX_PATH', '') or ''}); this " + "variant runs eval, so it would be scored against a baseline that " + "truncated at a different point" + ) + if eval_probe_targets_exist(probe_root): + log.error("grid_runner: %s; failing this benchmark", eval_bounds_msg) + return (EVAL_PROBE_UNPATCHABLE_RETURNCODE, "", eval_bounds_msg) + log.warning("grid_runner: %s", eval_bounds_msg) + # AgentX: deploy the aiperf client into InferenceX ``benchmarks/`` + preflight aiperf right before Magpie runs it, + # via the shared helper (also used by the baseline/profile shell-out). + from ._workload_envs import prepare_agentx_runtime + + _agx_err = prepare_agentx_runtime(env=env, inferencex_path=inferencex_path, config_path=config_path) + if _agx_err: + log.error("%s; failing this benchmark", _agx_err) + return (AGENTX_PREFLIGHT_RETURNCODE, "", _agx_err) + # RESULT_DIR default; leaks are picked up by the salvage path. + env["RESULT_DIR"] = result_dir or str(output_dir) + # InferenceX ``run_lm_eval`` cleans ``$EVAL_RESULT_DIR`` after processing lm-eval output. + env["EVAL_RESULT_DIR"] = str(Path(env["RESULT_DIR"]) / "eval_output") + # Pin SERVER_LOG / GPU_METRICS_CSV per-task so logs land alongside ``benchmark_report.json``. + env["SERVER_LOG"] = str(output_dir / "server.log") + env["GPU_METRICS_CSV"] = str(output_dir / "gpu_metrics.csv") + + # Ray-managed GPU execution (§12 T1): route the round through the serving lease's actor, which holds ``num_gpus`` + # across every round sharing this server. + if serving_lease is not None: + from ._ray_backend import strip_visible_devices_from_config + + ray_config_path = strip_visible_devices_from_config(config_path) + cmd = build_benchmark_command( + python_exe=magpie_python, + config_path=ray_config_path, + output_dir=output_dir, + ) + # ``on_output`` cannot follow the round here: the benchmark runs inside a Ray actor in another process + # (potentially on another node) and only its final ``(rc, stdout, stderr)`` comes back, so there is nothing + # local to call per line. + return serving_lease.run_session_kill( + cmd, + env=env, + cwd=cwd, + timeout=timeout_sec, + silence_timeout_sec=silence_timeout_sec, + server_log_path=server_log_path, + server_already_ready=server_already_ready, + # Converted here, at the last moment before the process boundary: the actor cannot read this process's + # monotonic clock. + session_remaining_sec=session_deadline_to_remaining_sec(session_deadline_sec), + ) + + cmd = build_benchmark_command( + python_exe=magpie_python, + config_path=config_path, + output_dir=output_dir, + ) + # The launch puts Magpie in its own POSIX session and tears down the whole + # descendant tree on every exit path. + proc = run_with_session_kill( + cmd, + env=env, + cwd=cwd, + timeout=timeout_sec, + silence_timeout_sec=silence_timeout_sec, + server_log_path=server_log_path, + server_already_ready=server_already_ready, + on_output=on_output, + session_deadline_sec=session_deadline_sec, + ) + return proc.returncode, proc.stdout or "", proc.stderr or "" + + +def _num_gpus_for_config(config_path: Path) -> float: + """Read the tensor-parallel size (``TP``) from a materialized benchmark YAML.""" + try: + with Path(config_path).open(encoding="utf-8") as fp: + cfg = yaml.safe_load(fp) or {} + envs = (cfg.get("benchmark") or {}).get("envs") or {} + return float(int(envs.get("TP", 1) or 1)) + except Exception: # noqa: BLE001 — best-effort; default to 1 GPU + return 1.0 + + +def _mn_restart_env( + reference_envs: dict[str, str], + variant: Any, + unset_envs: list[str], +) -> dict[str, str]: + """Merge the reference server envs under one variant's own overrides.""" + merged = {k: v for k, v in reference_envs.items() if k not in unset_envs} + merged.update({str(k): str(v) for k, v in (variant.extra_envs or {}).items()}) + return merged + + +def _resolve_mn_effective_server_args( + cfg_path: Path, + base_yaml_path: Path, + variant: Any, + *, + base_extra_args: str, + base_args_mode: str, +) -> str: + """Resolve the multi-node server args for a variant restart; prefer the materialized variant YAML, falling back to a recompose from the base YAML.""" + try: + with cfg_path.open(encoding="utf-8") as _f: + _variant_cfg = yaml.safe_load(_f) or {} + _variant_bench = _variant_cfg.get("benchmark") or {} + _variant_envs = _variant_bench.get("envs") or {} + _variant_framework_env = server_args_env_name(_variant_bench.get("framework")) + return str(_variant_envs.get(_variant_framework_env) or "") + except Exception: + log.debug( + "grid_runner: failed to read materialized variant args from %s", + cfg_path, + exc_info=True, + ) + try: + with base_yaml_path.open(encoding="utf-8") as _f: + _base_cfg = yaml.safe_load(_f) or {} + _base_bench = _base_cfg.get("benchmark") or {} + _base_envs = _base_bench.get("envs") or {} + _base_framework_env = server_args_env_name(_base_bench.get("framework")) + _fallback_inherited_args = str(_base_envs.get(_base_framework_env) or "") + except Exception: # noqa: BLE001 - best-effort parity fallback + _fallback_inherited_args = "" + return _shell_safe_dedupe( + compose_server_args( + inherited_args="" if str(base_args_mode).strip().lower() == "replace" else _fallback_inherited_args, + base_extra_args=base_extra_args, + variant_extra_args=variant.extra_server_args, + remove_args=getattr(variant, "remove_args", []), + args_mode=getattr(variant, "args_mode", "append"), + ) + ) + + +def _variant_progress_note( + grid: list[GridVariant], + results: list[VariantResult], + idx: int, +) -> dict[str, Any]: + """Build the progress note for the variant at ``idx`` from that variant's own row.""" + landed = results[idx] if idx < len(results) else None + return { + "unit": "variant", + "label": grid[idx].name, + "index": idx + 1, + "total": len(grid), + "status": getattr(landed, "status", None), + "output_throughput": getattr(landed, "output_throughput", None), + } + + +# The returncode side of :mod:`...stop_attribution`: the same two causes, keyed by the sentinel a subprocess comes +# back with. +_STOPPED_BY_THE_RUN: dict[int, StoppedByTheRun] = { + SESSION_TIME_EXHAUSTED_RETURNCODE: STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + ORCHESTRATOR_CANCELLED_RETURNCODE: STOPPED_BY_THE_RUN[ORCHESTRATOR_CANCELLED_CLASS], +} + + +def stopped_by_the_run(returncode: int | None) -> StoppedByTheRun | None: + """Return how to record a round the run itself stopped, if it did.""" + if returncode is None: + return None + return _STOPPED_BY_THE_RUN.get(int(returncode)) + + +def variant_conc(variant: Any) -> int | None: + """The concurrency a variant will run at, or ``None`` when it names none.""" + raw = (getattr(variant, "extra_envs", None) or {}).get("CONC") + try: + conc = int(raw) + except (TypeError, ValueError): + return None + return conc if conc > 0 else None + + +def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: + """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call.""" + if shared_state is None: + return (None, None) + deadline_fn = getattr(shared_state, "grid_session_deadline_sec", None) + deadline = deadline_fn() if callable(deadline_fn) else None + variant_sec = _phase_state.one_more_measurement_sec(shared_state) + if variant_sec is None: + variant_sec = _phase_state.measured_seconds(shared_state, "baseline_runtime_sec") + return (deadline, variant_sec) + + +async def run_grid( + *, + base_yaml_path: Path, + base_extra_args: str, + grid: list[GridVariant], + output_root: Path, + magpie_python: str | None = None, + keep_going_on_failure: bool = True, + model_path: str | None = None, + gpu_type: str | None = None, + benchmark_script: str | None = None, + result_dir: str | None = None, + server_lifecycle: dict[str, Any] | None = None, + base_args_mode: str = "append", + base_extra_envs: dict[str, str] | None = None, + base_remove_args: list[str] | None = None, + base_unset_envs: list[str] | None = None, + warmup_before_measure: bool | None = None, + server_already_ready: bool = False, + ready_server_log: str | None = None, + serving_lease: Any = None, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, + deadline_stop: StoppedByTheRun = STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + lifecycle_boot_only: bool = False, +) -> list[VariantResult]: + """Execute variants; ``deadline_stop`` names the owner of the supplied deadline. + + ``ready_server_log`` is the log of the caller's ready server, measured when no variant log exists. + """ + silence_timeout_sec, benchmark_timeout_sec = resolve_benchmark_timeouts() + if not magpie_python: + # Backend-aware: bypass uses a plain python3, not Magpie's venv. + from .benchmark_backend import resolve_benchmark_interpreter + + magpie_python = resolve_benchmark_interpreter() + else: + magpie_python = _validate_magpie_python_override(magpie_python) + if warmup_before_measure is None: + warmup_before_measure = _run_grid_warmup_enabled() + auto_warmup_requested = bool(warmup_before_measure and server_lifecycle is None) + results: list[VariantResult] = [] + # This function names the working directory, so it creates it: callers and the per-variant config writer both + # happen to create it first today, and neither is a contract. + output_root.mkdir(parents=True, exist_ok=True) + cwd = str(output_root) + + # Reap orphaned aiter JIT build locks before booting any server. + try: + from ._aiter_jit import sweep_stale_aiter_locks_if_dead + + _lock_sweep = sweep_stale_aiter_locks_if_dead() + if _lock_sweep.get("deleted"): + log.warning( + "grid_runner: reaped %d orphaned aiter JIT lock(s) under %s before server launch (compiler_alive=%s)", + _lock_sweep.get("deleted"), + _lock_sweep.get("dir"), + _lock_sweep.get("compiler_alive"), + ) + except Exception as exc: # noqa: BLE001 — best-effort hygiene, never blocks the grid + log.debug("grid_runner: aiter lock sweep swallowed: %r", exc) + + # Multi-node: the reference recipe's server envs do not reach the variant restart through + # restart_server_for_round, which never reads them. + from ._multi_node_env import is_multi_node as _mn_is_multi_node + + _mn_ref_envs: dict[str, str] = {} + if _mn_is_multi_node(): + from ._workload_envs import resolve_reference_base + + _, _mn_ref_envs = resolve_reference_base() + + # Reported on entry, not on completion: ``_report_finished_variant`` only runs once a result has been appended, so + # a first variant that hangs — or a branch that raises before reaching it — would emit nothing at all, which is + # exactly the silence the heartbeat exists to break. + async def _unit_started(idx: int, label: str) -> None: + """Report that a unit of variant ``idx`` is about to start.""" + await report_progress( + unit="variant_step", + label=f"{grid[idx].name}:{label}", + index=idx + 1, + total=len(grid), + status="started", + ) + + async def _reported_magpie(idx: int, label: str, **kwargs: Any) -> tuple[int, str, str]: + """Run one Magpie pass, announced on entry and kept alive by its output.""" + await _unit_started(idx, label) + async with heartbeat_while_output_flows( + unit="variant_step", + label=f"{grid[idx].name}:{label}", + index=idx + 1, + total=len(grid), + ) as activity: + return await asyncio.to_thread( + _run_magpie, on_output=activity.note, unset_envs=grid[idx].unset_envs, **kwargs + ) + + # Variant boundary: a progress heartbeat so a grid that runs for hours is distinguishable from one that hung on + # its first variant. + async def _report_finished_variant(idx: int) -> None: + """Report the variant that just landed.""" + await report_progress(**_variant_progress_note(grid, results, idx)) + + def _record_round_stop( + stopped: StoppedByTheRun, + *, + idx: int, + variant: GridVariant, + slot: Path, + round_label: str, + returncode: int | None, + started_unix: float, + server_log: Path, + server_survivors: list[int], + ) -> bool: + """Record a round the run stopped and say whether the grid is over. + + ``server_survivors`` are the servers the variant booted that outlived its teardown. + """ + if returncode == SESSION_TIME_EXHAUSTED_RETURNCODE: + stopped = deadline_stop + runtime_sec = round(max(0.0, time.time() - started_unix), 2) + log.warning( + "grid_runner: variant %d/%d name=%s %s round reaped after %.1fs: %s; recorded as skipped, not failed", + idx + 1, + len(grid), + variant.name, + round_label, + runtime_sec, + stopped.interrupted, + ) + teardown = "tree reaped" + if server_survivors: + teardown += f", but server pid(s) {', '.join(map(str, server_survivors))} survived SIGKILL" + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=stopped.error_class, + error_summary=f"{stopped.interrupted}; {teardown}", + extra_args=variant.extra_server_args, + ) + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="skipped", + returncode=returncode, + runtime_sec=runtime_sec, + error=stopped.interrupted, + error_class=stopped.error_class, + server_log_path=_measurement_server_log_path(server_log, slot=slot), + note=variant.note, + nonfatal_warnings=[f"server_survived_teardown:{pid}" for pid in server_survivors], + ) + ) + if stopped.ends_the_batch: + results.extend(_not_run_skip_result(rest, stopped) for rest in grid[idx + 1 :]) + return True + return not keep_going_on_failure + + # How many full benchmark passes one variant costs. + _mn_warmup_rounds = 0 + if variant_expected_sec is not None: + from ._multi_node_env import ( + is_multi_node as _mn_is_multi_node, + mn_bench_warmup_enabled as _mn_bench_warmup_enabled, + ) + + _mn_warmup_rounds = 1 if (_mn_is_multi_node() and _mn_bench_warmup_enabled()) else 0 + variant_rounds = 1 + (1 if auto_warmup_requested else 0) + _mn_warmup_rounds + + def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant_rounds) -> bool: + """Skip variant ``idx`` and every one after it, or say the budget still fits.""" + if session_deadline_sec is None: + return False + remaining_sec = session_deadline_sec - time.monotonic() + required_sec = float(variant_expected_sec) * rounds_left if variant_expected_sec is not None else None + if remaining_sec > 0 and (required_sec is None or remaining_sec >= required_sec): + return False + log.warning( + "grid_runner: %.0fs left cannot admit this variant's %d remaining round(s) " + "(expected_sec=%s, spent on: %s); skipping %d remaining variant(s)", + max(0.0, remaining_sec), + rounds_left, + required_sec, + spent_on, + len(grid) - idx, + ) + results.extend(_not_run_skip_result(variant, deadline_stop) for variant in grid[idx:]) + return True + + for i, variant in enumerate(grid): + try: + # Session-budget stop: skip the remaining variants once the wall-clock deadline is reached or the remaining + # budget cannot fit another variant, so a timeout halts the grid instead of draining it (and the last variant + # cannot overrun the close window). + if _skip_rest_for_budget(i, spent_on="the variants before this one"): + break + await _unit_started(i, "variant") + slot = output_root / f"variant_{i:02d}_{_safe(variant.name)}" + server_log = slot / "server.log" + # Capability fast-fail: drop a variant whose env flag the build cannot honour before booting a doomed server, + # still recording the failure so the LLM learns not to re-pick it. + cap_reason = unsupported_capability_reason(variant) or ungrounded_env_file_reason(variant) + if cap_reason: + log.warning( + "grid_runner: variant %d/%d name=%s aborted: capability_unsupported: %s", + i + 1, + len(grid), + variant.name, + cap_reason, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="capability_unsupported", + error_summary=cap_reason, + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=cap_reason, + error_class="capability_unsupported", + note=variant.note, + ) + ) + lifecycle: dict[str, Any] = {"eligible": False} + auto_warmup = False + try: + cfg_path = _build_variant_yaml( + base_yaml_path, + base_extra_args, + variant, + output_subdir=slot, + model_path=model_path, + gpu_type=gpu_type, + benchmark_script=benchmark_script, + server_lifecycle=server_lifecycle, + base_args_mode=base_args_mode, + base_extra_envs=base_extra_envs, + base_remove_args=base_remove_args, + base_unset_envs=base_unset_envs, + ) + except Exception as exc: # noqa: BLE001 + build_error = "yaml_build_error" + log.warning( + "grid_runner: variant %d/%d name=%s aborted: %s: %r", + i + 1, + len(grid), + variant.name, + build_error, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=build_error, + error_summary=repr(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"{build_error}: {exc!r}", + error_class=build_error, + note=variant.note, + ) + ) + + _mn_effective_args = _resolve_mn_effective_server_args( + cfg_path, + base_yaml_path, + variant, + base_extra_args=base_extra_args, + base_args_mode=base_args_mode, + ) + + if auto_warmup_requested: + try: + from ._server_lifecycle import resolve_lifecycle_params + + lifecycle = resolve_lifecycle_params(cfg_path) + auto_warmup = bool(lifecycle.get("eligible")) + if auto_warmup: + cfg_path = _build_variant_yaml( + base_yaml_path, + base_extra_args, + variant, + output_subdir=slot, + model_path=model_path, + gpu_type=gpu_type, + benchmark_script=benchmark_script, + server_lifecycle={ + "cleanup": True, + "pid_dir": str(slot), + "port": int(lifecycle.get("port") or 0), + }, + base_args_mode=base_args_mode, + base_extra_envs=base_extra_envs, + base_remove_args=base_remove_args, + base_unset_envs=base_unset_envs, + ) + else: + log.info( + "grid_runner: warmup-before-measure not eligible (%s); running single measured round.", + lifecycle.get("reason") or "unknown", + ) + except Exception as exc: # noqa: BLE001 + log.warning( + "grid_runner: warmup-before-measure eligibility/materialization failed (%r); running single measured round.", + exc, + ) + auto_warmup = False + + warmup_tput: float | None = None + if auto_warmup: + warmup_slot = slot / "warmup_round" + warmup_server_log = warmup_slot / "server.log" + warmup_lifecycle = { + "cleanup": False, + "pid_dir": str(slot), + "port": int(lifecycle.get("port") or 0), + } + try: + warmup_cfg_path = _build_variant_yaml( + base_yaml_path, + base_extra_args, + variant, + output_subdir=warmup_slot, + model_path=model_path, + gpu_type=gpu_type, + benchmark_script=benchmark_script, + server_lifecycle=warmup_lifecycle, + base_args_mode=base_args_mode, + base_extra_envs=base_extra_envs, + base_remove_args=base_remove_args, + base_unset_envs=base_unset_envs, + ) + except Exception as exc: # noqa: BLE001 + log.warning( + "grid_runner: variant %d/%d name=%s aborted: warmup_yaml_build_error: %r", + i + 1, + len(grid), + variant.name, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="warmup_yaml_build_error", + error_summary=repr(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"warmup_yaml_build_error: {exc!r}", + error_class="warmup_yaml_build_error", + note=variant.note, + ) + ) + + warmup_workspaces_before = snapshot_workspaces(warmup_slot) + warmup_started_unix = time.time() + # Held in a local because the abort line below has to name the cap the round was actually granted: the + # declared one is a hang backstop, and a round killed at the reserved cap logged as a two-hour timeout + # reads as a variant that hangs rather than a budget that ran out. + warmup_cap_sec = benchmark_timeout_sec + try: + warmup_rc, warmup_stdout, warmup_stderr = await _reported_magpie( + i, + "warmup", + magpie_python=magpie_python, + config_path=warmup_cfg_path, + output_dir=warmup_slot, + timeout_sec=warmup_cap_sec, + silence_timeout_sec=silence_timeout_sec, + cwd=cwd, + result_dir=result_dir, + serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + ) + except subprocess.TimeoutExpired as exc: + _teardown_variant_server(slot, lifecycle) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: warmup timeout (timeout_sec=%d): %s", + i + 1, + len(grid), + variant.name, + warmup_cap_sec, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="warmup_magpie_timeout", + error_summary=str(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"warmup_timeout: {exc}", + error_class="warmup_magpie_timeout", + server_log_path=_existing_log_path(warmup_server_log), + note=variant.note, + runtime_sec=round(max(0.0, time.time() - warmup_started_unix), 2), + nonfatal_warnings=["run_grid_warmup_round_failed"], + ) + ) + + warmup_stopped = stopped_by_the_run(warmup_rc) + if warmup_stopped is not None: + warmup_survivors = _teardown_variant_server(slot, lifecycle) + grid_is_over = _record_round_stop( + warmup_stopped, + idx=i, + variant=variant, + slot=slot, + round_label="warmup", + returncode=warmup_rc, + started_unix=warmup_started_unix, + server_log=warmup_server_log, + server_survivors=warmup_survivors, + ) + await _report_finished_variant(i) + if grid_is_over: + break + continue + + warmup_run_ws = select_run_workspace(warmup_slot, known_before=warmup_workspaces_before) + warmup_workspace = warmup_run_ws if warmup_run_ws is not None else warmup_slot + warmup_harvested = harvest_leaked_artifacts( + warmup_workspace, + subprocess_started_unix=warmup_started_unix, + ) + if warmup_run_ws is not None: + _, warmup_measurement = await _settled_measurement( + warmup_workspace, + subprocess_started_unix=warmup_started_unix, + settle_seconds=REPORT_SETTLE_SECONDS if warmup_rc == 0 else 0.0, + ) + else: + warmup_measurement = extract_benchmark_measurement( + None, + workspace=warmup_workspace, + subprocess_started_unix=warmup_started_unix, + ) + if warmup_rc != 0 or not warmup_measurement.get("valid_measurement"): + _teardown_variant_server(slot, lifecycle) + warmup_error = ( + server_log_death_excerpt(str(warmup_server_log)) + or redact_secret_values((warmup_stderr or warmup_stdout)[-2000:]) + if warmup_rc != 0 + else "warmup benchmark_report missing valid throughput/completed requests" + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: warmup_round_failed (rc=%s): %s", + i + 1, + len(grid), + variant.name, + warmup_rc, + warmup_error[:200], + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="warmup_round_failed", + error_summary=warmup_error, + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + workspace=str(warmup_workspace) if warmup_run_ws is not None else None, + report_path=( + str(warmup_workspace / "benchmark_report.json") + if (warmup_workspace / "benchmark_report.json").exists() + else None + ), + raw_result_path=warmup_measurement.get("raw_result_path"), + reported_success=warmup_measurement.get("reported_success"), + returncode=warmup_rc, + error=warmup_error, + error_class="warmup_round_failed", + server_log_path=_existing_log_path(warmup_server_log), + note=variant.note, + runtime_sec=round(max(0.0, time.time() - warmup_started_unix), 2), + nonfatal_warnings=[ + "run_grid_warmup_round_failed", + *[f"harvested_leaked_artifact:{src}" for src, _ in warmup_harvested], + ], + ) + ) + warmup_tput = warmup_measurement.get("output_throughput") + log.info( + "grid_runner: variant %s warmup tput=%.1f tok/s discarded; measuring hot round next", + variant.name, + warmup_tput or 0.0, + ) + + from ._multi_node_env import log_mn_banner + + log_mn_banner( + "grid_runner", + log, + variant=f"{i + 1}/{len(grid)}:{variant.name}", + ) + log.info( + "grid_runner: variant %d/%d name=%s args=%s", + i + 1, + len(grid), + variant.name, + variant.extra_server_args, + ) + + # Multi-node only: restart sglang/vllm with this variant's flags so each row runs against a fresh server. + from ._multi_node_server_lifecycle import ( + ServerRestartFailed, + restart_server_for_round, + ) + + try: + # PD knobs auto-resolved from $PD_* env; PD config stays constant across variants within one run. + _variant_unset = [str(k) for k in getattr(variant, "unset_envs", []) or [] if str(k).strip()] + _restart_env = _mn_restart_env(_mn_ref_envs, variant, _variant_unset) + await restart_server_for_round( + extra_server_args=_mn_effective_args, + # Reference recipe envs under this variant's own overrides (e.g. MORI_* MoE-dispatch tuning) so + # server-side env knobs proposed by specialists take effect on the restarted sglang without dropping + # the envs the baseline was measured with. + extra_env=_restart_env, + unset_env=_variant_unset, + model_path=model_path, + ep=int(os.environ.get("EP") or 0) or None, + ) + except ServerRestartFailed as exc: + log.warning( + "grid_runner: variant %d/%d name=%s aborted: mn_server_restart_failed: %s", + i + 1, + len(grid), + variant.name, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="mn_server_restart_failed", + error_summary=str(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"mn_server_restart_failed: {exc}", + error_class="mn_server_restart_failed", + note=variant.note, + ) + ) + + # The restart above is a launch of its own and is under no cap, so the budget admitted at the top of the loop + # may no longer be there. + if _skip_rest_for_budget( + i, + spent_on="this variant's server restart", + rounds_left=1 + _mn_warmup_rounds, + ): + if auto_warmup: + _teardown_variant_server(slot, lifecycle) + break + + # Multi-node client warmup: one discarded benchmark pass against the just-restarted, persistent remote server + # to warm JIT / steady-state before the measured pass. + from ._multi_node_env import ( + is_multi_node as _mn_imn, + mn_bench_warmup_enabled as _mn_warm, + ) + + if _mn_imn() and _mn_warm(): + _mn_warm_slot = slot / "mn_warmup" + _mn_warm_started_unix = time.time() + # The measurement is discarded, but the returncode is not: a warmup the run stopped is the same stop as + # one in the measured round, and discarding it launches the measured round after the cancel. + _mn_warm_rc: int | None = None + _mn_warm_rc, _, _ = await _reported_magpie( + i, + "mn_warmup", + magpie_python=magpie_python, + config_path=cfg_path, + output_dir=_mn_warm_slot, + timeout_sec=benchmark_timeout_sec, + silence_timeout_sec=silence_timeout_sec, + server_already_ready=True, + cwd=cwd, + result_dir=None, + serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + ) + log.info( + "grid_runner: MN warmup pass done (discarded) %d/%d name=%s rc=%s", + i + 1, + len(grid), + variant.name, + _mn_warm_rc, + ) + _mn_warm_stopped = stopped_by_the_run(_mn_warm_rc) + if _mn_warm_stopped is not None: + _mn_warm_survivors = _teardown_variant_server(slot, lifecycle if auto_warmup else None) + grid_is_over = _record_round_stop( + _mn_warm_stopped, + idx=i, + variant=variant, + slot=slot, + round_label="mn_warmup", + returncode=_mn_warm_rc, + started_unix=_mn_warm_started_unix, + server_log=_mn_warm_slot / "server.log", + server_survivors=_mn_warm_survivors, + ) + await _report_finished_variant(i) + if grid_is_over: + break + continue + + # Snapshot wall-clock before launch so the salvage path can mtime-gate leak destinations per-variant. + slot_workspaces_before = snapshot_workspaces(slot) + variant_started_unix = time.time() + measure_cap_sec = benchmark_timeout_sec + try: + rc, stdout, stderr = await _reported_magpie( + i, + "benchmark", + magpie_python=magpie_python, + config_path=cfg_path, + output_dir=slot, + timeout_sec=measure_cap_sec, + cwd=cwd, + result_dir=result_dir, + silence_timeout_sec=silence_timeout_sec, + server_already_ready=(server_already_ready or auto_warmup or _mn_imn()), + serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + ) + except subprocess.TimeoutExpired as exc: + # Harvest pre-timeout leaks. + to_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot + to_harvested = harvest_leaked_artifacts( + to_destination, + subprocess_started_unix=variant_started_unix, + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: magpie timeout (timeout_sec=%d): %s", + i + 1, + len(grid), + variant.name, + measure_cap_sec, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="magpie_timeout", + error_summary=str(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"timeout: {exc}", + error_class="magpie_timeout", + server_log_path=_existing_log_path(server_log), + note=variant.note, + runtime_sec=round( + max(0.0, time.time() - variant_started_unix), + 2, + ), + nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in to_harvested], + ) + ) + finally: + round_survivors = _teardown_variant_server(slot, lifecycle if auto_warmup else None) + + # Eval bounds could not be installed for a variant that runs eval, so nothing launched. + if rc == EVAL_PROBE_UNPATCHABLE_RETURNCODE: + variant_runtime_sec = round(max(0.0, time.time() - variant_started_unix), 2) + log.error( + "grid_runner: variant %d/%d name=%s aborted: eval_probe_unpatchable: %s", + i + 1, + len(grid), + variant.name, + stderr, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="eval_probe_unpatchable", + error_summary=stderr, + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + returncode=rc, + runtime_sec=variant_runtime_sec, + error=stderr, + error_class="eval_probe_unpatchable", + note=variant.note, + ) + ) + + # Server-liveness watchdog fired: engine/worker bootstrap died but the parent hung. + if rc == SERVER_DEAD_RETURNCODE: + variant_runtime_sec = round( + max(0.0, time.time() - variant_started_unix), + 2, + ) + sd_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot + sd_harvested = harvest_leaked_artifacts( + sd_destination, + subprocess_started_unix=variant_started_unix, + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: " + "server_init_dead (engine/worker bootstrap failed; parent " + "hung) after %.1fs", + i + 1, + len(grid), + variant.name, + variant_runtime_sec, + ) + death_excerpt = server_log_death_excerpt(str(server_log)) or ( + "server engine/worker init failed; parent process hung and was reaped by the liveness watchdog" + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="server_init_dead", + error_summary=death_excerpt, + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + returncode=rc, + runtime_sec=variant_runtime_sec, + error=death_excerpt, + error_class="server_init_dead", + server_log_path=_existing_log_path(server_log), + note=variant.note, + nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in sd_harvested], + ) + ) + + # Detokenizer-stall watchdog fired: the server came up healthy but then went silent (hung engine / wedged + # detokenizer). + if rc == DETOKENIZER_STALL_RETURNCODE: + variant_runtime_sec = round( + max(0.0, time.time() - variant_started_unix), + 2, + ) + ds_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot + ds_harvested = harvest_leaked_artifacts( + ds_destination, + subprocess_started_unix=variant_started_unix, + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: detokenizer_stall " + "(server ready but log went silent) after %.1fs", + i + 1, + len(grid), + variant.name, + variant_runtime_sec, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="detokenizer_stall", + error_summary=( + "server reported ready but emitted no log output (hung " + "engine / detokenizer stall); reaped by the " + "detokenizer-stall watchdog" + ), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + returncode=rc, + runtime_sec=variant_runtime_sec, + error="detokenizer_stall: server ready but log went silent", + error_class="detokenizer_stall", + server_log_path=_existing_log_path(server_log), + note=variant.note, + nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in ds_harvested], + ) + ) + + stopped = stopped_by_the_run(rc) + if stopped is not None: + grid_is_over = _record_round_stop( + stopped, + idx=i, + variant=variant, + slot=slot, + round_label="measured", + returncode=rc, + started_unix=variant_started_unix, + server_log=server_log, + server_survivors=round_survivors, + ) + await _report_finished_variant(i) + if grid_is_over: + break + continue + + workspace = select_run_workspace(slot, known_before=slot_workspaces_before) + # Always-on artifact harvest so each slot keeps its server.log / gpu_metrics / profile relay for Robustness + # RCA. + harvest_destination = workspace if workspace is not None else slot + harvested = harvest_leaked_artifacts( + harvest_destination, + subprocess_started_unix=variant_started_unix, + ) + if harvested: + log.info( + "_grid_runner: variant=%s harvested %d leaked artifact(s): %s", + variant.name, + len(harvested), + ", ".join(src.name for src, _ in harvested), + ) + if workspace is None: + harvest_tags = [f"harvested_leaked_artifact:{src}" for src, _ in harvested] + # An AgentX preflight abort never reaches Magpie, so of course no workspace exists -- but calling it + # ``no_benchmark_workspace`` erases the one fact that decides what to do next. + agentx_preflight_abort = rc == AGENTX_PREFLIGHT_RETURNCODE + if agentx_preflight_abort: + # Same class the baseline path uses for the same abort, so both routes classify it identically. + no_ws_error_class = AGENTX_PREFLIGHT_ERROR_CLASS + no_ws_error_summary = ( + redact_secret_values((stderr or "").strip()[-2000:]) or "AgentX preflight aborted the round" + ) + else: + no_ws_error_class = "no_benchmark_workspace" + no_ws_error_summary = server_log_death_excerpt(str(server_log)) or ( + redact_secret_values((stderr or stdout)[-2000:]) + if rc != 0 + else "no benchmark_* workspace produced" + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: %s (rc=%s)", + i + 1, + len(grid), + variant.name, + no_ws_error_class, + rc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=no_ws_error_class, + error_summary=no_ws_error_summary, + extra_args=variant.extra_server_args, + ) + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + returncode=rc, + error=no_ws_error_summary, + error_class=no_ws_error_class, + server_log_path=_existing_log_path(server_log), + nonfatal_warnings=harvest_tags, + note=variant.note, + ) + ) + await _report_finished_variant(i) + if agentx_preflight_abort: + # Environment, not variant. + log.error( + "grid_runner: variant %d/%d name=%s aborted on the AgentX preflight; " + "abandoning the remaining %d point(s) -- the client is missing for the " + "whole grid, not for this variant: %s", + i + 1, + len(grid), + variant.name, + len(grid) - (i + 1), + no_ws_error_summary, + ) + # Recorded rather than dropped: a grid that just returns fewer points than it declared reads as a grid + # that was that size. + results.extend( + VariantResult( + name=rest.name, + extra_server_args=rest.extra_server_args, + extra_envs=dict(rest.extra_envs), + status="skipped", + error="AgentX preflight aborted the grid before this variant ran", + error_class=AGENTX_PREFLIGHT_ERROR_CLASS, + note=rest.note, + ) + for rest in grid[i + 1 :] + ) + break + if rc != 0 and not keep_going_on_failure: + break + continue + report_path = workspace / "benchmark_report.json" + # The report is written during shutdown and the reader runs the moment the subprocess is reaped, so the read is + # worth waiting on however the process exited: a non-zero exit can still be a round that served its whole + # protocol, and that verdict is taken from this one read. + report, measurement = await _settled_measurement( + workspace, + subprocess_started_unix=variant_started_unix, + settle_seconds=REPORT_SETTLE_SECONDS, + ) + nonzero_kept_error: str | None = None + warnings = list(measurement.pop("nonfatal_warnings", []) or []) + for leak_src, _ in harvested: + warnings.append(f"harvested_leaked_artifact:{leak_src}") + if warmup_tput is not None: + warnings.append("run_grid_warmup_discarded_first") + warnings.append(f"warmup_round_tput:{float(warmup_tput):.1f}") + + if not measurement.get("valid_measurement"): + if lifecycle_boot_only and rc == 0: + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="succeeded", + workspace=str(workspace), + report_path=str(report_path) if report_path.exists() else None, + returncode=rc, + nonfatal_warnings=warnings, + note="server_lifecycle_boot_only", + ) + ) + await _report_finished_variant(i) + continue + death_excerpt = server_log_death_excerpt(str(server_log)) + if rc != 0: + error = death_excerpt or redact_secret_values((stderr or stdout)[-2000:]) + # The bypass/scriptable path runs the customer body in a child whose stderr is redirected to + # benchmark_stderr.log, not the parent pipe — so `stderr`/`stdout` here are often empty and the real + # diagnostic (e.g. argparse "unrecognized arguments") is only on disk. + if not error.strip(): + error = redact_secret_values(_on_disk_stderr_tail(workspace, slot)) + # Last resort: report.errors when the pipe and on-disk logs are all empty. + if not error.strip(): + error = redact_secret_values(_report_errors_summary(report)) + invalid_class = "magpie_nonzero_invalid_measurement" + elif not report: + error = death_excerpt or "benchmark_report missing" + invalid_class = "benchmark_report_missing" + else: + error = death_excerpt or "benchmark_report missing valid throughput/completed requests" + invalid_class = "benchmark_report_invalid_metric" + log.warning( + "grid_runner: variant %d/%d name=%s aborted: %s (rc=%s): %s", + i + 1, + len(grid), + variant.name, + invalid_class, + rc, + error[:200], + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=invalid_class, + error_summary=error, + extra_args=variant.extra_server_args, + ) + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + workspace=str(workspace), + report_path=str(report_path) if report_path.exists() else None, + raw_result_path=measurement.get("raw_result_path"), + reported_success=measurement.get("reported_success"), + returncode=rc, + nonfatal_warnings=warnings, + error=error, + error_class=invalid_class, + server_log_path=_existing_log_path(server_log), + note=variant.note, + ) + ) + await _report_finished_variant(i) + if rc != 0 and not keep_going_on_failure: + break + continue + + if rc != 0: + nonzero_error = redact_secret_values((stderr or stdout)[-2000:]) + if served_complete_protocol(measurement): + # Every request the client recorded as requested was served, so + # the exit code came from something the round had already + # finished with. The cause stays on the result: a reader + # comparing this point to its neighbours is owed the reason it + # is not a clean zero. + warnings.append(f"nonzero_rc_after_complete_protocol:{rc}") + nonzero_kept_error = nonzero_error + log.warning( + "grid_runner: variant %s exited %d after serving its whole protocol " + "(%s of %s requests); keeping the measurement: %s", + variant.name, + rc, + measurement.get("completed_requests"), + measurement.get("requested_requests"), + nonzero_error, + ) + else: + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="magpie_nonzero_after_valid_measurement", + error_summary=nonzero_error, + extra_args=variant.extra_server_args, + ) + log.warning( + "grid_runner: variant %s aborted: magpie_nonzero_after_valid_measurement " + "(rc=%d, served %s of %s requests)", + variant.name, + rc, + measurement.get("completed_requests"), + measurement.get("requested_requests"), + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + workspace=str(workspace), + report_path=str(report_path) if report_path.exists() else None, + raw_result_path=measurement.get("raw_result_path"), + reported_success=measurement.get("reported_success"), + returncode=rc, + nonfatal_warnings=warnings, + error=nonzero_error, + error_class="magpie_nonzero_after_valid_measurement", + server_log_path=None, + note=variant.note, + ) + ) + + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="succeeded", + output_throughput=measurement.get("output_throughput"), + request_throughput=measurement.get("request_throughput"), + total_token_throughput=measurement.get("total_token_throughput"), + completed_requests=measurement.get("completed_requests"), + duration_seconds=measurement.get("duration_seconds"), + ttft_mean_ms=measurement.get("ttft_mean_ms"), + **constraint_fields(measurement), + tpot_mean_ms=measurement.get("tpot_mean_ms"), + input_throughput=measurement.get("input_throughput"), + tpot_p90_ms=measurement.get("tpot_p90_ms"), + intvty_p90=measurement.get("e2e_norm_intvty_p90"), + intvty_p50=measurement.get("e2e_norm_intvty_p50"), + request_error_rate=measurement.get("request_error_rate"), + workspace=str(workspace), + report_path=str(report_path) if report_path.exists() else None, + raw_result_path=measurement.get("raw_result_path"), + reported_success=measurement.get("reported_success"), + returncode=rc, + nonfatal_warnings=warnings, + error=nonzero_kept_error, + server_log_path=None, + note=variant.note, + runtime_sec=round( + max(0.0, time.time() - variant_started_unix), + 2, + ), + ) + ) + log.info( + "grid_runner: variant %s tput=%.1f tok/s", + variant.name, + results[-1].output_throughput or 0.0, + ) + await _report_finished_variant(i) + except _VariantAborted as abort: + results.append(abort.result) + await _report_finished_variant(i) + if not keep_going_on_failure: + break + await asyncio.to_thread( + _attach_grid_launch_evidence, + results, + grid=grid, + output_root=output_root, + caller_reused_ready_server=server_already_ready, + ready_server_log=ready_server_log, + ) + return results + + +def _teardown_variant_server(slot: Path, lifecycle: dict[str, Any] | None) -> list[int]: + """Stop every server this variant booted for its own rounds; return the pids still alive after SIGKILL. + + ``lifecycle`` names the persistent server the rounds shared, ``None`` when they shared none. + """ + from ._server_lifecycle import stop_agentx_servers, teardown_lifecycle_server + + if lifecycle is not None: + teardown_lifecycle_server( + pid_dir=slot, + framework=str(lifecycle.get("framework") or ""), + port=int(lifecycle.get("port") or 0), + ) + return stop_agentx_servers(slot) + + +def _not_run_skip_result(variant: GridVariant, stopped: StoppedByTheRun) -> VariantResult: + """Build the ``skipped`` result for a variant the run never got to.""" + return VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="skipped", + error=stopped.never_started, + error_class=stopped.error_class, + note=variant.note, + ) + + +def _existing_log_path(path: Path) -> str | None: + """Return ``path`` as a string when it exists, else ``None``.""" + return str(path) if path.exists() else None + + +def _measurement_server_log_path( + server_log: Path, + workspace: Path | None = None, + *, + slot: Path | None = None, +) -> str | None: + """Return the server log attributable to this variant's measurement.""" + owning_slot = slot or server_log.parent + direct = _existing_log_path(server_log) + if not direct and workspace is not None: + direct = _existing_log_path(workspace / "server.log") + if direct: + return direct + warmup_dir = owning_slot / "warmup_round" + try: + candidates = [path for path in warmup_dir.glob("*/server.log") if path.is_file()] + except OSError: + candidates = [] + if not candidates: + return None + try: + return str(max(candidates, key=lambda path: path.stat().st_mtime)) + except OSError: + return None + + +def _attach_grid_launch_evidence( + results: list[VariantResult], + *, + grid: list[GridVariant], + output_root: Path, + caller_reused_ready_server: bool, + ready_server_log: str | None = None, +) -> None: + """Persist declared and observed launch evidence for each grid result.""" + from ._workload_envs import VLLM_SOURCE_ROOT_ENVS + + for idx, result in enumerate(results): + if idx >= len(grid): + break + # A skipped variant never launched a measurement. + if result.status == "skipped" or result.error_class in { + "capability_unsupported", + "yaml_build_error", + "warmup_yaml_build_error", + }: + continue + slot = output_root / f"variant_{idx:02d}_{_safe(grid[idx].name)}" + config_path = slot / "config.yaml" + workspace = Path(result.workspace) if result.workspace else None + primary_log = Path(result.server_log_path) if result.server_log_path else slot / "server.log" + actual_log = _measurement_server_log_path(primary_log, workspace, slot=slot) + if not actual_log and ready_server_log: + actual_log = _existing_log_path(Path(ready_server_log)) + result.server_log_path = actual_log + evidence = build_launch_evidence( + config_path=config_path, + actual_server_log=actual_log, + framework=os.environ.get("FRAMEWORK", ""), + slot=slot, + caller_reused_ready_server=caller_reused_ready_server, + ) + result.launch_evidence = evidence + result.launch_evidence_path = persist_launch_evidence(evidence, slot=slot) + declared_env = evidence["requested_server_env"].keys() - set(VLLM_SOURCE_ROOT_ENVS) + report = scan_server_log(actual_log, evidence["framework"], declared_env=declared_env) + persist_runtime_findings(report, slot=slot) + + +def _safe(name: str) -> str: + """Filesystem-safe slug for variant directory names.""" + return "".join(c if c.isalnum() or c in "-_." else "_" for c in name)[:60] + + +def _write_variant_abort_marker( + slot: Path, + *, + variant_name: str, + error_class: str, + error_summary: str, + extra_args: str = "", +) -> None: + """Write ``abort_reason.json`` into the variant slot directory.""" + try: + slot.mkdir(parents=True, exist_ok=True) + marker = { + "variant": variant_name, + "error_class": error_class, + "error": (error_summary or "")[:2000], + "extra_args": extra_args, + "aborted_at_utc": time.strftime( + "%Y-%m-%dT%H:%M:%SZ", + time.gmtime(), + ), + } + (slot / "abort_reason.json").write_text( + json.dumps(marker, indent=2, sort_keys=True), + encoding="utf-8", + ) + except OSError as exc: + log.warning( + "_grid_runner: failed to write abort_reason.json at %s: %s", + slot, + exc, + ) + + +def _report_errors_summary(report: dict[str, Any] | None, limit: int = 2000) -> str: + """Join ``benchmark_report.json`` ``errors`` into a single diagnostic.""" + if not isinstance(report, dict): + return "" + errors = report.get("errors") + if not isinstance(errors, list): + return "" + text = "; ".join(str(item).strip() for item in errors if str(item).strip()) + return text[-limit:] if text else "" + + +def _on_disk_stderr_tail(*dirs: Path, limit: int = 2000) -> str: + """Return the tail of the first non-empty on-disk benchmark log.""" + for d in dirs: + if not d: + continue + for name in ("benchmark_stderr.log", "benchmark_stdout.log"): + try: + p = d / name + if not p.is_file(): + continue + text = p.read_text(encoding="utf-8", errors="replace").strip() + if text: + return f"[{name}] {text[-limit:]}" + except OSError: + continue + return "" + + +__all__ = [ + "DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC", + "GridVariant", + "ORCHESTRATOR_CANCELLED_CLASS", + "SESSION_TIME_EXHAUSTED_CLASS", + "StoppedByTheRun", + "SGLANG_WATCHDOG_TIMEOUT_ENV", + "VariantResult", + "apply_multi_node_invalid_variants", + "apply_aiter_moe_pin_filter", + "apply_runtime_benchmark_overrides", + "reorder_grid_for_multi_node", + "inject_sglang_attention_backend", + "inject_sglang_context_length", + "inject_sglang_watchdog_timeout", + "merge_server_args", + "remove_server_args", + "resolve_sglang_watchdog_timeout", + "run_grid", + "sanitize_result_dir", + "sanitize_script_name", + "server_args_env_name", + "session_grid_bounds", + "stopped_by_the_run", + # Re-exported from the sibling modules to keep the namespace intact. + "_resolve_magpie_python", + "_resolve_probe_python", + "coerce_extra_envs", + "compact_json_server_args", + "_SPACE_VALUE_FLAGS", + "_MULTI_VALUE_FLAGS", + "_VLLM_SINGLE_VALUE_FLAGS", + "dedup_vllm_server_args", + "_SGLANG_WATCHDOG_FLAG", + "_SGLANG_WATCHDOG_RE", + "DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS", + "DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS", + "SGLANG_CONTEXT_HEADROOM_ENV", + "SGLANG_CONTEXT_FLOOR_ENV", + "_SGLANG_CONTEXT_LENGTH_FLAG", + "_SGLANG_CONTEXT_LENGTH_RE", + "_SGLANG_ATTN_BACKEND_FLAG", + "_SGLANG_ATTN_BACKEND_RE", + "_SGLANG_DUAL_CHUNK_BACKEND", + "_resolve_nonneg_int_env", + "resolve_sglang_context_cap", + "_resolve_dual_chunk_backend", + "_SGLANG_MOE_RUNNER_BACKEND_FLAG", + "_SGLANG_MOE_RUNNER_BACKEND_RE", + "moe_runner_requires_aiter", + "resolve_skip_spec", + "_parse_skip_spec", + "_RE_CUDA_GRAPH_MAX_BS", + "_MN_PARAMS_PRIORITY", + "_MN_BACKENDS_PRIORITY", + "_mn_priority_index", + "_COMPATIBILITY_FLAG_RULES", + "_XDIT_ENV_BLACKLIST", + "_XDIT_ENV_COMBO_BLACKLIST", + "xdit_blacklist_reason", + "_detect_model_class", + "apply_compatibility_filter", + "apply_user_skip_list", +] From 4ebc9c78058f6fef80ba2f62e7ab3598d0d7229d Mon Sep 17 00:00:00 2001 From: luochen Date: Tue, 20 Oct 2026 10:19:43 +0800 Subject: [PATCH 3/4] Restore LF line endings in the grid runner and its tests --- .../test_grid_runner_helpers_coverage_unit.py | 1074 ++-- .../actions/executors/_grid_runner.py | 4578 ++++++++--------- 2 files changed, 2826 insertions(+), 2826 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py index 017bb4a456..ec8bdca016 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py @@ -1,537 +1,537 @@ -# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Supplemental coverage for _grid_runner pure helpers: compatibility filter model-class drop, runtime override env branches, report parsing, and per-variant yaml env injection.""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path - -import pytest -import yaml - -from hyperloom.orchestrator.actions.executors import _grid_runner as gr -from hyperloom.orchestrator.actions.executors import _benchmark_interpreter as bi - -# Patch compatibility-filter helpers in the ``_grid_variant_filter`` sibling, where apply_compatibility_filter -# resolves them (not via the re-export). -from hyperloom.orchestrator.actions.executors import _grid_variant_filter as vf - - -def _variant(name: str, args: str = "", envs: dict | None = None) -> gr.GridVariant: - return gr.GridVariant(name=name, extra_server_args=args, extra_envs=envs or {}) - - -def test_validate_magpie_python_override_requires_python(tmp_path): - py = tmp_path / "python3" - py.write_text("", encoding="utf-8") - bash = tmp_path / "bash" - bash.write_text("", encoding="utf-8") - assert gr._validate_magpie_python_override(str(py)) == str(py) - with pytest.raises(ValueError, match="Python interpreter"): - gr._validate_magpie_python_override(str(bash)) - assert gr._validate_magpie_python_override(sys.executable).endswith(Path(sys.executable).name) - - -# -- apply_compatibility_filter ------------------------------------------- -def test_compatibility_filter_drops_on_model_class(monkeypatch) -> None: - monkeypatch.setattr(vf, "_detect_model_class", lambda mp: (False, False)) - grid = [_variant("mla", "--enable-flashinfer-mla"), _variant("plain", "")] - kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="meta-llama-3-8b") - assert [v.name for v in kept] == ["plain"] - assert len(dropped) == 1 - assert dropped[0]["source"] == "compatibility_filter" - assert "MLA" in dropped[0]["reason"] - - -def test_compatibility_filter_keeps_a_flag_the_model_class_supports(monkeypatch) -> None: - # Whether the installed server accepts the flag is left to its own argparse at launch. - monkeypatch.setattr(vf, "_detect_model_class", lambda mp: (True, True)) - grid = [_variant("moe", "--enable-ep-moe")] - kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="deepseek-moe") - assert [v.name for v in kept] == ["moe"] and dropped == [] - - -def test_compatibility_filter_no_model_path_assumes_compatible() -> None: - grid = [_variant("moe", "--enable-ep-moe")] - kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="") - assert [v.name for v in kept] == ["moe"] and dropped == [] - - -# -- ungrounded_env_file_reason (env-value file grounding) ---------------- -def test_ungrounded_env_file_reason_drops_missing_config_path(tmp_path) -> None: - # A proposal may point a config env at an artifact no stage authored; the server would read it - # unconditionally and die at config load, so the variant must be dropped with a reason. - v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(tmp_path / "nope.csv")}) - reason = gr.ungrounded_env_file_reason(v) - assert reason and "AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE" in reason - assert str(tmp_path / "nope.csv") in reason - - -def test_ungrounded_env_file_reason_passes_authored_file(tmp_path) -> None: - csv = tmp_path / "tuned.csv" - csv.write_text("gfx,M,N,K,splitK\ngfx950,64,5120,5120,2\n", encoding="utf-8") - v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(csv)}) - assert gr.ungrounded_env_file_reason(v) is None - - -def test_ungrounded_env_file_reason_accepts_colon_joined_existing_files(tmp_path) -> None: - # A set AITER_CONFIG_* is aiter's SET form: a `:`-joined list, each element loaded (csvs_aiter_will_load). - # Both files on disk is a valid variant and must not be dropped. - a = tmp_path / "t.csv" - a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") - b = tmp_path / "t2.csv" - b.write_text("gfx,M,N,K\ngfx950,4,5,6\n", encoding="utf-8") - v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{b}"}) - assert gr.ungrounded_env_file_reason(v) is None - - -def test_ungrounded_env_file_reason_drops_colon_joined_missing_element(tmp_path) -> None: - a = tmp_path / "t.csv" - a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") - missing = str(tmp_path / "never_authored.csv") - v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{missing}"}) - reason = gr.ungrounded_env_file_reason(v) - assert reason and missing in reason and str(a) not in reason.split("element(s)")[1].split(" do not exist")[0] - - -def test_ungrounded_env_file_reason_treats_an_unreadable_path_as_missing(monkeypatch) -> None: - # An unreadable parent or an overlong name makes Path.is_file() raise; the variant is dropped, the grid runs on. - def raise_eacces(self): - raise PermissionError(13, "Permission denied") - - monkeypatch.setattr(gr.Path, "is_file", raise_eacces) - v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": "/root/private/tuned.csv"}) - reason = gr.ungrounded_env_file_reason(v) - assert reason and "/root/private/tuned.csv" in reason - - -def test_ungrounded_env_file_reason_noop_without_config_envs(tmp_path) -> None: - # Non-file envs (even under aiter-ish prefixes like the JSON-payload AITER_CONFIG) must stay untouched. - v = _variant( - "plain", - envs={ - "AITER_CONFIG": '{"path": "/tmp/never_scanned.json"}', - "SGLANG_USE_AITER": "1", - "AITER_CONFIG_GEMM_A8W8": "", - }, - ) - assert gr.ungrounded_env_file_reason(v) is None - - -# -- unsupported_capability_reason (env-flag build probe) ----------------- -def _clear_cap_cache() -> None: - gr._CAP_PROBE_CACHE.clear() - - -def test_capability_reason_noop_when_flag_absent(monkeypatch) -> None: - monkeypatch.setenv("FRAMEWORK", "vllm") - _clear_cap_cache() - # Probe must NOT even run when the flag isn't set on the variant. - monkeypatch.setattr( - gr, - "_probe_vllm_aiter_shared_expert_unsupported", - lambda: (_ for _ in ()).throw(AssertionError("probe should not run")), - ) - assert gr.unsupported_capability_reason(_variant("plain")) is None - - -def test_capability_reason_noop_for_non_vllm_framework(monkeypatch) -> None: - monkeypatch.setenv("FRAMEWORK", "sglang") - _clear_cap_cache() - v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1"}) - assert gr.unsupported_capability_reason(v) is None - - -def test_capability_reason_drops_when_module_missing(monkeypatch) -> None: - monkeypatch.setenv("FRAMEWORK", "vllm") - _clear_cap_cache() - monkeypatch.setattr( - gr, - "_probe_vllm_aiter_shared_expert_unsupported", - lambda: "missing module(s): vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe", - ) - v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1"}) - reason = gr.unsupported_capability_reason(v) - assert reason is not None and "rocm_aiter_fused_moe" in reason - - -def test_capability_reason_falsey_flag_not_probed(monkeypatch) -> None: - monkeypatch.setenv("FRAMEWORK", "vllm") - _clear_cap_cache() - monkeypatch.setattr( - gr, - "_probe_vllm_aiter_shared_expert_unsupported", - lambda: (_ for _ in ()).throw(AssertionError("probe should not run")), - ) - v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0"}) - assert gr.unsupported_capability_reason(v) is None - - -def test_probe_caches_ok_and_unsupported(monkeypatch) -> None: - _clear_cap_cache() - calls = {"n": 0} - - class _Proc: - def __init__(self, out: str) -> None: - self.stdout = out - self.stderr = "" - - def fake_run(*_a, **_k): - calls["n"] += 1 - return _Proc(json.dumps({"status": "ok"})) - - monkeypatch.setattr(gr.subprocess, "run", fake_run) - assert gr._probe_vllm_aiter_shared_expert_unsupported() is None - # Second call must hit the cache (no second subprocess). - assert gr._probe_vllm_aiter_shared_expert_unsupported() is None - assert calls["n"] == 1 - - -def test_probe_unknown_not_cached(monkeypatch) -> None: - _clear_cap_cache() - - class _Proc: - stdout = json.dumps({"status": "unknown"}) - stderr = "" - - monkeypatch.setattr(gr.subprocess, "run", lambda *_a, **_k: _Proc()) - assert gr._probe_vllm_aiter_shared_expert_unsupported() is None - assert "vllm" not in gr._CAP_PROBE_CACHE - - -def test_probe_swallows_subprocess_error(monkeypatch) -> None: - _clear_cap_cache() - - def boom(*_a, **_k): - raise OSError("no python3") - - monkeypatch.setattr(gr.subprocess, "run", boom) - assert gr._probe_vllm_aiter_shared_expert_unsupported() is None - - -# -- _resolve_probe_python / probe interpreter selection ------------------ -def test_resolve_probe_python_prefers_magpie_interpreter(monkeypatch) -> None: - # The harness interpreter is used directly; no vllm-exe resolution is tried. - monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/srv/venv/bin/python") - monkeypatch.setattr( - gr.shutil, "which", lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("which() should not be called")) - ) - assert gr._resolve_probe_python() == "/srv/venv/bin/python" - - -def test_resolve_probe_python_falls_back_to_vllm_venv(monkeypatch) -> None: - # magpie_python is the canonical default -> pin the venv that backs ``vllm serve``. - monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/opt/venv/bin/python") - monkeypatch.setattr(gr.shutil, "which", lambda name: "/other/venv/bin/vllm" if name == "vllm" else None) - monkeypatch.setattr(gr.os.path, "exists", lambda p: p == "/other/venv/bin/python") - assert gr._resolve_probe_python() == "/other/venv/bin/python" - - -def test_resolve_probe_python_no_bare_python3_fallback(monkeypatch) -> None: - # With no resolvable vllm exe, fall back to the canonical magpie default — never a bare "python3". - monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/opt/venv/bin/python") - monkeypatch.setattr(gr.shutil, "which", lambda *_a, **_k: None) - assert gr._resolve_probe_python() == "/opt/venv/bin/python" - - -def test_probe_invokes_resolved_interpreter(monkeypatch) -> None: - # The capability probe must run under the resolved interpreter, not "python3". - _clear_cap_cache() - monkeypatch.setattr(gr, "_resolve_probe_python", lambda: "/srv/venv/bin/python") - seen: dict = {} - - class _Proc: - stdout = json.dumps({"status": "ok"}) - stderr = "" - - def fake_run(cmd, *_a, **_k): - seen["cmd"] = list(cmd) - return _Proc() - - monkeypatch.setattr(gr.subprocess, "run", fake_run) - gr._probe_vllm_aiter_shared_expert_unsupported() - assert seen["cmd"][0] == "/srv/venv/bin/python" - assert seen["cmd"][0] != "python3" - - -# -- apply_runtime_benchmark_overrides ------------------------------------ -def test_runtime_overrides_model_precision_and_gpu_no_framework_agent(monkeypatch) -> None: - monkeypatch.setenv("PRECISION", "fp8") - for k in ("ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): - monkeypatch.delenv(k, raising=False) - bench: dict = {} # no framework -> benchmark_script popped - gr.apply_runtime_benchmark_overrides( - bench, - model_path="/models/x", - gpu_type="mi355x", - ) - assert bench["model"] == "/models/x" - assert bench["precision"] == "fp8" - assert bench["runner_type"] == "mi355x" - assert "benchmark_script" not in bench - - -def test_runtime_overrides_framework_pins_generic_script(monkeypatch) -> None: - for k in ("PRECISION", "ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): - monkeypatch.delenv(k, raising=False) - bench = {"framework": "sglang"} - gr.apply_runtime_benchmark_overrides(bench, gpu_type="mi300x") - assert bench["benchmark_script"] == "sglang_mi300x.sh" - - -def test_runtime_overrides_env_ints_and_rocr_autofill(monkeypatch) -> None: - for k in ("PRECISION",): - monkeypatch.delenv(k, raising=False) - monkeypatch.setenv("ISL", "128") - monkeypatch.setenv("TP", "2") - monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising=False) - bench: dict = {} - envs = gr.apply_runtime_benchmark_overrides(bench) - assert envs["ISL"] == 128 - assert envs["TP"] == 2 - # TP>1 with no explicit ROCR -> auto-filled device list - assert envs["ROCR_VISIBLE_DEVICES"] == "0,1" - - -def test_runtime_overrides_explicit_benchmark_script_wins(monkeypatch) -> None: - for k in ("PRECISION", "ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): - monkeypatch.delenv(k, raising=False) - bench = {"framework": "vllm"} - gr.apply_runtime_benchmark_overrides( - bench, - gpu_type="mi300x", - benchmark_script="custom.sh", - ) - assert bench["benchmark_script"] == "custom.sh" - - -# -- _parse_report --------------------------------------------------------- -def test_parse_report_missing(tmp_path: Path) -> None: - assert gr._parse_report(tmp_path) is None - - -def test_parse_report_invalid_json(tmp_path: Path) -> None: - (tmp_path / "benchmark_report.json").write_text("{bad", encoding="utf-8") - assert gr._parse_report(tmp_path) is None - - -def test_parse_report_non_dict(tmp_path: Path) -> None: - (tmp_path / "benchmark_report.json").write_text("[1,2,3]", encoding="utf-8") - assert gr._parse_report(tmp_path) is None - - -def test_parse_report_valid(tmp_path: Path) -> None: - (tmp_path / "benchmark_report.json").write_text( - json.dumps({"output_throughput": 100.0}), - encoding="utf-8", - ) - assert gr._parse_report(tmp_path) == {"output_throughput": 100.0} - - -# -- _build_variant_yaml --------------------------------------------------- -def test_build_variant_yaml_injects_extra_envs(tmp_path: Path) -> None: - base = tmp_path / "base.yaml" - base.write_text( - yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), - encoding="utf-8", - ) - variant = _variant("v1", "--foo 1", envs={"USE_AITER": "1"}) - out = gr._build_variant_yaml( - base, - "", - variant, - output_subdir=tmp_path / "v1", - ) - cfg = yaml.safe_load(out.read_text(encoding="utf-8")) - envs = cfg["benchmark"]["envs"] - assert envs["USE_AITER"] == "1" - # variant + base server args merged into the framework's args env - arg_key = gr.server_args_env_name("sglang") - assert "--foo 1" in envs[arg_key] - - -def test_build_variant_yaml_dedupes_repeated_flags(tmp_path: Path) -> None: - """When base YAML + base_extra_args + variant all set the same flag, the materialized YAML must contain each flag - only once (last wins). - """ - base = tmp_path / "base.yaml" - base.write_text( - yaml.safe_dump( - { - "benchmark": { - "framework": "vllm", - "envs": {"EXTRA_VLLM_ARGS": "--attention-backend ROCM_ATTN"}, - }, - } - ), - encoding="utf-8", - ) - variant = _variant("v1", "--attention-backend ROCM_AITER_FA") - out = gr._build_variant_yaml( - base, - "", - variant, - output_subdir=tmp_path / "v1", - ) - cfg = yaml.safe_load(out.read_text(encoding="utf-8")) - args = cfg["benchmark"]["envs"]["EXTRA_VLLM_ARGS"] - assert args.count("--attention-backend") == 1, f"duplicate flag: {args}" - assert "ROCM_AITER_FA" in args, "last-wins should keep variant value" - - -def test_build_variant_yaml_prepends_legit_overlay(tmp_path: Path) -> None: - # SWSPLAT-42358: a legitimate overlay is a single existing directory; it is prepended to PYTHONPATH unchanged. - base = tmp_path / "base.yaml" - base.write_text( - yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), - encoding="utf-8", - ) - overlay = tmp_path / "overlay" - overlay.mkdir() - variant = _variant("v1") - variant.overlay_pythonpath = str(overlay) # type: ignore[attr-defined] - out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") - envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] - assert envs.get("PYTHONPATH", "").startswith(str(overlay)) - - -@pytest.mark.parametrize( - ("config_envs", "tail"), - [({}, ["/inh/a", "/inh/b"]), ({"PYTHONPATH": "/cfg"}, ["/cfg"])], - ids=["inherited", "config-authoritative"], -) -def test_build_variant_yaml_overlay_builds_on_the_launch_pythonpath(tmp_path, monkeypatch, config_envs, tail) -> None: - # The launch overlays benchmark.envs on os.environ, so a config without PYTHONPATH carries the inherited one. - # The overlay is relative so the path carries no drive colon on any host. - import os - - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/inh/a", "/inh/b"])) - (tmp_path / "overlay").mkdir() - base = tmp_path / "base.yaml" - base.write_text(yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": config_envs}}), encoding="utf-8") - variant = _variant("v1") - variant.overlay_pythonpath = "overlay" # type: ignore[attr-defined] - out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") - envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] - assert envs["PYTHONPATH"] == os.pathsep.join(["overlay", *tail]) - - -def test_build_variant_yaml_drops_unsafe_overlay(tmp_path: Path) -> None: - # SWSPLAT-42358: a ``:``-joined / traversal / non-existent overlay is dropped (would otherwise smuggle extra - # PYTHONPATH entries or escape). - base = tmp_path / "base.yaml" - base.write_text( - yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), - encoding="utf-8", - ) - overlay = tmp_path / "overlay" - overlay.mkdir() - for bad in (f"{overlay}:/etc", f"{overlay}/../evil", str(tmp_path / "missing")): - variant = _variant("v1") - variant.overlay_pythonpath = bad # type: ignore[attr-defined] - out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") - envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] - assert "PYTHONPATH" not in envs or bad not in envs.get("PYTHONPATH", ""), bad - - -def test_shell_safe_dedupe_preserves_json_and_dedupes_other_flags() -> None: - """JSON stays an opaque argv token while unrelated flags dedupe last-wins.""" - args = '--json-model-override-args {"rope_scaling":null} --context-length 8192 --context-length 4096' - out = gr._shell_safe_dedupe(args) - assert out == '--json-model-override-args {"rope_scaling":null} --context-length 4096' - assert json.loads(out.split("--json-model-override-args ", 1)[1].split(" ", 1)[0]) == {"rope_scaling": None} - - -def test_shell_safe_dedupe_leaves_multi_value_arg_untouched() -> None: - """Multi-token flags must not be reassembled as stray positional tokens.""" - args = "--cuda-graph-bs 1 2 4 --cuda-graph-bs 8 16" - out = gr._shell_safe_dedupe(args) - assert out == args - - -def test_shell_safe_dedupe_normalizes_equals_form() -> None: - """--flag=value and --flag value must dedupe to one (last wins).""" - out = gr._shell_safe_dedupe("--attention-backend=ROCM_ATTN --attention-backend ROCM_AITER_FA") - assert out.count("--attention-backend") == 1, out - assert "ROCM_AITER_FA" in out, "last-wins should keep the later value" - assert "ROCM_ATTN" not in out - - -def test_shell_safe_dedupe_simple_last_wins() -> None: - out = gr._shell_safe_dedupe("--tp 1 --tp 8 --mem-fraction-static 0.9") - assert out == "--tp 8 --mem-fraction-static 0.9" - - -def test_compose_server_args_replace_still_applies_remove_args() -> None: - out = gr.compose_server_args( - inherited_args="--bad-base 1", - base_extra_args="--also-bad 2", - variant_extra_args="--bad-base 3 --keep 4", - remove_args=["--bad-base"], - args_mode="replace", - ) - assert out == "--also-bad 2 --keep 4" - - -def test_compose_server_args_launches_with_the_harness_flags_the_baseline_ran_with() -> None: - out = gr.compose_server_args( - inherited_args="--no-enable-prefix-caching --block-size 128", - base_extra_args="--no-enable-prefix-caching", - variant_extra_args="--kv-cache-dtype fp8 --no-enable-prefix-caching", - ) - assert "--no-enable-prefix-caching" in out - assert "--block-size 128" in out - assert "--kv-cache-dtype fp8" in out - - -def test_compose_server_args_keeps_the_harness_flag_in_replace_mode() -> None: - out = gr.compose_server_args( - inherited_args="--ignored", - base_extra_args="--no-enable-prefix-caching --max-num-seqs 256", - variant_extra_args="--kv-cache-dtype fp8", - args_mode="replace", - ) - assert out == "--no-enable-prefix-caching --max-num-seqs 256 --kv-cache-dtype fp8" - - -def test_launch_keeps_what_the_persisted_recipe_strips() -> None: - from hyperloom.inference_optimizer.grid_server_args import strip_benchmark_harness_flags - - launched = gr.compose_server_args(base_extra_args="--no-enable-prefix-caching --max-num-seqs 256") - assert "--no-enable-prefix-caching" in launched - assert strip_benchmark_harness_flags(launched) == "--max-num-seqs 256" - - -def test_remove_server_args_accepts_multi_flag_string() -> None: - out = gr.remove_server_args( - "--flag-a --flag-b --flag-c 3 --keep 4", - "--flag-a --flag-b --flag-c", - ) - assert out == "--keep 4" - - -def test_remove_server_args_keeps_a_sibling_json_value_parseable() -> None: - """Removing one flag must not corrupt a JSON-valued sibling.""" - args = ( - '--online_quant_config {"global_quant_config":"ptpc_fp8",' - '"exclude_layer":["*.mlp.gate","*expert*"]} ' - "--no-enable-prefix-caching --tp 8" - ) - out = gr.remove_server_args(args, ["--no-enable-prefix-caching"]) - tokens = out.split(" ") - assert tokens[0] == "--online_quant_config" - assert json.loads(tokens[1]) == { - "global_quant_config": "ptpc_fp8", - "exclude_layer": ["*.mlp.gate", "*expert*"], - } - assert tokens[2:] == ["--tp", "8"] - - -def test_remove_server_args_unbalanced_brace_drops_only_its_own_flag() -> None: - """A stray ``}`` is not a JSON blob and must not take the tail with it.""" - out = gr.remove_server_args("--foo a} --tp 8 --max-num-seqs 64", ["--foo"]) - assert out == "--tp 8 --max-num-seqs 64" +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Supplemental coverage for _grid_runner pure helpers: compatibility filter model-class drop, runtime override env branches, report parsing, and per-variant yaml env injection.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +import yaml + +from hyperloom.orchestrator.actions.executors import _grid_runner as gr +from hyperloom.orchestrator.actions.executors import _benchmark_interpreter as bi + +# Patch compatibility-filter helpers in the ``_grid_variant_filter`` sibling, where apply_compatibility_filter +# resolves them (not via the re-export). +from hyperloom.orchestrator.actions.executors import _grid_variant_filter as vf + + +def _variant(name: str, args: str = "", envs: dict | None = None) -> gr.GridVariant: + return gr.GridVariant(name=name, extra_server_args=args, extra_envs=envs or {}) + + +def test_validate_magpie_python_override_requires_python(tmp_path): + py = tmp_path / "python3" + py.write_text("", encoding="utf-8") + bash = tmp_path / "bash" + bash.write_text("", encoding="utf-8") + assert gr._validate_magpie_python_override(str(py)) == str(py) + with pytest.raises(ValueError, match="Python interpreter"): + gr._validate_magpie_python_override(str(bash)) + assert gr._validate_magpie_python_override(sys.executable).endswith(Path(sys.executable).name) + + +# -- apply_compatibility_filter ------------------------------------------- +def test_compatibility_filter_drops_on_model_class(monkeypatch) -> None: + monkeypatch.setattr(vf, "_detect_model_class", lambda mp: (False, False)) + grid = [_variant("mla", "--enable-flashinfer-mla"), _variant("plain", "")] + kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="meta-llama-3-8b") + assert [v.name for v in kept] == ["plain"] + assert len(dropped) == 1 + assert dropped[0]["source"] == "compatibility_filter" + assert "MLA" in dropped[0]["reason"] + + +def test_compatibility_filter_keeps_a_flag_the_model_class_supports(monkeypatch) -> None: + # Whether the installed server accepts the flag is left to its own argparse at launch. + monkeypatch.setattr(vf, "_detect_model_class", lambda mp: (True, True)) + grid = [_variant("moe", "--enable-ep-moe")] + kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="deepseek-moe") + assert [v.name for v in kept] == ["moe"] and dropped == [] + + +def test_compatibility_filter_no_model_path_assumes_compatible() -> None: + grid = [_variant("moe", "--enable-ep-moe")] + kept, dropped = gr.apply_compatibility_filter(grid, framework="sglang", model_path="") + assert [v.name for v in kept] == ["moe"] and dropped == [] + + +# -- ungrounded_env_file_reason (env-value file grounding) ---------------- +def test_ungrounded_env_file_reason_drops_missing_config_path(tmp_path) -> None: + # A proposal may point a config env at an artifact no stage authored; the server would read it + # unconditionally and die at config load, so the variant must be dropped with a reason. + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(tmp_path / "nope.csv")}) + reason = gr.ungrounded_env_file_reason(v) + assert reason and "AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE" in reason + assert str(tmp_path / "nope.csv") in reason + + +def test_ungrounded_env_file_reason_passes_authored_file(tmp_path) -> None: + csv = tmp_path / "tuned.csv" + csv.write_text("gfx,M,N,K,splitK\ngfx950,64,5120,5120,2\n", encoding="utf-8") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(csv)}) + assert gr.ungrounded_env_file_reason(v) is None + + +def test_ungrounded_env_file_reason_accepts_colon_joined_existing_files(tmp_path) -> None: + # A set AITER_CONFIG_* is aiter's SET form: a `:`-joined list, each element loaded (csvs_aiter_will_load). + # Both files on disk is a valid variant and must not be dropped. + a = tmp_path / "t.csv" + a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") + b = tmp_path / "t2.csv" + b.write_text("gfx,M,N,K\ngfx950,4,5,6\n", encoding="utf-8") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{b}"}) + assert gr.ungrounded_env_file_reason(v) is None + + +def test_ungrounded_env_file_reason_drops_colon_joined_missing_element(tmp_path) -> None: + a = tmp_path / "t.csv" + a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") + missing = str(tmp_path / "never_authored.csv") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{missing}"}) + reason = gr.ungrounded_env_file_reason(v) + assert reason and missing in reason and str(a) not in reason.split("element(s)")[1].split(" do not exist")[0] + + +def test_ungrounded_env_file_reason_treats_an_unreadable_path_as_missing(monkeypatch) -> None: + # An unreadable parent or an overlong name makes Path.is_file() raise; the variant is dropped, the grid runs on. + def raise_eacces(self): + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(gr.Path, "is_file", raise_eacces) + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": "/root/private/tuned.csv"}) + reason = gr.ungrounded_env_file_reason(v) + assert reason and "/root/private/tuned.csv" in reason + + +def test_ungrounded_env_file_reason_noop_without_config_envs(tmp_path) -> None: + # Non-file envs (even under aiter-ish prefixes like the JSON-payload AITER_CONFIG) must stay untouched. + v = _variant( + "plain", + envs={ + "AITER_CONFIG": '{"path": "/tmp/never_scanned.json"}', + "SGLANG_USE_AITER": "1", + "AITER_CONFIG_GEMM_A8W8": "", + }, + ) + assert gr.ungrounded_env_file_reason(v) is None + + +# -- unsupported_capability_reason (env-flag build probe) ----------------- +def _clear_cap_cache() -> None: + gr._CAP_PROBE_CACHE.clear() + + +def test_capability_reason_noop_when_flag_absent(monkeypatch) -> None: + monkeypatch.setenv("FRAMEWORK", "vllm") + _clear_cap_cache() + # Probe must NOT even run when the flag isn't set on the variant. + monkeypatch.setattr( + gr, + "_probe_vllm_aiter_shared_expert_unsupported", + lambda: (_ for _ in ()).throw(AssertionError("probe should not run")), + ) + assert gr.unsupported_capability_reason(_variant("plain")) is None + + +def test_capability_reason_noop_for_non_vllm_framework(monkeypatch) -> None: + monkeypatch.setenv("FRAMEWORK", "sglang") + _clear_cap_cache() + v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1"}) + assert gr.unsupported_capability_reason(v) is None + + +def test_capability_reason_drops_when_module_missing(monkeypatch) -> None: + monkeypatch.setenv("FRAMEWORK", "vllm") + _clear_cap_cache() + monkeypatch.setattr( + gr, + "_probe_vllm_aiter_shared_expert_unsupported", + lambda: "missing module(s): vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe", + ) + v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1"}) + reason = gr.unsupported_capability_reason(v) + assert reason is not None and "rocm_aiter_fused_moe" in reason + + +def test_capability_reason_falsey_flag_not_probed(monkeypatch) -> None: + monkeypatch.setenv("FRAMEWORK", "vllm") + _clear_cap_cache() + monkeypatch.setattr( + gr, + "_probe_vllm_aiter_shared_expert_unsupported", + lambda: (_ for _ in ()).throw(AssertionError("probe should not run")), + ) + v = _variant("se", envs={"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0"}) + assert gr.unsupported_capability_reason(v) is None + + +def test_probe_caches_ok_and_unsupported(monkeypatch) -> None: + _clear_cap_cache() + calls = {"n": 0} + + class _Proc: + def __init__(self, out: str) -> None: + self.stdout = out + self.stderr = "" + + def fake_run(*_a, **_k): + calls["n"] += 1 + return _Proc(json.dumps({"status": "ok"})) + + monkeypatch.setattr(gr.subprocess, "run", fake_run) + assert gr._probe_vllm_aiter_shared_expert_unsupported() is None + # Second call must hit the cache (no second subprocess). + assert gr._probe_vllm_aiter_shared_expert_unsupported() is None + assert calls["n"] == 1 + + +def test_probe_unknown_not_cached(monkeypatch) -> None: + _clear_cap_cache() + + class _Proc: + stdout = json.dumps({"status": "unknown"}) + stderr = "" + + monkeypatch.setattr(gr.subprocess, "run", lambda *_a, **_k: _Proc()) + assert gr._probe_vllm_aiter_shared_expert_unsupported() is None + assert "vllm" not in gr._CAP_PROBE_CACHE + + +def test_probe_swallows_subprocess_error(monkeypatch) -> None: + _clear_cap_cache() + + def boom(*_a, **_k): + raise OSError("no python3") + + monkeypatch.setattr(gr.subprocess, "run", boom) + assert gr._probe_vllm_aiter_shared_expert_unsupported() is None + + +# -- _resolve_probe_python / probe interpreter selection ------------------ +def test_resolve_probe_python_prefers_magpie_interpreter(monkeypatch) -> None: + # The harness interpreter is used directly; no vllm-exe resolution is tried. + monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/srv/venv/bin/python") + monkeypatch.setattr( + gr.shutil, "which", lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("which() should not be called")) + ) + assert gr._resolve_probe_python() == "/srv/venv/bin/python" + + +def test_resolve_probe_python_falls_back_to_vllm_venv(monkeypatch) -> None: + # magpie_python is the canonical default -> pin the venv that backs ``vllm serve``. + monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/opt/venv/bin/python") + monkeypatch.setattr(gr.shutil, "which", lambda name: "/other/venv/bin/vllm" if name == "vllm" else None) + monkeypatch.setattr(gr.os.path, "exists", lambda p: p == "/other/venv/bin/python") + assert gr._resolve_probe_python() == "/other/venv/bin/python" + + +def test_resolve_probe_python_no_bare_python3_fallback(monkeypatch) -> None: + # With no resolvable vllm exe, fall back to the canonical magpie default — never a bare "python3". + monkeypatch.setattr(bi, "_resolve_magpie_python", lambda: "/opt/venv/bin/python") + monkeypatch.setattr(gr.shutil, "which", lambda *_a, **_k: None) + assert gr._resolve_probe_python() == "/opt/venv/bin/python" + + +def test_probe_invokes_resolved_interpreter(monkeypatch) -> None: + # The capability probe must run under the resolved interpreter, not "python3". + _clear_cap_cache() + monkeypatch.setattr(gr, "_resolve_probe_python", lambda: "/srv/venv/bin/python") + seen: dict = {} + + class _Proc: + stdout = json.dumps({"status": "ok"}) + stderr = "" + + def fake_run(cmd, *_a, **_k): + seen["cmd"] = list(cmd) + return _Proc() + + monkeypatch.setattr(gr.subprocess, "run", fake_run) + gr._probe_vllm_aiter_shared_expert_unsupported() + assert seen["cmd"][0] == "/srv/venv/bin/python" + assert seen["cmd"][0] != "python3" + + +# -- apply_runtime_benchmark_overrides ------------------------------------ +def test_runtime_overrides_model_precision_and_gpu_no_framework_agent(monkeypatch) -> None: + monkeypatch.setenv("PRECISION", "fp8") + for k in ("ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): + monkeypatch.delenv(k, raising=False) + bench: dict = {} # no framework -> benchmark_script popped + gr.apply_runtime_benchmark_overrides( + bench, + model_path="/models/x", + gpu_type="mi355x", + ) + assert bench["model"] == "/models/x" + assert bench["precision"] == "fp8" + assert bench["runner_type"] == "mi355x" + assert "benchmark_script" not in bench + + +def test_runtime_overrides_framework_pins_generic_script(monkeypatch) -> None: + for k in ("PRECISION", "ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): + monkeypatch.delenv(k, raising=False) + bench = {"framework": "sglang"} + gr.apply_runtime_benchmark_overrides(bench, gpu_type="mi300x") + assert bench["benchmark_script"] == "sglang_mi300x.sh" + + +def test_runtime_overrides_env_ints_and_rocr_autofill(monkeypatch) -> None: + for k in ("PRECISION",): + monkeypatch.delenv(k, raising=False) + monkeypatch.setenv("ISL", "128") + monkeypatch.setenv("TP", "2") + monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising=False) + bench: dict = {} + envs = gr.apply_runtime_benchmark_overrides(bench) + assert envs["ISL"] == 128 + assert envs["TP"] == 2 + # TP>1 with no explicit ROCR -> auto-filled device list + assert envs["ROCR_VISIBLE_DEVICES"] == "0,1" + + +def test_runtime_overrides_explicit_benchmark_script_wins(monkeypatch) -> None: + for k in ("PRECISION", "ISL", "OSL", "MAX_MODEL_LEN", "TP", "CONC", "ROCR_VISIBLE_DEVICES"): + monkeypatch.delenv(k, raising=False) + bench = {"framework": "vllm"} + gr.apply_runtime_benchmark_overrides( + bench, + gpu_type="mi300x", + benchmark_script="custom.sh", + ) + assert bench["benchmark_script"] == "custom.sh" + + +# -- _parse_report --------------------------------------------------------- +def test_parse_report_missing(tmp_path: Path) -> None: + assert gr._parse_report(tmp_path) is None + + +def test_parse_report_invalid_json(tmp_path: Path) -> None: + (tmp_path / "benchmark_report.json").write_text("{bad", encoding="utf-8") + assert gr._parse_report(tmp_path) is None + + +def test_parse_report_non_dict(tmp_path: Path) -> None: + (tmp_path / "benchmark_report.json").write_text("[1,2,3]", encoding="utf-8") + assert gr._parse_report(tmp_path) is None + + +def test_parse_report_valid(tmp_path: Path) -> None: + (tmp_path / "benchmark_report.json").write_text( + json.dumps({"output_throughput": 100.0}), + encoding="utf-8", + ) + assert gr._parse_report(tmp_path) == {"output_throughput": 100.0} + + +# -- _build_variant_yaml --------------------------------------------------- +def test_build_variant_yaml_injects_extra_envs(tmp_path: Path) -> None: + base = tmp_path / "base.yaml" + base.write_text( + yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), + encoding="utf-8", + ) + variant = _variant("v1", "--foo 1", envs={"USE_AITER": "1"}) + out = gr._build_variant_yaml( + base, + "", + variant, + output_subdir=tmp_path / "v1", + ) + cfg = yaml.safe_load(out.read_text(encoding="utf-8")) + envs = cfg["benchmark"]["envs"] + assert envs["USE_AITER"] == "1" + # variant + base server args merged into the framework's args env + arg_key = gr.server_args_env_name("sglang") + assert "--foo 1" in envs[arg_key] + + +def test_build_variant_yaml_dedupes_repeated_flags(tmp_path: Path) -> None: + """When base YAML + base_extra_args + variant all set the same flag, the materialized YAML must contain each flag + only once (last wins). + """ + base = tmp_path / "base.yaml" + base.write_text( + yaml.safe_dump( + { + "benchmark": { + "framework": "vllm", + "envs": {"EXTRA_VLLM_ARGS": "--attention-backend ROCM_ATTN"}, + }, + } + ), + encoding="utf-8", + ) + variant = _variant("v1", "--attention-backend ROCM_AITER_FA") + out = gr._build_variant_yaml( + base, + "", + variant, + output_subdir=tmp_path / "v1", + ) + cfg = yaml.safe_load(out.read_text(encoding="utf-8")) + args = cfg["benchmark"]["envs"]["EXTRA_VLLM_ARGS"] + assert args.count("--attention-backend") == 1, f"duplicate flag: {args}" + assert "ROCM_AITER_FA" in args, "last-wins should keep variant value" + + +def test_build_variant_yaml_prepends_legit_overlay(tmp_path: Path) -> None: + # SWSPLAT-42358: a legitimate overlay is a single existing directory; it is prepended to PYTHONPATH unchanged. + base = tmp_path / "base.yaml" + base.write_text( + yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), + encoding="utf-8", + ) + overlay = tmp_path / "overlay" + overlay.mkdir() + variant = _variant("v1") + variant.overlay_pythonpath = str(overlay) # type: ignore[attr-defined] + out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") + envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] + assert envs.get("PYTHONPATH", "").startswith(str(overlay)) + + +@pytest.mark.parametrize( + ("config_envs", "tail"), + [({}, ["/inh/a", "/inh/b"]), ({"PYTHONPATH": "/cfg"}, ["/cfg"])], + ids=["inherited", "config-authoritative"], +) +def test_build_variant_yaml_overlay_builds_on_the_launch_pythonpath(tmp_path, monkeypatch, config_envs, tail) -> None: + # The launch overlays benchmark.envs on os.environ, so a config without PYTHONPATH carries the inherited one. + # The overlay is relative so the path carries no drive colon on any host. + import os + + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/inh/a", "/inh/b"])) + (tmp_path / "overlay").mkdir() + base = tmp_path / "base.yaml" + base.write_text(yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": config_envs}}), encoding="utf-8") + variant = _variant("v1") + variant.overlay_pythonpath = "overlay" # type: ignore[attr-defined] + out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") + envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] + assert envs["PYTHONPATH"] == os.pathsep.join(["overlay", *tail]) + + +def test_build_variant_yaml_drops_unsafe_overlay(tmp_path: Path) -> None: + # SWSPLAT-42358: a ``:``-joined / traversal / non-existent overlay is dropped (would otherwise smuggle extra + # PYTHONPATH entries or escape). + base = tmp_path / "base.yaml" + base.write_text( + yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": {}}}), + encoding="utf-8", + ) + overlay = tmp_path / "overlay" + overlay.mkdir() + for bad in (f"{overlay}:/etc", f"{overlay}/../evil", str(tmp_path / "missing")): + variant = _variant("v1") + variant.overlay_pythonpath = bad # type: ignore[attr-defined] + out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1") + envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"] + assert "PYTHONPATH" not in envs or bad not in envs.get("PYTHONPATH", ""), bad + + +def test_shell_safe_dedupe_preserves_json_and_dedupes_other_flags() -> None: + """JSON stays an opaque argv token while unrelated flags dedupe last-wins.""" + args = '--json-model-override-args {"rope_scaling":null} --context-length 8192 --context-length 4096' + out = gr._shell_safe_dedupe(args) + assert out == '--json-model-override-args {"rope_scaling":null} --context-length 4096' + assert json.loads(out.split("--json-model-override-args ", 1)[1].split(" ", 1)[0]) == {"rope_scaling": None} + + +def test_shell_safe_dedupe_leaves_multi_value_arg_untouched() -> None: + """Multi-token flags must not be reassembled as stray positional tokens.""" + args = "--cuda-graph-bs 1 2 4 --cuda-graph-bs 8 16" + out = gr._shell_safe_dedupe(args) + assert out == args + + +def test_shell_safe_dedupe_normalizes_equals_form() -> None: + """--flag=value and --flag value must dedupe to one (last wins).""" + out = gr._shell_safe_dedupe("--attention-backend=ROCM_ATTN --attention-backend ROCM_AITER_FA") + assert out.count("--attention-backend") == 1, out + assert "ROCM_AITER_FA" in out, "last-wins should keep the later value" + assert "ROCM_ATTN" not in out + + +def test_shell_safe_dedupe_simple_last_wins() -> None: + out = gr._shell_safe_dedupe("--tp 1 --tp 8 --mem-fraction-static 0.9") + assert out == "--tp 8 --mem-fraction-static 0.9" + + +def test_compose_server_args_replace_still_applies_remove_args() -> None: + out = gr.compose_server_args( + inherited_args="--bad-base 1", + base_extra_args="--also-bad 2", + variant_extra_args="--bad-base 3 --keep 4", + remove_args=["--bad-base"], + args_mode="replace", + ) + assert out == "--also-bad 2 --keep 4" + + +def test_compose_server_args_launches_with_the_harness_flags_the_baseline_ran_with() -> None: + out = gr.compose_server_args( + inherited_args="--no-enable-prefix-caching --block-size 128", + base_extra_args="--no-enable-prefix-caching", + variant_extra_args="--kv-cache-dtype fp8 --no-enable-prefix-caching", + ) + assert "--no-enable-prefix-caching" in out + assert "--block-size 128" in out + assert "--kv-cache-dtype fp8" in out + + +def test_compose_server_args_keeps_the_harness_flag_in_replace_mode() -> None: + out = gr.compose_server_args( + inherited_args="--ignored", + base_extra_args="--no-enable-prefix-caching --max-num-seqs 256", + variant_extra_args="--kv-cache-dtype fp8", + args_mode="replace", + ) + assert out == "--no-enable-prefix-caching --max-num-seqs 256 --kv-cache-dtype fp8" + + +def test_launch_keeps_what_the_persisted_recipe_strips() -> None: + from hyperloom.inference_optimizer.grid_server_args import strip_benchmark_harness_flags + + launched = gr.compose_server_args(base_extra_args="--no-enable-prefix-caching --max-num-seqs 256") + assert "--no-enable-prefix-caching" in launched + assert strip_benchmark_harness_flags(launched) == "--max-num-seqs 256" + + +def test_remove_server_args_accepts_multi_flag_string() -> None: + out = gr.remove_server_args( + "--flag-a --flag-b --flag-c 3 --keep 4", + "--flag-a --flag-b --flag-c", + ) + assert out == "--keep 4" + + +def test_remove_server_args_keeps_a_sibling_json_value_parseable() -> None: + """Removing one flag must not corrupt a JSON-valued sibling.""" + args = ( + '--online_quant_config {"global_quant_config":"ptpc_fp8",' + '"exclude_layer":["*.mlp.gate","*expert*"]} ' + "--no-enable-prefix-caching --tp 8" + ) + out = gr.remove_server_args(args, ["--no-enable-prefix-caching"]) + tokens = out.split(" ") + assert tokens[0] == "--online_quant_config" + assert json.loads(tokens[1]) == { + "global_quant_config": "ptpc_fp8", + "exclude_layer": ["*.mlp.gate", "*expert*"], + } + assert tokens[2:] == ["--tp", "8"] + + +def test_remove_server_args_unbalanced_brace_drops_only_its_own_flag() -> None: + """A stray ``}`` is not a JSON blob and must not take the tail with it.""" + out = gr.remove_server_args("--foo a} --tp 8 --max-num-seqs 64", ["--foo"]) + assert out == "--tp 8 --max-num-seqs 64" diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index aef86b46d2..c64ebf7f13 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1,2289 +1,2289 @@ -# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Shared helper for the ``explore`` executor's grid runs.""" - -from __future__ import annotations - -import asyncio -import json -import logging -import os -import re -import shutil -import subprocess -import time -from pathlib import Path -from typing import Any, Callable - -import yaml - -from hyperloom.common.coerce import to_str_list -from hyperloom.common.env import env_flag, is_truthy -from hyperloom.common.perf_metric import constraint_fields -from hyperloom.common.env_safety import ( - BLOCKED_CHILD_ENV_NAMES, - BLOCKED_EXTERNAL_ENV_NAMES, - _ENV_KEY_RE, - is_python_package_root, - redact_secret_values, - scrub_benchmark_process_env, -) - -from ...phases import machine_state as _phase_state -from hyperloom.inference_optimizer.trace.task_progress import heartbeat_while_output_flows, report_progress -from ..stop_attribution import ( - ORCHESTRATOR_CANCELLED_CLASS, - SESSION_TIME_EXHAUSTED_CLASS, - STOPPED_BY_THE_RUN, - StoppedByTheRun, -) -from ._benchmark_runtime import apply_runtime_benchmark_overrides as apply_runtime_benchmark_overrides -from ._benchmark_interpreter import ( - _resolve_magpie_python as _resolve_magpie_python, - _resolve_probe_python as _resolve_probe_python, -) -from ._accuracy_gate import materialized_run_eval_disabled -from ._recipe_script import launcher_overwritten_envs -from ._subprocess_kill import ( - AGENTX_PREFLIGHT_ERROR_CLASS, - AGENTX_PREFLIGHT_RETURNCODE, - DETOKENIZER_STALL_RETURNCODE, - EVAL_PROBE_UNPATCHABLE_RETURNCODE, - ORCHESTRATOR_CANCELLED_RETURNCODE, - SERVER_DEAD_RETURNCODE, - SESSION_TIME_EXHAUSTED_RETURNCODE, - resolve_benchmark_timeouts, - run_with_session_kill, - server_log_death_excerpt, - session_deadline_to_remaining_sec, - sync_agentx_phase_wait, -) -from .benchmark_result import ( - extract_benchmark_measurement, - harvest_leaked_artifacts, - select_run_workspace, - served_complete_protocol, - snapshot_workspaces, -) -from ._gpu_metrics import write_gpu_metrics_from_report -from .benchmark_backend import build_benchmark_command -from ._inferencex_patcher import ( - ensure_benchmark_lib_eval_start_patched, - ensure_eval_probe_patched, - ensure_eval_unbound_outputs_patched, - eval_probe_targets_exist, -) -from ._launch_evidence import build_launch_evidence, persist_launch_evidence -from ...measurement.runtime_findings import persist_runtime_findings, scan_server_log -from ._server_argv import seal_server_argv - -# Re-exported from sibling modules to keep the module namespace intact. -from ._grid_base import ( - DEFAULT_KEEP_THRESHOLD_PCT as DEFAULT_KEEP_THRESHOLD_PCT, - GridVariant as GridVariant, - coerce_extra_envs as coerce_extra_envs, - VariantResult as VariantResult, -) -from hyperloom.inference_optimizer.grid_server_args import ( - server_args_env_name as server_args_env_name, - merge_server_args as merge_server_args, - compose_server_args as compose_server_args, - remove_server_args as remove_server_args, - compact_json_server_args as compact_json_server_args, - _SPACE_VALUE_FLAGS as _SPACE_VALUE_FLAGS, - _MULTI_VALUE_FLAGS as _MULTI_VALUE_FLAGS, - _VLLM_SINGLE_VALUE_FLAGS as _VLLM_SINGLE_VALUE_FLAGS, - dedup_vllm_server_args as dedup_vllm_server_args, - _shell_safe_dedupe as _shell_safe_dedupe, - DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC as DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC, - SGLANG_WATCHDOG_TIMEOUT_ENV as SGLANG_WATCHDOG_TIMEOUT_ENV, - _SGLANG_WATCHDOG_FLAG as _SGLANG_WATCHDOG_FLAG, - _SGLANG_WATCHDOG_RE as _SGLANG_WATCHDOG_RE, - resolve_sglang_watchdog_timeout as resolve_sglang_watchdog_timeout, - inject_sglang_watchdog_timeout as inject_sglang_watchdog_timeout, - DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS as DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS, - DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS as DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS, - SGLANG_CONTEXT_HEADROOM_ENV as SGLANG_CONTEXT_HEADROOM_ENV, - SGLANG_CONTEXT_FLOOR_ENV as SGLANG_CONTEXT_FLOOR_ENV, - _SGLANG_CONTEXT_LENGTH_FLAG as _SGLANG_CONTEXT_LENGTH_FLAG, - _SGLANG_CONTEXT_LENGTH_RE as _SGLANG_CONTEXT_LENGTH_RE, - _SGLANG_ATTN_BACKEND_FLAG as _SGLANG_ATTN_BACKEND_FLAG, - _SGLANG_ATTN_BACKEND_RE as _SGLANG_ATTN_BACKEND_RE, - _SGLANG_DUAL_CHUNK_BACKEND as _SGLANG_DUAL_CHUNK_BACKEND, - _resolve_nonneg_int_env as _resolve_nonneg_int_env, - resolve_sglang_context_cap as resolve_sglang_context_cap, - inject_sglang_context_length as inject_sglang_context_length, - _resolve_dual_chunk_backend as _resolve_dual_chunk_backend, - inject_sglang_attention_backend as inject_sglang_attention_backend, - _SGLANG_MOE_RUNNER_BACKEND_FLAG as _SGLANG_MOE_RUNNER_BACKEND_FLAG, - _SGLANG_MOE_RUNNER_BACKEND_RE as _SGLANG_MOE_RUNNER_BACKEND_RE, - moe_runner_requires_aiter as moe_runner_requires_aiter, -) -from ._grid_variant_filter import ( - resolve_skip_spec as resolve_skip_spec, - _parse_skip_spec as _parse_skip_spec, - _RE_CUDA_GRAPH_MAX_BS as _RE_CUDA_GRAPH_MAX_BS, - _MN_PARAMS_PRIORITY as _MN_PARAMS_PRIORITY, - _MN_BACKENDS_PRIORITY as _MN_BACKENDS_PRIORITY, - _mn_priority_index as _mn_priority_index, - reorder_grid_for_multi_node as reorder_grid_for_multi_node, - apply_multi_node_invalid_variants as apply_multi_node_invalid_variants, - apply_aiter_moe_pin_filter as apply_aiter_moe_pin_filter, - _COMPATIBILITY_FLAG_RULES as _COMPATIBILITY_FLAG_RULES, - _XDIT_ENV_BLACKLIST as _XDIT_ENV_BLACKLIST, - _XDIT_ENV_COMBO_BLACKLIST as _XDIT_ENV_COMBO_BLACKLIST, - xdit_blacklist_reason as xdit_blacklist_reason, - _detect_model_class as _detect_model_class, - apply_compatibility_filter as apply_compatibility_filter, - apply_user_skip_list as apply_user_skip_list, -) - - -log = logging.getLogger(__name__) - - -def _validate_magpie_python_override(value: str) -> str: - """Validate caller-supplied Magpie interpreter overrides before argv[0].""" - raw = str(value or "").strip() - if not raw: - return "" - if any(ch in raw for ch in ("\x00", "\n", "\r")): - raise ValueError("magpie_python contains control characters") - resolved = Path(raw) - if not resolved.is_absolute(): - found = shutil.which(raw) - if not found: - raise ValueError(f"magpie_python is not executable on PATH: {raw}") - resolved = Path(found) - if not resolved.is_file(): - raise ValueError(f"magpie_python is not a file: {raw}") - if not resolved.name.lower().startswith("python"): - raise ValueError(f"magpie_python must point to a Python interpreter, got: {raw}") - return str(resolved) - - -class _VariantAborted(Exception): - """A variant ended before it reached its benchmark, carrying the result that records why.""" - - def __init__(self, result: VariantResult) -> None: - super().__init__(result.error_class or "variant_aborted") - self.result = result - - -def _resolve_session_dir() -> Path: - """Resolve the active session_dir for executors that need an output root.""" - from hyperloom.inference_optimizer.session.paths import session_dir as _sd - - return _sd() - - -class SessionDirField: - """An executor's ``session_dir``, resolved on read rather than at construction. - - The executors here are module-level singletons built at import, which is - before the CLI pins the session. Resolving in ``__init__`` freezes them on - the workspace root that concurrent sessions share. - """ - - def __set_name__(self, owner: type, name: str) -> None: - self._slot = f"_{name}" - - def __get__(self, obj: Any, objtype: type | None = None) -> Any: - if obj is None: - return self - explicit = getattr(obj, self._slot, None) - return explicit if explicit is not None else _resolve_session_dir() - - def __set__(self, obj: Any, value: Path | str | None) -> None: - setattr(obj, self._slot, Path(value) if value else None) - - -# SKIP_VARIANTS: comma/whitespace patterns matched (exact or fnmatch) against ``GridVariant.name``. - - -# Env-flag capability probe: a serving env flag can be defined in the build yet still crash the server at engine init -# because the code path it activates imports a module the build did not package. -_UNSET = object() - -# Cached probe result keyed by framework. -_CAP_PROBE_CACHE: dict[str, str | None] = {} - -# Subprocess probe: locate the installed vLLM package via ``find_spec`` without importing it, read the aiter -# shared-expert router source, extract the ``fused_moe.*`` modules it imports, and verify each resolves to a real file -# in THIS build. -_AITER_SHARED_EXPERT_PROBE_SCRIPT = ( - "import importlib.util as u, os, re, json\n" - "def go():\n" - " try:\n" - " spec = u.find_spec('vllm')\n" - " except Exception:\n" - " return {'status': 'unknown'}\n" - " if not spec or not spec.origin:\n" - " return {'status': 'unknown'}\n" - " vdir = os.path.dirname(spec.origin)\n" - " router = os.path.join(vdir, 'model_executor', 'layers', 'fused_moe',\n" - " 'router', 'aiter_shared_routed_fused_moe_router.py')\n" - " if not os.path.exists(router):\n" - " return {'status': 'unknown'}\n" - " try:\n" - " text = open(router, encoding='utf-8').read()\n" - " except Exception:\n" - " return {'status': 'unknown'}\n" - " mods = re.findall(r'from\\s+(vllm\\.model_executor\\.layers\\.fused_moe\\.[A-Za-z0-9_.]+)\\s+import', text)\n" - " missing = []\n" - " for m in sorted(set(mods)):\n" - " rel = m[len('vllm.'):].replace('.', os.sep)\n" - " if not (os.path.exists(os.path.join(vdir, rel + '.py'))\n" - " or os.path.exists(os.path.join(vdir, rel, '__init__.py'))):\n" - " missing.append(m)\n" - " return {'status': 'unsupported', 'missing': missing} if missing else {'status': 'ok'}\n" - "print(json.dumps(go()))\n" -) - - -def _probe_vllm_aiter_shared_expert_unsupported() -> str | None: - """Return a drop reason if the installed vLLM build can't honour the aiter shared-expert fusion flag, else ``None``.""" - cached = _CAP_PROBE_CACHE.get("vllm", _UNSET) - if cached is not _UNSET: - return cached # type: ignore[return-value] - try: - proc = subprocess.run( - [_resolve_probe_python(), "-c", _AITER_SHARED_EXPERT_PROBE_SCRIPT], - capture_output=True, - text=True, - timeout=30, - ) - lines = [ln for ln in (proc.stdout or "").splitlines() if ln.strip()] - payload = json.loads(lines[-1]) if lines else {} - except Exception: # noqa: BLE001 - return None - status = payload.get("status") - if status == "ok": - _CAP_PROBE_CACHE["vllm"] = None - return None - if status == "unsupported": - missing = ", ".join(payload.get("missing") or []) or "(unknown module)" - reason = ( - "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS enabled but the installed " - "vLLM build's aiter shared-expert router imports missing module(s): " - f"{missing}. Flag unusable on this build (server crashes at engine " - "init); upgrade vLLM to a build that ships the module." - ) - _CAP_PROBE_CACHE["vllm"] = reason - return reason - return None # unknown => do not drop or cache - - -def unsupported_capability_reason(variant: "GridVariant") -> str | None: - """Return a drop reason if a variant sets an env flag the installed framework build cannot honour, else ``None``.""" - fw = (os.environ.get("FRAMEWORK", "") or "sglang").strip().lower() - if fw != "vllm": - return None - envs = {str(k): str(v) for k, v in (getattr(variant, "extra_envs", None) or {}).items()} - val = envs.get("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS") - if val is None or not is_truthy(val, default=True): - return None - return _probe_vllm_aiter_shared_expert_unsupported() - - -# Env vars whose value names a config file the serving stack reads unconditionally: aiter's tuned-GEMM loader skips -# its merge default and pd.read_csv's exactly the AITER_CONFIG_* path when one is set, so a missing file kills the -# server at config load instead of falling back to the default table. Values that are not file paths (e.g. geak's -# JSON-payload AITER_CONFIG) do not carry the trailing underscore and stay untouched. -_FILE_REFERENCING_ENV_PREFIXES: tuple[str, ...] = ("AITER_CONFIG_",) - - -def _is_file(path: str) -> bool: - # Path.is_file() raises for an unreadable parent or an overlong name (before Python 3.14); such a path is not a - # file the server can read either, and one variant's proposal must not crash the whole grid. - try: - return Path(path).is_file() - except OSError: - return False - - -def ungrounded_env_file_reason(variant: "GridVariant") -> str | None: - """Return a drop reason if an extra_envs value names a config file that does not exist, else ``None``. - - Grounds file-like env values the same way ``vet_patches`` grounds patch targets: a proposal may point a config - env at an artifact only a later (possibly rejected) authoring step would create, and the benchmark would boot a - server that dies at config load. The check is existence-only; anything already on disk (a promoted, durable - artifact) passes. A set value is a ``:``-joined list of paths (see ``csvs_aiter_will_load``), so every - non-empty element must exist, not only the first. - """ - for raw_key, raw_value in (getattr(variant, "extra_envs", None) or {}).items(): - key = str(raw_key or "").strip().upper() - if not key.startswith(_FILE_REFERENCING_ENV_PREFIXES): - continue - value = str(raw_value or "").strip() - if not value: - continue - missing = [p for p in (part.strip() for part in value.split(":")) if p and not _is_file(p)] - if missing: - return ( - f"{key} points at {value}, whose element(s) {':'.join(missing)} do not exist; the server would read it " - "unconditionally and die at config load. Point it at a file a stage authored " - "(e.g. a promoted tuned-GEMM CSV) or leave it unset to use the default config table." - ) - return None - - -# Sanitization for LLM-supplied overrides (benchmark_script / result_dir): reject path separators / shell -# metacharacters, raising ``ValueError`` instead of running an unsafe subprocess. -_SCRIPT_NAME_RE = re.compile(r"^[A-Za-z0-9_.\-]+\.sh$") -_RESULT_DIR_FORBID_RE = re.compile(r"[\s\"'`$;&|<>(){}\[\]\\*?!]") - - -def sanitize_script_name(value: Any) -> str | None: - """Return ``value`` if it's a safe Magpie benchmark script file name.""" - if value is None: - return None - text = str(value).strip() - if not text: - return None - if not _SCRIPT_NAME_RE.match(text): - raise ValueError( - f"benchmark_script={text!r} rejected: must be a bare *.sh " - "file name (no path separators, no shell metacharacters)" - ) - return text - - -def sanitize_result_dir(value: Any) -> str | None: - """Return ``value`` if it's a safe absolute (or workspace-relative) dir.""" - if value is None: - return None - text = str(value).strip() - if not text: - return None - if _RESULT_DIR_FORBID_RE.search(text): - raise ValueError( - f"result_dir={text!r} rejected: contains whitespace or shell " - "metacharacters; pass an absolute or workspace-relative path" - ) - return text - - -def _is_safe_path_entry(entry: str) -> bool: - """A single path entry is safe iff it has no ``:``, no ``..``, no control chars.""" - return ( - bool(entry) - and ":" not in entry - and ".." not in Path(entry).parts - and not any(c in entry for c in ("\n", "\r", "\x00")) - ) - - -# Reserved keys carried by dedicated override fields; never accepted via runtime_env. -_RUNTIME_ENV_RESERVED: frozenset[str] = frozenset({"PATH", "PYTHONPATH", "LD_LIBRARY_PATH"}) - - -def apply_runtime_override(envs: dict[str, str], override: dict[str, Any]) -> None: - """Inject an attempt runtime override into the materialized YAML envs dict.""" - if not override: - return - from ._workload_envs import prepend_launch_path - - path_prefix = str(override.get("path_prefix") or "").strip() - if path_prefix: - prepend_launch_path(envs, "PATH", path_prefix) - entrypoint_bin = str(override.get("entrypoint_bin_dir") or "").strip() - if entrypoint_bin: - if _is_safe_path_entry(entrypoint_bin): - prepend_launch_path(envs, "PATH", entrypoint_bin) - else: - log.warning("apply_runtime_override: dropping unsafe entrypoint_bin_dir %r", entrypoint_bin) - pp_prefix = str(override.get("pythonpath_prefix") or "").strip() - if pp_prefix: - if _is_safe_path_entry(pp_prefix): - prepend_launch_path(envs, "PYTHONPATH", pp_prefix) - else: - log.warning("apply_runtime_override: dropping unsafe pythonpath_prefix %r", pp_prefix) - # Multi-entry prefixes: prepend in order so the first entry wins, ahead of any single-dir pythonpath_prefix and - # the inherited value. - for entry in reversed(_coerce_prefix_list(override.get("pythonpath_prefixes"))): - if _is_safe_path_entry(entry): - prepend_launch_path(envs, "PYTHONPATH", entry) - else: - log.warning("apply_runtime_override: dropping unsafe pythonpath entry %r", entry) - # Native loader path: prepend attempt entries while preserving any inherited entries (e.g. /opt/rocm/lib). - for entry in reversed(_coerce_prefix_list(override.get("ld_library_path_prefix"))): - if _is_safe_path_entry(entry): - prepend_launch_path(envs, "LD_LIBRARY_PATH", entry) - else: - log.warning("apply_runtime_override: dropping unsafe ld_library_path entry %r", entry) - _runtime_env = override.get("runtime_env") - if isinstance(_runtime_env, dict): - for raw_k, raw_v in _runtime_env.items(): - key = str(raw_k) - if not _ENV_KEY_RE.match(key) or key in BLOCKED_CHILD_ENV_NAMES or key in _RUNTIME_ENV_RESERVED: - log.warning("apply_runtime_override: dropping unsafe runtime_env key %r", key) - continue - envs[key] = str(raw_v) - for key in ("framework_bin", "framework_python", "framework_venv_root"): - val = str(override.get(key) or "").strip() - if val: - envs[f"HYPERLOOM_{key.upper()}"] = val - # runtime_python_exe takes priority over framework_python as the launch interpreter. - rpe = str(override.get("runtime_python_exe") or "").strip() - if rpe: - envs["HYPERLOOM_FRAMEWORK_PYTHON"] = rpe - - -def _coerce_prefix_list(value: Any) -> list[str]: - """Normalize a runtime-prefix field (list/tuple/str) to a list of entries.""" - if value is None: - return [] - if isinstance(value, str): - return [value.strip()] if value.strip() else [] - if isinstance(value, (list, tuple)): - return [str(v).strip() for v in value if str(v).strip()] - return [] - - -def _build_variant_yaml( - base_yaml_path: Path, - base_extra_args: str, - variant: GridVariant, - *, - output_subdir: Path, - model_path: str | None = None, - gpu_type: str | None = None, - benchmark_script: str | None = None, - server_lifecycle: dict[str, Any] | None = None, - base_args_mode: str = "append", - base_extra_envs: dict[str, str] | None = None, - base_remove_args: list[str] | None = None, - base_unset_envs: list[str] | None = None, -) -> Path: - """Materialize a per-variant Magpie YAML on disk.""" - with base_yaml_path.open(encoding="utf-8") as f: - cfg = yaml.safe_load(f) - bench = cfg.setdefault("benchmark", {}) - replacing = str(base_args_mode).strip().lower() == "replace" - envs = apply_runtime_benchmark_overrides( - bench, - model_path=model_path, - gpu_type=gpu_type, - benchmark_script=benchmark_script, - conc=variant_conc(variant), - ) - extra_args_env = server_args_env_name(bench.get("framework")) - - variant_remove = to_str_list(getattr(variant, "remove_args", [])) - # A replacing base drops the inherited string wholesale, so only the - # variant's own removals still name flags that survive to be stripped. - effective_remove = ( - variant_remove if replacing else list(dict.fromkeys(to_str_list(base_remove_args) + variant_remove)) - ) - combined = compose_server_args( - inherited_args="" if replacing else str(envs.get(extra_args_env, "")), - base_extra_args=base_extra_args, - variant_extra_args=variant.extra_server_args, - remove_args=effective_remove, - args_mode=getattr(variant, "args_mode", "append"), - ) - # A grid variant never injects a MoE runner backend itself, but it does inherit one -- from the baseline recipe it - # was seeded with, or from an explicitly authored variant. - if combined and _SGLANG_MOE_RUNNER_BACKEND_RE.search(combined): - from ._workload_envs import _remove_moe_runner_backend_arg - - if extra_args_env == "EXTRA_SGLANG_ARGS" and moe_runner_requires_aiter(combined, model_path): - log.warning( - "grid: dropping inherited --moe-runner-backend for variant %s: " - "this checkpoint's MoE quant scheme is only implemented on the " - "aiter runner and would crash on the first forward pass.", - variant.name, - ) - combined = _remove_moe_runner_backend_arg(combined) - if combined: - envs[extra_args_env] = _shell_safe_dedupe(combined) - elif extra_args_env in envs or variant.args_mode == "replace" or base_args_mode == "replace": - envs[extra_args_env] = "" - # Composed base-then-variant, so a variant unsetting a key the base sets - # removes it: the last layer to name a key is the one that decides it. - for k in to_str_list(base_unset_envs): - if k.strip().upper() in BLOCKED_EXTERNAL_ENV_NAMES: - continue - envs.pop(k, None) - for k, v in (base_extra_envs or {}).items(): - envs[str(k)] = str(v) - for k in getattr(variant, "unset_envs", []) or []: - # Unsetting a pin retargets the benchmark rather than toggling a knob. - if str(k).strip().upper() in BLOCKED_EXTERNAL_ENV_NAMES: - log.warning("grid: refusing to unset pinned env %s for variant %s", k, variant.name) - continue - envs.pop(str(k), None) - for k, v in variant.extra_envs.items(): - envs[str(k)] = str(v) - from ._workload_envs import pin_mlperf_round_concurrency, prepend_launch_path - - pin_mlperf_round_concurrency(envs) - # The launcher re-exports these unconditionally, so a value carried here is - # one the run never used. - for k in launcher_overwritten_envs(bench) & envs.keys(): - log.warning("grid: dropping %s for variant %s; the recipe overwrites it", k, variant.name) - envs.pop(k, None) - # The three AgentX bounds took this rung's CONC through ``variant_conc`` above, not through this merge: raising - # the client's grace alone would make the round wait inside a cap that did not move with it. - _overlay = str(getattr(variant, "overlay_pythonpath", "") or "").strip() - if _overlay: - # Structural containment on the overlay dir before it is prepended to PYTHONPATH: a legitimate authored-kernel - # overlay is a single existing directory (never a ``:``-joined list, never a ``..`` traversal or control - # char). - _overlay_ok = ( - ":" not in _overlay - and ".." not in Path(_overlay).parts - and not any(c in _overlay for c in ("\n", "\r", "\x00")) - and Path(_overlay).is_dir() - ) - if _overlay_ok: - prepend_launch_path(envs, "PYTHONPATH", _overlay) - else: - log.warning( - "grid: dropping unsafe overlay_pythonpath %r (not a single " - "existing directory / contains separator or traversal)", - _overlay, - ) - - # Attempt runtime override: inject path_prefix/pythonpath_prefix/framework_bin etc. into benchmark.envs so the - # server subprocess resolves the attempt runtime. - _rt_override = getattr(variant, "runtime_override", None) or {} - if _rt_override: - apply_runtime_override(envs, _rt_override) - - # PATH guard: the xdit wrapper needs both `/venv/bin` (the `xdit` console script) and `/opt/rocm/bin` (`hipcc`); - # force-prepend both so an LLM-supplied PATH can't drop one. - if str(bench.get("framework", "")).strip().lower() == "xdit": - _cur_path = str(envs.get("PATH", "") or "") - _parts = [p for p in _cur_path.split(":") if p] - for _essential in ("/opt/rocm/bin", "/venv/bin"): - if _essential not in _parts: - _parts.insert(0, _essential) - envs["PATH"] = ":".join(_parts) - - if server_lifecycle is not None: - from ._server_lifecycle import inject_lifecycle - - inject_lifecycle( - bench, - cleanup=bool(server_lifecycle.get("cleanup", True)), - pid_dir=server_lifecycle["pid_dir"], - port=int(server_lifecycle["port"]), - ) - - # The final write to the argument env; nothing below may touch it. - seal_server_argv(envs, bench.get("framework")) - output_subdir.mkdir(parents=True, exist_ok=True) - out_path = output_subdir / "config.yaml" - with out_path.open("w", encoding="utf-8") as f: - yaml.safe_dump(cfg, f, sort_keys=False) - return out_path - - -def _parse_report(workspace: Path) -> dict[str, Any] | None: - """Load ``benchmark_report.json`` from a benchmark workspace.""" - report = workspace / "benchmark_report.json" - if not report.exists(): - return None - try: - with report.open(encoding="utf-8") as f: - data = json.load(f) - except (OSError, json.JSONDecodeError): - return None - return data if isinstance(data, dict) else None - - -# How long to keep re-reading ``benchmark_report.json`` when the report does not yet parse into a valid measurement. -REPORT_SETTLE_SECONDS = 30.0 -REPORT_SETTLE_POLL_SECONDS = 1.0 - - -async def _settled_measurement( - workspace: Path, - *, - subprocess_started_unix: float | None, - settle_seconds: float = REPORT_SETTLE_SECONDS, - poll_seconds: float = REPORT_SETTLE_POLL_SECONDS, -) -> tuple[dict[str, Any] | None, dict[str, Any]]: - """Read the benchmark report, re-reading briefly while it is still settling.""" - deadline = time.monotonic() + max(0.0, float(settle_seconds)) - attempts = 0 - while True: - report = _parse_report(workspace) - measurement = extract_benchmark_measurement( - report, - workspace=workspace, - subprocess_started_unix=subprocess_started_unix, - ) - attempts += 1 - if measurement.get("valid_measurement") or time.monotonic() >= deadline: - if attempts > 1 and measurement.get("valid_measurement"): - log.info( - "grid_runner: benchmark_report became valid after %d read(s); " - "the report was still being written when the subprocess was reaped", - attempts, - ) - # Harvest already tried this, but it runs before the report is guaranteed to exist: a report Magpie - # finishes writing after the subprocess is reaped would otherwise leave the round with no GPU artifact at - # all. Here the report is in hand, so write it from that rather than reading the file a second time. - if report is not None: - write_gpu_metrics_from_report(workspace, report, source="benchmark_report.json") - return report, measurement - await asyncio.sleep(max(0.01, float(poll_seconds))) - - -def _run_grid_warmup_enabled() -> bool: - """Whether ``run_grid`` should discard a cold warmup round when possible.""" - return env_flag("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", default=not os.environ.get("PYTEST_CURRENT_TEST")) - - -def _prepend_magpie_pythonpath(magpie_dir: str, current_pythonpath: str) -> str: - """Prepend Magpie's import root to PYTHONPATH, skipping package-root dirs.""" - if not magpie_dir or is_python_package_root(magpie_dir): - return current_pythonpath - return f"{magpie_dir}:{current_pythonpath}" if current_pythonpath else magpie_dir - - -def sync_benchmark_timeout(config_path: Path, timeout_sec: float) -> None: - """Give Magpie and bypass the same cap as the enclosing benchmark process.""" - cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) - bench = cfg["benchmark"] - bench["timeout_seconds"] = timeout_sec - if bench.get("server_lifecycle"): - bench["server_lifecycle"]["server_ready_timeout_s"] = timeout_sec - envs = bench.setdefault("envs", {}) - envs["PYTHONUNBUFFERED"] = "1" - sync_agentx_phase_wait(envs, timeout_sec) - config_path.write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8") - - -def _benchmark_server_log(config_path: Path, output_dir: Path) -> str | None: - """Scriptable benchmarks have no server whose readiness can arm silence.""" - cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) - framework = str(cfg["benchmark"].get("framework", "")).lower() - return None if framework in {"custom", "xdit"} else str(output_dir / "server.log") - - -def _run_magpie( - *, - magpie_python: str, - config_path: Path, - output_dir: Path, - timeout_sec: int, - cwd: str, - result_dir: str | None = None, - silence_timeout_sec: float | None = None, - server_already_ready: bool = False, - serving_lease: Any = None, - on_output: Callable[[], None] | None = None, - session_deadline_sec: float | None = None, - unset_envs: list[str] | None = None, -) -> tuple[int, str, str]: - """Blocking subprocess wrapper. Returns (rc, stdout, stderr).""" - sync_benchmark_timeout(config_path, timeout_sec) - server_log_path = _benchmark_server_log(config_path, output_dir) - env = scrub_benchmark_process_env(os.environ.copy()) - env["PYTHONUNBUFFERED"] = "1" - from ._workload_envs import resolve_reference_launch - - _args, _envs, reference_controls = resolve_reference_launch() - for name in [*(reference_controls.get("unset_envs") or []), *(unset_envs or [])]: - if name.strip().upper() not in BLOCKED_EXTERNAL_ENV_NAMES: - env.pop(name, None) - env["PATH"] = f"/opt/venv/bin:{env.get('PATH', '')}" - magpie_dir = os.environ.get("MAGPIE_PATH") or "" - if magpie_dir: - env["PYTHONPATH"] = _prepend_magpie_pythonpath(magpie_dir, env.get("PYTHONPATH", "")) - - # Multi-node: tell Magpie to skip its local-server launch and point benchmark_serving at the head pod's ClusterIP. - from ._multi_node_env import magpie_remote_env - - env.update(magpie_remote_env()) - - # Pin Magpie's InferenceX resolution to ``$INFERENCEX_PATH`` (its highest-precedence rung) so it loads the patched - # checkout, not a stale copy. - inferencex_path = os.environ.get("INFERENCEX_PATH", "").strip() - if inferencex_path: - env["MAGPIE_INFERENCEX_PATH"] = inferencex_path - # Baseline patches its own checkout, but explore / sweep never pass through that hook: re-assert here so a - # resumed session or a re-cloned checkout still emits the eval-start marker. - ensure_benchmark_lib_eval_start_patched(Path(inferencex_path)) - - # The generation bounds + pathology probe are asserted whether or not ``$INFERENCEX_PATH`` is set: unset falls - # back to the same env discovery the baseline arm uses ($MAGPIE_PATH/InferenceX). - probe_root = Path(inferencex_path) if inferencex_path else None - # Best-effort, unlike the probe below: a missing guard only costs the reason a failed round reports. - ensure_eval_unbound_outputs_patched(probe_root) - if not ensure_eval_probe_patched(probe_root) and not materialized_run_eval_disabled(config_path): - eval_bounds_msg = ( - "eval generation bounds + pathology probe are not installed " - "(utils/evals/patches/lm_eval_sitecustomize.py, inferencex=" - f"{inferencex_path or ''}, INFERENCEX_PATH=" - f"{os.environ.get('INFERENCEX_PATH', '') or ''}); this " - "variant runs eval, so it would be scored against a baseline that " - "truncated at a different point" - ) - if eval_probe_targets_exist(probe_root): - log.error("grid_runner: %s; failing this benchmark", eval_bounds_msg) - return (EVAL_PROBE_UNPATCHABLE_RETURNCODE, "", eval_bounds_msg) - log.warning("grid_runner: %s", eval_bounds_msg) - # AgentX: deploy the aiperf client into InferenceX ``benchmarks/`` + preflight aiperf right before Magpie runs it, - # via the shared helper (also used by the baseline/profile shell-out). - from ._workload_envs import prepare_agentx_runtime - - _agx_err = prepare_agentx_runtime(env=env, inferencex_path=inferencex_path, config_path=config_path) - if _agx_err: - log.error("%s; failing this benchmark", _agx_err) - return (AGENTX_PREFLIGHT_RETURNCODE, "", _agx_err) - # RESULT_DIR default; leaks are picked up by the salvage path. - env["RESULT_DIR"] = result_dir or str(output_dir) - # InferenceX ``run_lm_eval`` cleans ``$EVAL_RESULT_DIR`` after processing lm-eval output. - env["EVAL_RESULT_DIR"] = str(Path(env["RESULT_DIR"]) / "eval_output") - # Pin SERVER_LOG / GPU_METRICS_CSV per-task so logs land alongside ``benchmark_report.json``. - env["SERVER_LOG"] = str(output_dir / "server.log") - env["GPU_METRICS_CSV"] = str(output_dir / "gpu_metrics.csv") - - # Ray-managed GPU execution (§12 T1): route the round through the serving lease's actor, which holds ``num_gpus`` - # across every round sharing this server. - if serving_lease is not None: - from ._ray_backend import strip_visible_devices_from_config - - ray_config_path = strip_visible_devices_from_config(config_path) - cmd = build_benchmark_command( - python_exe=magpie_python, - config_path=ray_config_path, - output_dir=output_dir, - ) - # ``on_output`` cannot follow the round here: the benchmark runs inside a Ray actor in another process - # (potentially on another node) and only its final ``(rc, stdout, stderr)`` comes back, so there is nothing - # local to call per line. - return serving_lease.run_session_kill( - cmd, - env=env, - cwd=cwd, - timeout=timeout_sec, - silence_timeout_sec=silence_timeout_sec, - server_log_path=server_log_path, - server_already_ready=server_already_ready, - # Converted here, at the last moment before the process boundary: the actor cannot read this process's - # monotonic clock. - session_remaining_sec=session_deadline_to_remaining_sec(session_deadline_sec), - ) - - cmd = build_benchmark_command( - python_exe=magpie_python, - config_path=config_path, - output_dir=output_dir, - ) - # The launch puts Magpie in its own POSIX session and tears down the whole - # descendant tree on every exit path. - proc = run_with_session_kill( - cmd, - env=env, - cwd=cwd, - timeout=timeout_sec, - silence_timeout_sec=silence_timeout_sec, - server_log_path=server_log_path, - server_already_ready=server_already_ready, - on_output=on_output, - session_deadline_sec=session_deadline_sec, - ) - return proc.returncode, proc.stdout or "", proc.stderr or "" - - -def _num_gpus_for_config(config_path: Path) -> float: - """Read the tensor-parallel size (``TP``) from a materialized benchmark YAML.""" - try: - with Path(config_path).open(encoding="utf-8") as fp: - cfg = yaml.safe_load(fp) or {} - envs = (cfg.get("benchmark") or {}).get("envs") or {} - return float(int(envs.get("TP", 1) or 1)) - except Exception: # noqa: BLE001 — best-effort; default to 1 GPU - return 1.0 - - -def _mn_restart_env( - reference_envs: dict[str, str], - variant: Any, - unset_envs: list[str], -) -> dict[str, str]: - """Merge the reference server envs under one variant's own overrides.""" - merged = {k: v for k, v in reference_envs.items() if k not in unset_envs} - merged.update({str(k): str(v) for k, v in (variant.extra_envs or {}).items()}) - return merged - - -def _resolve_mn_effective_server_args( - cfg_path: Path, - base_yaml_path: Path, - variant: Any, - *, - base_extra_args: str, - base_args_mode: str, -) -> str: - """Resolve the multi-node server args for a variant restart; prefer the materialized variant YAML, falling back to a recompose from the base YAML.""" - try: - with cfg_path.open(encoding="utf-8") as _f: - _variant_cfg = yaml.safe_load(_f) or {} - _variant_bench = _variant_cfg.get("benchmark") or {} - _variant_envs = _variant_bench.get("envs") or {} - _variant_framework_env = server_args_env_name(_variant_bench.get("framework")) - return str(_variant_envs.get(_variant_framework_env) or "") - except Exception: - log.debug( - "grid_runner: failed to read materialized variant args from %s", - cfg_path, - exc_info=True, - ) - try: - with base_yaml_path.open(encoding="utf-8") as _f: - _base_cfg = yaml.safe_load(_f) or {} - _base_bench = _base_cfg.get("benchmark") or {} - _base_envs = _base_bench.get("envs") or {} - _base_framework_env = server_args_env_name(_base_bench.get("framework")) - _fallback_inherited_args = str(_base_envs.get(_base_framework_env) or "") - except Exception: # noqa: BLE001 - best-effort parity fallback - _fallback_inherited_args = "" - return _shell_safe_dedupe( - compose_server_args( - inherited_args="" if str(base_args_mode).strip().lower() == "replace" else _fallback_inherited_args, - base_extra_args=base_extra_args, - variant_extra_args=variant.extra_server_args, - remove_args=getattr(variant, "remove_args", []), - args_mode=getattr(variant, "args_mode", "append"), - ) - ) - - -def _variant_progress_note( - grid: list[GridVariant], - results: list[VariantResult], - idx: int, -) -> dict[str, Any]: - """Build the progress note for the variant at ``idx`` from that variant's own row.""" - landed = results[idx] if idx < len(results) else None - return { - "unit": "variant", - "label": grid[idx].name, - "index": idx + 1, - "total": len(grid), - "status": getattr(landed, "status", None), - "output_throughput": getattr(landed, "output_throughput", None), - } - - -# The returncode side of :mod:`...stop_attribution`: the same two causes, keyed by the sentinel a subprocess comes -# back with. -_STOPPED_BY_THE_RUN: dict[int, StoppedByTheRun] = { - SESSION_TIME_EXHAUSTED_RETURNCODE: STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], - ORCHESTRATOR_CANCELLED_RETURNCODE: STOPPED_BY_THE_RUN[ORCHESTRATOR_CANCELLED_CLASS], -} - - -def stopped_by_the_run(returncode: int | None) -> StoppedByTheRun | None: - """Return how to record a round the run itself stopped, if it did.""" - if returncode is None: - return None - return _STOPPED_BY_THE_RUN.get(int(returncode)) - - -def variant_conc(variant: Any) -> int | None: - """The concurrency a variant will run at, or ``None`` when it names none.""" - raw = (getattr(variant, "extra_envs", None) or {}).get("CONC") - try: - conc = int(raw) - except (TypeError, ValueError): - return None - return conc if conc > 0 else None - - -def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: - """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call.""" - if shared_state is None: - return (None, None) - deadline_fn = getattr(shared_state, "grid_session_deadline_sec", None) - deadline = deadline_fn() if callable(deadline_fn) else None - variant_sec = _phase_state.one_more_measurement_sec(shared_state) - if variant_sec is None: - variant_sec = _phase_state.measured_seconds(shared_state, "baseline_runtime_sec") - return (deadline, variant_sec) - - -async def run_grid( - *, - base_yaml_path: Path, - base_extra_args: str, - grid: list[GridVariant], - output_root: Path, - magpie_python: str | None = None, - keep_going_on_failure: bool = True, - model_path: str | None = None, - gpu_type: str | None = None, - benchmark_script: str | None = None, - result_dir: str | None = None, - server_lifecycle: dict[str, Any] | None = None, - base_args_mode: str = "append", - base_extra_envs: dict[str, str] | None = None, - base_remove_args: list[str] | None = None, - base_unset_envs: list[str] | None = None, - warmup_before_measure: bool | None = None, - server_already_ready: bool = False, - ready_server_log: str | None = None, - serving_lease: Any = None, - session_deadline_sec: float | None = None, - variant_expected_sec: float | None = None, - deadline_stop: StoppedByTheRun = STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], - lifecycle_boot_only: bool = False, -) -> list[VariantResult]: - """Execute variants; ``deadline_stop`` names the owner of the supplied deadline. - - ``ready_server_log`` is the log of the caller's ready server, measured when no variant log exists. - """ - silence_timeout_sec, benchmark_timeout_sec = resolve_benchmark_timeouts() - if not magpie_python: - # Backend-aware: bypass uses a plain python3, not Magpie's venv. - from .benchmark_backend import resolve_benchmark_interpreter - - magpie_python = resolve_benchmark_interpreter() - else: - magpie_python = _validate_magpie_python_override(magpie_python) - if warmup_before_measure is None: - warmup_before_measure = _run_grid_warmup_enabled() - auto_warmup_requested = bool(warmup_before_measure and server_lifecycle is None) - results: list[VariantResult] = [] - # This function names the working directory, so it creates it: callers and the per-variant config writer both - # happen to create it first today, and neither is a contract. - output_root.mkdir(parents=True, exist_ok=True) - cwd = str(output_root) - - # Reap orphaned aiter JIT build locks before booting any server. - try: - from ._aiter_jit import sweep_stale_aiter_locks_if_dead - - _lock_sweep = sweep_stale_aiter_locks_if_dead() - if _lock_sweep.get("deleted"): - log.warning( - "grid_runner: reaped %d orphaned aiter JIT lock(s) under %s before server launch (compiler_alive=%s)", - _lock_sweep.get("deleted"), - _lock_sweep.get("dir"), - _lock_sweep.get("compiler_alive"), - ) - except Exception as exc: # noqa: BLE001 — best-effort hygiene, never blocks the grid - log.debug("grid_runner: aiter lock sweep swallowed: %r", exc) - - # Multi-node: the reference recipe's server envs do not reach the variant restart through - # restart_server_for_round, which never reads them. - from ._multi_node_env import is_multi_node as _mn_is_multi_node - - _mn_ref_envs: dict[str, str] = {} - if _mn_is_multi_node(): - from ._workload_envs import resolve_reference_base - - _, _mn_ref_envs = resolve_reference_base() - - # Reported on entry, not on completion: ``_report_finished_variant`` only runs once a result has been appended, so - # a first variant that hangs — or a branch that raises before reaching it — would emit nothing at all, which is - # exactly the silence the heartbeat exists to break. - async def _unit_started(idx: int, label: str) -> None: - """Report that a unit of variant ``idx`` is about to start.""" - await report_progress( - unit="variant_step", - label=f"{grid[idx].name}:{label}", - index=idx + 1, - total=len(grid), - status="started", - ) - - async def _reported_magpie(idx: int, label: str, **kwargs: Any) -> tuple[int, str, str]: - """Run one Magpie pass, announced on entry and kept alive by its output.""" - await _unit_started(idx, label) - async with heartbeat_while_output_flows( - unit="variant_step", - label=f"{grid[idx].name}:{label}", - index=idx + 1, - total=len(grid), - ) as activity: - return await asyncio.to_thread( - _run_magpie, on_output=activity.note, unset_envs=grid[idx].unset_envs, **kwargs - ) - - # Variant boundary: a progress heartbeat so a grid that runs for hours is distinguishable from one that hung on - # its first variant. - async def _report_finished_variant(idx: int) -> None: - """Report the variant that just landed.""" - await report_progress(**_variant_progress_note(grid, results, idx)) - - def _record_round_stop( - stopped: StoppedByTheRun, - *, - idx: int, - variant: GridVariant, - slot: Path, - round_label: str, - returncode: int | None, - started_unix: float, - server_log: Path, - server_survivors: list[int], - ) -> bool: - """Record a round the run stopped and say whether the grid is over. - - ``server_survivors`` are the servers the variant booted that outlived its teardown. - """ - if returncode == SESSION_TIME_EXHAUSTED_RETURNCODE: - stopped = deadline_stop - runtime_sec = round(max(0.0, time.time() - started_unix), 2) - log.warning( - "grid_runner: variant %d/%d name=%s %s round reaped after %.1fs: %s; recorded as skipped, not failed", - idx + 1, - len(grid), - variant.name, - round_label, - runtime_sec, - stopped.interrupted, - ) - teardown = "tree reaped" - if server_survivors: - teardown += f", but server pid(s) {', '.join(map(str, server_survivors))} survived SIGKILL" - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class=stopped.error_class, - error_summary=f"{stopped.interrupted}; {teardown}", - extra_args=variant.extra_server_args, - ) - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="skipped", - returncode=returncode, - runtime_sec=runtime_sec, - error=stopped.interrupted, - error_class=stopped.error_class, - server_log_path=_measurement_server_log_path(server_log, slot=slot), - note=variant.note, - nonfatal_warnings=[f"server_survived_teardown:{pid}" for pid in server_survivors], - ) - ) - if stopped.ends_the_batch: - results.extend(_not_run_skip_result(rest, stopped) for rest in grid[idx + 1 :]) - return True - return not keep_going_on_failure - - # How many full benchmark passes one variant costs. - _mn_warmup_rounds = 0 - if variant_expected_sec is not None: - from ._multi_node_env import ( - is_multi_node as _mn_is_multi_node, - mn_bench_warmup_enabled as _mn_bench_warmup_enabled, - ) - - _mn_warmup_rounds = 1 if (_mn_is_multi_node() and _mn_bench_warmup_enabled()) else 0 - variant_rounds = 1 + (1 if auto_warmup_requested else 0) + _mn_warmup_rounds - - def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant_rounds) -> bool: - """Skip variant ``idx`` and every one after it, or say the budget still fits.""" - if session_deadline_sec is None: - return False - remaining_sec = session_deadline_sec - time.monotonic() - required_sec = float(variant_expected_sec) * rounds_left if variant_expected_sec is not None else None - if remaining_sec > 0 and (required_sec is None or remaining_sec >= required_sec): - return False - log.warning( - "grid_runner: %.0fs left cannot admit this variant's %d remaining round(s) " - "(expected_sec=%s, spent on: %s); skipping %d remaining variant(s)", - max(0.0, remaining_sec), - rounds_left, - required_sec, - spent_on, - len(grid) - idx, - ) - results.extend(_not_run_skip_result(variant, deadline_stop) for variant in grid[idx:]) - return True - - for i, variant in enumerate(grid): - try: - # Session-budget stop: skip the remaining variants once the wall-clock deadline is reached or the remaining - # budget cannot fit another variant, so a timeout halts the grid instead of draining it (and the last variant - # cannot overrun the close window). - if _skip_rest_for_budget(i, spent_on="the variants before this one"): - break - await _unit_started(i, "variant") - slot = output_root / f"variant_{i:02d}_{_safe(variant.name)}" - server_log = slot / "server.log" - # Capability fast-fail: drop a variant whose env flag the build cannot honour before booting a doomed server, - # still recording the failure so the LLM learns not to re-pick it. - cap_reason = unsupported_capability_reason(variant) or ungrounded_env_file_reason(variant) - if cap_reason: - log.warning( - "grid_runner: variant %d/%d name=%s aborted: capability_unsupported: %s", - i + 1, - len(grid), - variant.name, - cap_reason, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="capability_unsupported", - error_summary=cap_reason, - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=cap_reason, - error_class="capability_unsupported", - note=variant.note, - ) - ) - lifecycle: dict[str, Any] = {"eligible": False} - auto_warmup = False - try: - cfg_path = _build_variant_yaml( - base_yaml_path, - base_extra_args, - variant, - output_subdir=slot, - model_path=model_path, - gpu_type=gpu_type, - benchmark_script=benchmark_script, - server_lifecycle=server_lifecycle, - base_args_mode=base_args_mode, - base_extra_envs=base_extra_envs, - base_remove_args=base_remove_args, - base_unset_envs=base_unset_envs, - ) - except Exception as exc: # noqa: BLE001 - build_error = "yaml_build_error" - log.warning( - "grid_runner: variant %d/%d name=%s aborted: %s: %r", - i + 1, - len(grid), - variant.name, - build_error, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class=build_error, - error_summary=repr(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"{build_error}: {exc!r}", - error_class=build_error, - note=variant.note, - ) - ) - - _mn_effective_args = _resolve_mn_effective_server_args( - cfg_path, - base_yaml_path, - variant, - base_extra_args=base_extra_args, - base_args_mode=base_args_mode, - ) - - if auto_warmup_requested: - try: - from ._server_lifecycle import resolve_lifecycle_params - - lifecycle = resolve_lifecycle_params(cfg_path) - auto_warmup = bool(lifecycle.get("eligible")) - if auto_warmup: - cfg_path = _build_variant_yaml( - base_yaml_path, - base_extra_args, - variant, - output_subdir=slot, - model_path=model_path, - gpu_type=gpu_type, - benchmark_script=benchmark_script, - server_lifecycle={ - "cleanup": True, - "pid_dir": str(slot), - "port": int(lifecycle.get("port") or 0), - }, - base_args_mode=base_args_mode, - base_extra_envs=base_extra_envs, - base_remove_args=base_remove_args, - base_unset_envs=base_unset_envs, - ) - else: - log.info( - "grid_runner: warmup-before-measure not eligible (%s); running single measured round.", - lifecycle.get("reason") or "unknown", - ) - except Exception as exc: # noqa: BLE001 - log.warning( - "grid_runner: warmup-before-measure eligibility/materialization failed (%r); running single measured round.", - exc, - ) - auto_warmup = False - - warmup_tput: float | None = None - if auto_warmup: - warmup_slot = slot / "warmup_round" - warmup_server_log = warmup_slot / "server.log" - warmup_lifecycle = { - "cleanup": False, - "pid_dir": str(slot), - "port": int(lifecycle.get("port") or 0), - } - try: - warmup_cfg_path = _build_variant_yaml( - base_yaml_path, - base_extra_args, - variant, - output_subdir=warmup_slot, - model_path=model_path, - gpu_type=gpu_type, - benchmark_script=benchmark_script, - server_lifecycle=warmup_lifecycle, - base_args_mode=base_args_mode, - base_extra_envs=base_extra_envs, - base_remove_args=base_remove_args, - base_unset_envs=base_unset_envs, - ) - except Exception as exc: # noqa: BLE001 - log.warning( - "grid_runner: variant %d/%d name=%s aborted: warmup_yaml_build_error: %r", - i + 1, - len(grid), - variant.name, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="warmup_yaml_build_error", - error_summary=repr(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"warmup_yaml_build_error: {exc!r}", - error_class="warmup_yaml_build_error", - note=variant.note, - ) - ) - - warmup_workspaces_before = snapshot_workspaces(warmup_slot) - warmup_started_unix = time.time() - # Held in a local because the abort line below has to name the cap the round was actually granted: the - # declared one is a hang backstop, and a round killed at the reserved cap logged as a two-hour timeout - # reads as a variant that hangs rather than a budget that ran out. - warmup_cap_sec = benchmark_timeout_sec - try: - warmup_rc, warmup_stdout, warmup_stderr = await _reported_magpie( - i, - "warmup", - magpie_python=magpie_python, - config_path=warmup_cfg_path, - output_dir=warmup_slot, - timeout_sec=warmup_cap_sec, - silence_timeout_sec=silence_timeout_sec, - cwd=cwd, - result_dir=result_dir, - serving_lease=serving_lease, - session_deadline_sec=session_deadline_sec, - ) - except subprocess.TimeoutExpired as exc: - _teardown_variant_server(slot, lifecycle) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: warmup timeout (timeout_sec=%d): %s", - i + 1, - len(grid), - variant.name, - warmup_cap_sec, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="warmup_magpie_timeout", - error_summary=str(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"warmup_timeout: {exc}", - error_class="warmup_magpie_timeout", - server_log_path=_existing_log_path(warmup_server_log), - note=variant.note, - runtime_sec=round(max(0.0, time.time() - warmup_started_unix), 2), - nonfatal_warnings=["run_grid_warmup_round_failed"], - ) - ) - - warmup_stopped = stopped_by_the_run(warmup_rc) - if warmup_stopped is not None: - warmup_survivors = _teardown_variant_server(slot, lifecycle) - grid_is_over = _record_round_stop( - warmup_stopped, - idx=i, - variant=variant, - slot=slot, - round_label="warmup", - returncode=warmup_rc, - started_unix=warmup_started_unix, - server_log=warmup_server_log, - server_survivors=warmup_survivors, - ) - await _report_finished_variant(i) - if grid_is_over: - break - continue - - warmup_run_ws = select_run_workspace(warmup_slot, known_before=warmup_workspaces_before) - warmup_workspace = warmup_run_ws if warmup_run_ws is not None else warmup_slot - warmup_harvested = harvest_leaked_artifacts( - warmup_workspace, - subprocess_started_unix=warmup_started_unix, - ) - if warmup_run_ws is not None: - _, warmup_measurement = await _settled_measurement( - warmup_workspace, - subprocess_started_unix=warmup_started_unix, - settle_seconds=REPORT_SETTLE_SECONDS if warmup_rc == 0 else 0.0, - ) - else: - warmup_measurement = extract_benchmark_measurement( - None, - workspace=warmup_workspace, - subprocess_started_unix=warmup_started_unix, - ) - if warmup_rc != 0 or not warmup_measurement.get("valid_measurement"): - _teardown_variant_server(slot, lifecycle) - warmup_error = ( - server_log_death_excerpt(str(warmup_server_log)) - or redact_secret_values((warmup_stderr or warmup_stdout)[-2000:]) - if warmup_rc != 0 - else "warmup benchmark_report missing valid throughput/completed requests" - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: warmup_round_failed (rc=%s): %s", - i + 1, - len(grid), - variant.name, - warmup_rc, - warmup_error[:200], - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="warmup_round_failed", - error_summary=warmup_error, - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - workspace=str(warmup_workspace) if warmup_run_ws is not None else None, - report_path=( - str(warmup_workspace / "benchmark_report.json") - if (warmup_workspace / "benchmark_report.json").exists() - else None - ), - raw_result_path=warmup_measurement.get("raw_result_path"), - reported_success=warmup_measurement.get("reported_success"), - returncode=warmup_rc, - error=warmup_error, - error_class="warmup_round_failed", - server_log_path=_existing_log_path(warmup_server_log), - note=variant.note, - runtime_sec=round(max(0.0, time.time() - warmup_started_unix), 2), - nonfatal_warnings=[ - "run_grid_warmup_round_failed", - *[f"harvested_leaked_artifact:{src}" for src, _ in warmup_harvested], - ], - ) - ) - warmup_tput = warmup_measurement.get("output_throughput") - log.info( - "grid_runner: variant %s warmup tput=%.1f tok/s discarded; measuring hot round next", - variant.name, - warmup_tput or 0.0, - ) - - from ._multi_node_env import log_mn_banner - - log_mn_banner( - "grid_runner", - log, - variant=f"{i + 1}/{len(grid)}:{variant.name}", - ) - log.info( - "grid_runner: variant %d/%d name=%s args=%s", - i + 1, - len(grid), - variant.name, - variant.extra_server_args, - ) - - # Multi-node only: restart sglang/vllm with this variant's flags so each row runs against a fresh server. - from ._multi_node_server_lifecycle import ( - ServerRestartFailed, - restart_server_for_round, - ) - - try: - # PD knobs auto-resolved from $PD_* env; PD config stays constant across variants within one run. - _variant_unset = [str(k) for k in getattr(variant, "unset_envs", []) or [] if str(k).strip()] - _restart_env = _mn_restart_env(_mn_ref_envs, variant, _variant_unset) - await restart_server_for_round( - extra_server_args=_mn_effective_args, - # Reference recipe envs under this variant's own overrides (e.g. MORI_* MoE-dispatch tuning) so - # server-side env knobs proposed by specialists take effect on the restarted sglang without dropping - # the envs the baseline was measured with. - extra_env=_restart_env, - unset_env=_variant_unset, - model_path=model_path, - ep=int(os.environ.get("EP") or 0) or None, - ) - except ServerRestartFailed as exc: - log.warning( - "grid_runner: variant %d/%d name=%s aborted: mn_server_restart_failed: %s", - i + 1, - len(grid), - variant.name, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="mn_server_restart_failed", - error_summary=str(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"mn_server_restart_failed: {exc}", - error_class="mn_server_restart_failed", - note=variant.note, - ) - ) - - # The restart above is a launch of its own and is under no cap, so the budget admitted at the top of the loop - # may no longer be there. - if _skip_rest_for_budget( - i, - spent_on="this variant's server restart", - rounds_left=1 + _mn_warmup_rounds, - ): - if auto_warmup: - _teardown_variant_server(slot, lifecycle) - break - - # Multi-node client warmup: one discarded benchmark pass against the just-restarted, persistent remote server - # to warm JIT / steady-state before the measured pass. - from ._multi_node_env import ( - is_multi_node as _mn_imn, - mn_bench_warmup_enabled as _mn_warm, - ) - - if _mn_imn() and _mn_warm(): - _mn_warm_slot = slot / "mn_warmup" - _mn_warm_started_unix = time.time() - # The measurement is discarded, but the returncode is not: a warmup the run stopped is the same stop as - # one in the measured round, and discarding it launches the measured round after the cancel. - _mn_warm_rc: int | None = None - _mn_warm_rc, _, _ = await _reported_magpie( - i, - "mn_warmup", - magpie_python=magpie_python, - config_path=cfg_path, - output_dir=_mn_warm_slot, - timeout_sec=benchmark_timeout_sec, - silence_timeout_sec=silence_timeout_sec, - server_already_ready=True, - cwd=cwd, - result_dir=None, - serving_lease=serving_lease, - session_deadline_sec=session_deadline_sec, - ) - log.info( - "grid_runner: MN warmup pass done (discarded) %d/%d name=%s rc=%s", - i + 1, - len(grid), - variant.name, - _mn_warm_rc, - ) - _mn_warm_stopped = stopped_by_the_run(_mn_warm_rc) - if _mn_warm_stopped is not None: - _mn_warm_survivors = _teardown_variant_server(slot, lifecycle if auto_warmup else None) - grid_is_over = _record_round_stop( - _mn_warm_stopped, - idx=i, - variant=variant, - slot=slot, - round_label="mn_warmup", - returncode=_mn_warm_rc, - started_unix=_mn_warm_started_unix, - server_log=_mn_warm_slot / "server.log", - server_survivors=_mn_warm_survivors, - ) - await _report_finished_variant(i) - if grid_is_over: - break - continue - - # Snapshot wall-clock before launch so the salvage path can mtime-gate leak destinations per-variant. - slot_workspaces_before = snapshot_workspaces(slot) - variant_started_unix = time.time() - measure_cap_sec = benchmark_timeout_sec - try: - rc, stdout, stderr = await _reported_magpie( - i, - "benchmark", - magpie_python=magpie_python, - config_path=cfg_path, - output_dir=slot, - timeout_sec=measure_cap_sec, - cwd=cwd, - result_dir=result_dir, - silence_timeout_sec=silence_timeout_sec, - server_already_ready=(server_already_ready or auto_warmup or _mn_imn()), - serving_lease=serving_lease, - session_deadline_sec=session_deadline_sec, - ) - except subprocess.TimeoutExpired as exc: - # Harvest pre-timeout leaks. - to_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot - to_harvested = harvest_leaked_artifacts( - to_destination, - subprocess_started_unix=variant_started_unix, - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: magpie timeout (timeout_sec=%d): %s", - i + 1, - len(grid), - variant.name, - measure_cap_sec, - exc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="magpie_timeout", - error_summary=str(exc), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - error=f"timeout: {exc}", - error_class="magpie_timeout", - server_log_path=_existing_log_path(server_log), - note=variant.note, - runtime_sec=round( - max(0.0, time.time() - variant_started_unix), - 2, - ), - nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in to_harvested], - ) - ) - finally: - round_survivors = _teardown_variant_server(slot, lifecycle if auto_warmup else None) - - # Eval bounds could not be installed for a variant that runs eval, so nothing launched. - if rc == EVAL_PROBE_UNPATCHABLE_RETURNCODE: - variant_runtime_sec = round(max(0.0, time.time() - variant_started_unix), 2) - log.error( - "grid_runner: variant %d/%d name=%s aborted: eval_probe_unpatchable: %s", - i + 1, - len(grid), - variant.name, - stderr, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="eval_probe_unpatchable", - error_summary=stderr, - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - returncode=rc, - runtime_sec=variant_runtime_sec, - error=stderr, - error_class="eval_probe_unpatchable", - note=variant.note, - ) - ) - - # Server-liveness watchdog fired: engine/worker bootstrap died but the parent hung. - if rc == SERVER_DEAD_RETURNCODE: - variant_runtime_sec = round( - max(0.0, time.time() - variant_started_unix), - 2, - ) - sd_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot - sd_harvested = harvest_leaked_artifacts( - sd_destination, - subprocess_started_unix=variant_started_unix, - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: " - "server_init_dead (engine/worker bootstrap failed; parent " - "hung) after %.1fs", - i + 1, - len(grid), - variant.name, - variant_runtime_sec, - ) - death_excerpt = server_log_death_excerpt(str(server_log)) or ( - "server engine/worker init failed; parent process hung and was reaped by the liveness watchdog" - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="server_init_dead", - error_summary=death_excerpt, - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - returncode=rc, - runtime_sec=variant_runtime_sec, - error=death_excerpt, - error_class="server_init_dead", - server_log_path=_existing_log_path(server_log), - note=variant.note, - nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in sd_harvested], - ) - ) - - # Detokenizer-stall watchdog fired: the server came up healthy but then went silent (hung engine / wedged - # detokenizer). - if rc == DETOKENIZER_STALL_RETURNCODE: - variant_runtime_sec = round( - max(0.0, time.time() - variant_started_unix), - 2, - ) - ds_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot - ds_harvested = harvest_leaked_artifacts( - ds_destination, - subprocess_started_unix=variant_started_unix, - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: detokenizer_stall " - "(server ready but log went silent) after %.1fs", - i + 1, - len(grid), - variant.name, - variant_runtime_sec, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="detokenizer_stall", - error_summary=( - "server reported ready but emitted no log output (hung " - "engine / detokenizer stall); reaped by the " - "detokenizer-stall watchdog" - ), - extra_args=variant.extra_server_args, - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - returncode=rc, - runtime_sec=variant_runtime_sec, - error="detokenizer_stall: server ready but log went silent", - error_class="detokenizer_stall", - server_log_path=_existing_log_path(server_log), - note=variant.note, - nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in ds_harvested], - ) - ) - - stopped = stopped_by_the_run(rc) - if stopped is not None: - grid_is_over = _record_round_stop( - stopped, - idx=i, - variant=variant, - slot=slot, - round_label="measured", - returncode=rc, - started_unix=variant_started_unix, - server_log=server_log, - server_survivors=round_survivors, - ) - await _report_finished_variant(i) - if grid_is_over: - break - continue - - workspace = select_run_workspace(slot, known_before=slot_workspaces_before) - # Always-on artifact harvest so each slot keeps its server.log / gpu_metrics / profile relay for Robustness - # RCA. - harvest_destination = workspace if workspace is not None else slot - harvested = harvest_leaked_artifacts( - harvest_destination, - subprocess_started_unix=variant_started_unix, - ) - if harvested: - log.info( - "_grid_runner: variant=%s harvested %d leaked artifact(s): %s", - variant.name, - len(harvested), - ", ".join(src.name for src, _ in harvested), - ) - if workspace is None: - harvest_tags = [f"harvested_leaked_artifact:{src}" for src, _ in harvested] - # An AgentX preflight abort never reaches Magpie, so of course no workspace exists -- but calling it - # ``no_benchmark_workspace`` erases the one fact that decides what to do next. - agentx_preflight_abort = rc == AGENTX_PREFLIGHT_RETURNCODE - if agentx_preflight_abort: - # Same class the baseline path uses for the same abort, so both routes classify it identically. - no_ws_error_class = AGENTX_PREFLIGHT_ERROR_CLASS - no_ws_error_summary = ( - redact_secret_values((stderr or "").strip()[-2000:]) or "AgentX preflight aborted the round" - ) - else: - no_ws_error_class = "no_benchmark_workspace" - no_ws_error_summary = server_log_death_excerpt(str(server_log)) or ( - redact_secret_values((stderr or stdout)[-2000:]) - if rc != 0 - else "no benchmark_* workspace produced" - ) - log.warning( - "grid_runner: variant %d/%d name=%s aborted: %s (rc=%s)", - i + 1, - len(grid), - variant.name, - no_ws_error_class, - rc, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class=no_ws_error_class, - error_summary=no_ws_error_summary, - extra_args=variant.extra_server_args, - ) - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - returncode=rc, - error=no_ws_error_summary, - error_class=no_ws_error_class, - server_log_path=_existing_log_path(server_log), - nonfatal_warnings=harvest_tags, - note=variant.note, - ) - ) - await _report_finished_variant(i) - if agentx_preflight_abort: - # Environment, not variant. - log.error( - "grid_runner: variant %d/%d name=%s aborted on the AgentX preflight; " - "abandoning the remaining %d point(s) -- the client is missing for the " - "whole grid, not for this variant: %s", - i + 1, - len(grid), - variant.name, - len(grid) - (i + 1), - no_ws_error_summary, - ) - # Recorded rather than dropped: a grid that just returns fewer points than it declared reads as a grid - # that was that size. - results.extend( - VariantResult( - name=rest.name, - extra_server_args=rest.extra_server_args, - extra_envs=dict(rest.extra_envs), - status="skipped", - error="AgentX preflight aborted the grid before this variant ran", - error_class=AGENTX_PREFLIGHT_ERROR_CLASS, - note=rest.note, - ) - for rest in grid[i + 1 :] - ) - break - if rc != 0 and not keep_going_on_failure: - break - continue - report_path = workspace / "benchmark_report.json" - # The report is written during shutdown and the reader runs the moment the subprocess is reaped, so the read is - # worth waiting on however the process exited: a non-zero exit can still be a round that served its whole - # protocol, and that verdict is taken from this one read. - report, measurement = await _settled_measurement( - workspace, - subprocess_started_unix=variant_started_unix, - settle_seconds=REPORT_SETTLE_SECONDS, - ) - nonzero_kept_error: str | None = None - warnings = list(measurement.pop("nonfatal_warnings", []) or []) - for leak_src, _ in harvested: - warnings.append(f"harvested_leaked_artifact:{leak_src}") - if warmup_tput is not None: - warnings.append("run_grid_warmup_discarded_first") - warnings.append(f"warmup_round_tput:{float(warmup_tput):.1f}") - - if not measurement.get("valid_measurement"): - if lifecycle_boot_only and rc == 0: - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="succeeded", - workspace=str(workspace), - report_path=str(report_path) if report_path.exists() else None, - returncode=rc, - nonfatal_warnings=warnings, - note="server_lifecycle_boot_only", - ) - ) - await _report_finished_variant(i) - continue - death_excerpt = server_log_death_excerpt(str(server_log)) - if rc != 0: - error = death_excerpt or redact_secret_values((stderr or stdout)[-2000:]) - # The bypass/scriptable path runs the customer body in a child whose stderr is redirected to - # benchmark_stderr.log, not the parent pipe — so `stderr`/`stdout` here are often empty and the real - # diagnostic (e.g. argparse "unrecognized arguments") is only on disk. - if not error.strip(): - error = redact_secret_values(_on_disk_stderr_tail(workspace, slot)) - # Last resort: report.errors when the pipe and on-disk logs are all empty. - if not error.strip(): - error = redact_secret_values(_report_errors_summary(report)) - invalid_class = "magpie_nonzero_invalid_measurement" - elif not report: - error = death_excerpt or "benchmark_report missing" - invalid_class = "benchmark_report_missing" - else: - error = death_excerpt or "benchmark_report missing valid throughput/completed requests" - invalid_class = "benchmark_report_invalid_metric" - log.warning( - "grid_runner: variant %d/%d name=%s aborted: %s (rc=%s): %s", - i + 1, - len(grid), - variant.name, - invalid_class, - rc, - error[:200], - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class=invalid_class, - error_summary=error, - extra_args=variant.extra_server_args, - ) - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - workspace=str(workspace), - report_path=str(report_path) if report_path.exists() else None, - raw_result_path=measurement.get("raw_result_path"), - reported_success=measurement.get("reported_success"), - returncode=rc, - nonfatal_warnings=warnings, - error=error, - error_class=invalid_class, - server_log_path=_existing_log_path(server_log), - note=variant.note, - ) - ) - await _report_finished_variant(i) - if rc != 0 and not keep_going_on_failure: - break - continue - - if rc != 0: - nonzero_error = redact_secret_values((stderr or stdout)[-2000:]) - if served_complete_protocol(measurement): - # Every request the client recorded as requested was served, so - # the exit code came from something the round had already - # finished with. The cause stays on the result: a reader - # comparing this point to its neighbours is owed the reason it - # is not a clean zero. - warnings.append(f"nonzero_rc_after_complete_protocol:{rc}") - nonzero_kept_error = nonzero_error - log.warning( - "grid_runner: variant %s exited %d after serving its whole protocol " - "(%s of %s requests); keeping the measurement: %s", - variant.name, - rc, - measurement.get("completed_requests"), - measurement.get("requested_requests"), - nonzero_error, - ) - else: - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class="magpie_nonzero_after_valid_measurement", - error_summary=nonzero_error, - extra_args=variant.extra_server_args, - ) - log.warning( - "grid_runner: variant %s aborted: magpie_nonzero_after_valid_measurement " - "(rc=%d, served %s of %s requests)", - variant.name, - rc, - measurement.get("completed_requests"), - measurement.get("requested_requests"), - ) - raise _VariantAborted( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="failed", - workspace=str(workspace), - report_path=str(report_path) if report_path.exists() else None, - raw_result_path=measurement.get("raw_result_path"), - reported_success=measurement.get("reported_success"), - returncode=rc, - nonfatal_warnings=warnings, - error=nonzero_error, - error_class="magpie_nonzero_after_valid_measurement", - server_log_path=None, - note=variant.note, - ) - ) - - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="succeeded", - output_throughput=measurement.get("output_throughput"), - request_throughput=measurement.get("request_throughput"), - total_token_throughput=measurement.get("total_token_throughput"), - completed_requests=measurement.get("completed_requests"), - duration_seconds=measurement.get("duration_seconds"), - ttft_mean_ms=measurement.get("ttft_mean_ms"), - **constraint_fields(measurement), - tpot_mean_ms=measurement.get("tpot_mean_ms"), - input_throughput=measurement.get("input_throughput"), - tpot_p90_ms=measurement.get("tpot_p90_ms"), - intvty_p90=measurement.get("e2e_norm_intvty_p90"), - intvty_p50=measurement.get("e2e_norm_intvty_p50"), - request_error_rate=measurement.get("request_error_rate"), - workspace=str(workspace), - report_path=str(report_path) if report_path.exists() else None, - raw_result_path=measurement.get("raw_result_path"), - reported_success=measurement.get("reported_success"), - returncode=rc, - nonfatal_warnings=warnings, - error=nonzero_kept_error, - server_log_path=None, - note=variant.note, - runtime_sec=round( - max(0.0, time.time() - variant_started_unix), - 2, - ), - ) - ) - log.info( - "grid_runner: variant %s tput=%.1f tok/s", - variant.name, - results[-1].output_throughput or 0.0, - ) - await _report_finished_variant(i) - except _VariantAborted as abort: - results.append(abort.result) - await _report_finished_variant(i) - if not keep_going_on_failure: - break - await asyncio.to_thread( - _attach_grid_launch_evidence, - results, - grid=grid, - output_root=output_root, - caller_reused_ready_server=server_already_ready, - ready_server_log=ready_server_log, - ) - return results - - -def _teardown_variant_server(slot: Path, lifecycle: dict[str, Any] | None) -> list[int]: - """Stop every server this variant booted for its own rounds; return the pids still alive after SIGKILL. - - ``lifecycle`` names the persistent server the rounds shared, ``None`` when they shared none. - """ - from ._server_lifecycle import stop_agentx_servers, teardown_lifecycle_server - - if lifecycle is not None: - teardown_lifecycle_server( - pid_dir=slot, - framework=str(lifecycle.get("framework") or ""), - port=int(lifecycle.get("port") or 0), - ) - return stop_agentx_servers(slot) - - -def _not_run_skip_result(variant: GridVariant, stopped: StoppedByTheRun) -> VariantResult: - """Build the ``skipped`` result for a variant the run never got to.""" - return VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="skipped", - error=stopped.never_started, - error_class=stopped.error_class, - note=variant.note, - ) - - -def _existing_log_path(path: Path) -> str | None: - """Return ``path`` as a string when it exists, else ``None``.""" - return str(path) if path.exists() else None - - -def _measurement_server_log_path( - server_log: Path, - workspace: Path | None = None, - *, - slot: Path | None = None, -) -> str | None: - """Return the server log attributable to this variant's measurement.""" - owning_slot = slot or server_log.parent - direct = _existing_log_path(server_log) - if not direct and workspace is not None: - direct = _existing_log_path(workspace / "server.log") - if direct: - return direct - warmup_dir = owning_slot / "warmup_round" - try: - candidates = [path for path in warmup_dir.glob("*/server.log") if path.is_file()] - except OSError: - candidates = [] - if not candidates: - return None - try: - return str(max(candidates, key=lambda path: path.stat().st_mtime)) - except OSError: - return None - - -def _attach_grid_launch_evidence( - results: list[VariantResult], - *, - grid: list[GridVariant], - output_root: Path, - caller_reused_ready_server: bool, - ready_server_log: str | None = None, -) -> None: - """Persist declared and observed launch evidence for each grid result.""" - from ._workload_envs import VLLM_SOURCE_ROOT_ENVS - - for idx, result in enumerate(results): - if idx >= len(grid): - break - # A skipped variant never launched a measurement. - if result.status == "skipped" or result.error_class in { - "capability_unsupported", - "yaml_build_error", - "warmup_yaml_build_error", - }: - continue - slot = output_root / f"variant_{idx:02d}_{_safe(grid[idx].name)}" - config_path = slot / "config.yaml" - workspace = Path(result.workspace) if result.workspace else None - primary_log = Path(result.server_log_path) if result.server_log_path else slot / "server.log" - actual_log = _measurement_server_log_path(primary_log, workspace, slot=slot) - if not actual_log and ready_server_log: - actual_log = _existing_log_path(Path(ready_server_log)) - result.server_log_path = actual_log - evidence = build_launch_evidence( - config_path=config_path, - actual_server_log=actual_log, - framework=os.environ.get("FRAMEWORK", ""), - slot=slot, - caller_reused_ready_server=caller_reused_ready_server, - ) - result.launch_evidence = evidence - result.launch_evidence_path = persist_launch_evidence(evidence, slot=slot) - declared_env = evidence["requested_server_env"].keys() - set(VLLM_SOURCE_ROOT_ENVS) - report = scan_server_log(actual_log, evidence["framework"], declared_env=declared_env) - persist_runtime_findings(report, slot=slot) - - -def _safe(name: str) -> str: - """Filesystem-safe slug for variant directory names.""" - return "".join(c if c.isalnum() or c in "-_." else "_" for c in name)[:60] - - -def _write_variant_abort_marker( - slot: Path, - *, - variant_name: str, - error_class: str, - error_summary: str, - extra_args: str = "", -) -> None: - """Write ``abort_reason.json`` into the variant slot directory.""" - try: - slot.mkdir(parents=True, exist_ok=True) - marker = { - "variant": variant_name, - "error_class": error_class, - "error": (error_summary or "")[:2000], - "extra_args": extra_args, - "aborted_at_utc": time.strftime( - "%Y-%m-%dT%H:%M:%SZ", - time.gmtime(), - ), - } - (slot / "abort_reason.json").write_text( - json.dumps(marker, indent=2, sort_keys=True), - encoding="utf-8", - ) - except OSError as exc: - log.warning( - "_grid_runner: failed to write abort_reason.json at %s: %s", - slot, - exc, - ) - - -def _report_errors_summary(report: dict[str, Any] | None, limit: int = 2000) -> str: - """Join ``benchmark_report.json`` ``errors`` into a single diagnostic.""" - if not isinstance(report, dict): - return "" - errors = report.get("errors") - if not isinstance(errors, list): - return "" - text = "; ".join(str(item).strip() for item in errors if str(item).strip()) - return text[-limit:] if text else "" - - -def _on_disk_stderr_tail(*dirs: Path, limit: int = 2000) -> str: - """Return the tail of the first non-empty on-disk benchmark log.""" - for d in dirs: - if not d: - continue - for name in ("benchmark_stderr.log", "benchmark_stdout.log"): - try: - p = d / name - if not p.is_file(): - continue - text = p.read_text(encoding="utf-8", errors="replace").strip() - if text: - return f"[{name}] {text[-limit:]}" - except OSError: - continue - return "" - - -__all__ = [ - "DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC", - "GridVariant", - "ORCHESTRATOR_CANCELLED_CLASS", - "SESSION_TIME_EXHAUSTED_CLASS", - "StoppedByTheRun", - "SGLANG_WATCHDOG_TIMEOUT_ENV", - "VariantResult", - "apply_multi_node_invalid_variants", - "apply_aiter_moe_pin_filter", - "apply_runtime_benchmark_overrides", - "reorder_grid_for_multi_node", - "inject_sglang_attention_backend", - "inject_sglang_context_length", - "inject_sglang_watchdog_timeout", - "merge_server_args", - "remove_server_args", - "resolve_sglang_watchdog_timeout", - "run_grid", - "sanitize_result_dir", - "sanitize_script_name", - "server_args_env_name", - "session_grid_bounds", - "stopped_by_the_run", - # Re-exported from the sibling modules to keep the namespace intact. - "_resolve_magpie_python", - "_resolve_probe_python", - "coerce_extra_envs", - "compact_json_server_args", - "_SPACE_VALUE_FLAGS", - "_MULTI_VALUE_FLAGS", - "_VLLM_SINGLE_VALUE_FLAGS", - "dedup_vllm_server_args", - "_SGLANG_WATCHDOG_FLAG", - "_SGLANG_WATCHDOG_RE", - "DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS", - "DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS", - "SGLANG_CONTEXT_HEADROOM_ENV", - "SGLANG_CONTEXT_FLOOR_ENV", - "_SGLANG_CONTEXT_LENGTH_FLAG", - "_SGLANG_CONTEXT_LENGTH_RE", - "_SGLANG_ATTN_BACKEND_FLAG", - "_SGLANG_ATTN_BACKEND_RE", - "_SGLANG_DUAL_CHUNK_BACKEND", - "_resolve_nonneg_int_env", - "resolve_sglang_context_cap", - "_resolve_dual_chunk_backend", - "_SGLANG_MOE_RUNNER_BACKEND_FLAG", - "_SGLANG_MOE_RUNNER_BACKEND_RE", - "moe_runner_requires_aiter", - "resolve_skip_spec", - "_parse_skip_spec", - "_RE_CUDA_GRAPH_MAX_BS", - "_MN_PARAMS_PRIORITY", - "_MN_BACKENDS_PRIORITY", - "_mn_priority_index", - "_COMPATIBILITY_FLAG_RULES", - "_XDIT_ENV_BLACKLIST", - "_XDIT_ENV_COMBO_BLACKLIST", - "xdit_blacklist_reason", - "_detect_model_class", - "apply_compatibility_filter", - "apply_user_skip_list", -] +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Shared helper for the ``explore`` executor's grid runs.""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import re +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any, Callable + +import yaml + +from hyperloom.common.coerce import to_str_list +from hyperloom.common.env import env_flag, is_truthy +from hyperloom.common.perf_metric import constraint_fields +from hyperloom.common.env_safety import ( + BLOCKED_CHILD_ENV_NAMES, + BLOCKED_EXTERNAL_ENV_NAMES, + _ENV_KEY_RE, + is_python_package_root, + redact_secret_values, + scrub_benchmark_process_env, +) + +from ...phases import machine_state as _phase_state +from hyperloom.inference_optimizer.trace.task_progress import heartbeat_while_output_flows, report_progress +from ..stop_attribution import ( + ORCHESTRATOR_CANCELLED_CLASS, + SESSION_TIME_EXHAUSTED_CLASS, + STOPPED_BY_THE_RUN, + StoppedByTheRun, +) +from ._benchmark_runtime import apply_runtime_benchmark_overrides as apply_runtime_benchmark_overrides +from ._benchmark_interpreter import ( + _resolve_magpie_python as _resolve_magpie_python, + _resolve_probe_python as _resolve_probe_python, +) +from ._accuracy_gate import materialized_run_eval_disabled +from ._recipe_script import launcher_overwritten_envs +from ._subprocess_kill import ( + AGENTX_PREFLIGHT_ERROR_CLASS, + AGENTX_PREFLIGHT_RETURNCODE, + DETOKENIZER_STALL_RETURNCODE, + EVAL_PROBE_UNPATCHABLE_RETURNCODE, + ORCHESTRATOR_CANCELLED_RETURNCODE, + SERVER_DEAD_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, + resolve_benchmark_timeouts, + run_with_session_kill, + server_log_death_excerpt, + session_deadline_to_remaining_sec, + sync_agentx_phase_wait, +) +from .benchmark_result import ( + extract_benchmark_measurement, + harvest_leaked_artifacts, + select_run_workspace, + served_complete_protocol, + snapshot_workspaces, +) +from ._gpu_metrics import write_gpu_metrics_from_report +from .benchmark_backend import build_benchmark_command +from ._inferencex_patcher import ( + ensure_benchmark_lib_eval_start_patched, + ensure_eval_probe_patched, + ensure_eval_unbound_outputs_patched, + eval_probe_targets_exist, +) +from ._launch_evidence import build_launch_evidence, persist_launch_evidence +from ...measurement.runtime_findings import persist_runtime_findings, scan_server_log +from ._server_argv import seal_server_argv + +# Re-exported from sibling modules to keep the module namespace intact. +from ._grid_base import ( + DEFAULT_KEEP_THRESHOLD_PCT as DEFAULT_KEEP_THRESHOLD_PCT, + GridVariant as GridVariant, + coerce_extra_envs as coerce_extra_envs, + VariantResult as VariantResult, +) +from hyperloom.inference_optimizer.grid_server_args import ( + server_args_env_name as server_args_env_name, + merge_server_args as merge_server_args, + compose_server_args as compose_server_args, + remove_server_args as remove_server_args, + compact_json_server_args as compact_json_server_args, + _SPACE_VALUE_FLAGS as _SPACE_VALUE_FLAGS, + _MULTI_VALUE_FLAGS as _MULTI_VALUE_FLAGS, + _VLLM_SINGLE_VALUE_FLAGS as _VLLM_SINGLE_VALUE_FLAGS, + dedup_vllm_server_args as dedup_vllm_server_args, + _shell_safe_dedupe as _shell_safe_dedupe, + DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC as DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC, + SGLANG_WATCHDOG_TIMEOUT_ENV as SGLANG_WATCHDOG_TIMEOUT_ENV, + _SGLANG_WATCHDOG_FLAG as _SGLANG_WATCHDOG_FLAG, + _SGLANG_WATCHDOG_RE as _SGLANG_WATCHDOG_RE, + resolve_sglang_watchdog_timeout as resolve_sglang_watchdog_timeout, + inject_sglang_watchdog_timeout as inject_sglang_watchdog_timeout, + DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS as DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS, + DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS as DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS, + SGLANG_CONTEXT_HEADROOM_ENV as SGLANG_CONTEXT_HEADROOM_ENV, + SGLANG_CONTEXT_FLOOR_ENV as SGLANG_CONTEXT_FLOOR_ENV, + _SGLANG_CONTEXT_LENGTH_FLAG as _SGLANG_CONTEXT_LENGTH_FLAG, + _SGLANG_CONTEXT_LENGTH_RE as _SGLANG_CONTEXT_LENGTH_RE, + _SGLANG_ATTN_BACKEND_FLAG as _SGLANG_ATTN_BACKEND_FLAG, + _SGLANG_ATTN_BACKEND_RE as _SGLANG_ATTN_BACKEND_RE, + _SGLANG_DUAL_CHUNK_BACKEND as _SGLANG_DUAL_CHUNK_BACKEND, + _resolve_nonneg_int_env as _resolve_nonneg_int_env, + resolve_sglang_context_cap as resolve_sglang_context_cap, + inject_sglang_context_length as inject_sglang_context_length, + _resolve_dual_chunk_backend as _resolve_dual_chunk_backend, + inject_sglang_attention_backend as inject_sglang_attention_backend, + _SGLANG_MOE_RUNNER_BACKEND_FLAG as _SGLANG_MOE_RUNNER_BACKEND_FLAG, + _SGLANG_MOE_RUNNER_BACKEND_RE as _SGLANG_MOE_RUNNER_BACKEND_RE, + moe_runner_requires_aiter as moe_runner_requires_aiter, +) +from ._grid_variant_filter import ( + resolve_skip_spec as resolve_skip_spec, + _parse_skip_spec as _parse_skip_spec, + _RE_CUDA_GRAPH_MAX_BS as _RE_CUDA_GRAPH_MAX_BS, + _MN_PARAMS_PRIORITY as _MN_PARAMS_PRIORITY, + _MN_BACKENDS_PRIORITY as _MN_BACKENDS_PRIORITY, + _mn_priority_index as _mn_priority_index, + reorder_grid_for_multi_node as reorder_grid_for_multi_node, + apply_multi_node_invalid_variants as apply_multi_node_invalid_variants, + apply_aiter_moe_pin_filter as apply_aiter_moe_pin_filter, + _COMPATIBILITY_FLAG_RULES as _COMPATIBILITY_FLAG_RULES, + _XDIT_ENV_BLACKLIST as _XDIT_ENV_BLACKLIST, + _XDIT_ENV_COMBO_BLACKLIST as _XDIT_ENV_COMBO_BLACKLIST, + xdit_blacklist_reason as xdit_blacklist_reason, + _detect_model_class as _detect_model_class, + apply_compatibility_filter as apply_compatibility_filter, + apply_user_skip_list as apply_user_skip_list, +) + + +log = logging.getLogger(__name__) + + +def _validate_magpie_python_override(value: str) -> str: + """Validate caller-supplied Magpie interpreter overrides before argv[0].""" + raw = str(value or "").strip() + if not raw: + return "" + if any(ch in raw for ch in ("\x00", "\n", "\r")): + raise ValueError("magpie_python contains control characters") + resolved = Path(raw) + if not resolved.is_absolute(): + found = shutil.which(raw) + if not found: + raise ValueError(f"magpie_python is not executable on PATH: {raw}") + resolved = Path(found) + if not resolved.is_file(): + raise ValueError(f"magpie_python is not a file: {raw}") + if not resolved.name.lower().startswith("python"): + raise ValueError(f"magpie_python must point to a Python interpreter, got: {raw}") + return str(resolved) + + +class _VariantAborted(Exception): + """A variant ended before it reached its benchmark, carrying the result that records why.""" + + def __init__(self, result: VariantResult) -> None: + super().__init__(result.error_class or "variant_aborted") + self.result = result + + +def _resolve_session_dir() -> Path: + """Resolve the active session_dir for executors that need an output root.""" + from hyperloom.inference_optimizer.session.paths import session_dir as _sd + + return _sd() + + +class SessionDirField: + """An executor's ``session_dir``, resolved on read rather than at construction. + + The executors here are module-level singletons built at import, which is + before the CLI pins the session. Resolving in ``__init__`` freezes them on + the workspace root that concurrent sessions share. + """ + + def __set_name__(self, owner: type, name: str) -> None: + self._slot = f"_{name}" + + def __get__(self, obj: Any, objtype: type | None = None) -> Any: + if obj is None: + return self + explicit = getattr(obj, self._slot, None) + return explicit if explicit is not None else _resolve_session_dir() + + def __set__(self, obj: Any, value: Path | str | None) -> None: + setattr(obj, self._slot, Path(value) if value else None) + + +# SKIP_VARIANTS: comma/whitespace patterns matched (exact or fnmatch) against ``GridVariant.name``. + + +# Env-flag capability probe: a serving env flag can be defined in the build yet still crash the server at engine init +# because the code path it activates imports a module the build did not package. +_UNSET = object() + +# Cached probe result keyed by framework. +_CAP_PROBE_CACHE: dict[str, str | None] = {} + +# Subprocess probe: locate the installed vLLM package via ``find_spec`` without importing it, read the aiter +# shared-expert router source, extract the ``fused_moe.*`` modules it imports, and verify each resolves to a real file +# in THIS build. +_AITER_SHARED_EXPERT_PROBE_SCRIPT = ( + "import importlib.util as u, os, re, json\n" + "def go():\n" + " try:\n" + " spec = u.find_spec('vllm')\n" + " except Exception:\n" + " return {'status': 'unknown'}\n" + " if not spec or not spec.origin:\n" + " return {'status': 'unknown'}\n" + " vdir = os.path.dirname(spec.origin)\n" + " router = os.path.join(vdir, 'model_executor', 'layers', 'fused_moe',\n" + " 'router', 'aiter_shared_routed_fused_moe_router.py')\n" + " if not os.path.exists(router):\n" + " return {'status': 'unknown'}\n" + " try:\n" + " text = open(router, encoding='utf-8').read()\n" + " except Exception:\n" + " return {'status': 'unknown'}\n" + " mods = re.findall(r'from\\s+(vllm\\.model_executor\\.layers\\.fused_moe\\.[A-Za-z0-9_.]+)\\s+import', text)\n" + " missing = []\n" + " for m in sorted(set(mods)):\n" + " rel = m[len('vllm.'):].replace('.', os.sep)\n" + " if not (os.path.exists(os.path.join(vdir, rel + '.py'))\n" + " or os.path.exists(os.path.join(vdir, rel, '__init__.py'))):\n" + " missing.append(m)\n" + " return {'status': 'unsupported', 'missing': missing} if missing else {'status': 'ok'}\n" + "print(json.dumps(go()))\n" +) + + +def _probe_vllm_aiter_shared_expert_unsupported() -> str | None: + """Return a drop reason if the installed vLLM build can't honour the aiter shared-expert fusion flag, else ``None``.""" + cached = _CAP_PROBE_CACHE.get("vllm", _UNSET) + if cached is not _UNSET: + return cached # type: ignore[return-value] + try: + proc = subprocess.run( + [_resolve_probe_python(), "-c", _AITER_SHARED_EXPERT_PROBE_SCRIPT], + capture_output=True, + text=True, + timeout=30, + ) + lines = [ln for ln in (proc.stdout or "").splitlines() if ln.strip()] + payload = json.loads(lines[-1]) if lines else {} + except Exception: # noqa: BLE001 + return None + status = payload.get("status") + if status == "ok": + _CAP_PROBE_CACHE["vllm"] = None + return None + if status == "unsupported": + missing = ", ".join(payload.get("missing") or []) or "(unknown module)" + reason = ( + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS enabled but the installed " + "vLLM build's aiter shared-expert router imports missing module(s): " + f"{missing}. Flag unusable on this build (server crashes at engine " + "init); upgrade vLLM to a build that ships the module." + ) + _CAP_PROBE_CACHE["vllm"] = reason + return reason + return None # unknown => do not drop or cache + + +def unsupported_capability_reason(variant: "GridVariant") -> str | None: + """Return a drop reason if a variant sets an env flag the installed framework build cannot honour, else ``None``.""" + fw = (os.environ.get("FRAMEWORK", "") or "sglang").strip().lower() + if fw != "vllm": + return None + envs = {str(k): str(v) for k, v in (getattr(variant, "extra_envs", None) or {}).items()} + val = envs.get("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS") + if val is None or not is_truthy(val, default=True): + return None + return _probe_vllm_aiter_shared_expert_unsupported() + + +# Env vars whose value names a config file the serving stack reads unconditionally: aiter's tuned-GEMM loader skips +# its merge default and pd.read_csv's exactly the AITER_CONFIG_* path when one is set, so a missing file kills the +# server at config load instead of falling back to the default table. Values that are not file paths (e.g. geak's +# JSON-payload AITER_CONFIG) do not carry the trailing underscore and stay untouched. +_FILE_REFERENCING_ENV_PREFIXES: tuple[str, ...] = ("AITER_CONFIG_",) + + +def _is_file(path: str) -> bool: + # Path.is_file() raises for an unreadable parent or an overlong name (before Python 3.14); such a path is not a + # file the server can read either, and one variant's proposal must not crash the whole grid. + try: + return Path(path).is_file() + except OSError: + return False + + +def ungrounded_env_file_reason(variant: "GridVariant") -> str | None: + """Return a drop reason if an extra_envs value names a config file that does not exist, else ``None``. + + Grounds file-like env values the same way ``vet_patches`` grounds patch targets: a proposal may point a config + env at an artifact only a later (possibly rejected) authoring step would create, and the benchmark would boot a + server that dies at config load. The check is existence-only; anything already on disk (a promoted, durable + artifact) passes. A set value is a ``:``-joined list of paths (see ``csvs_aiter_will_load``), so every + non-empty element must exist, not only the first. + """ + for raw_key, raw_value in (getattr(variant, "extra_envs", None) or {}).items(): + key = str(raw_key or "").strip().upper() + if not key.startswith(_FILE_REFERENCING_ENV_PREFIXES): + continue + value = str(raw_value or "").strip() + if not value: + continue + missing = [p for p in (part.strip() for part in value.split(":")) if p and not _is_file(p)] + if missing: + return ( + f"{key} points at {value}, whose element(s) {':'.join(missing)} do not exist; the server would read it " + "unconditionally and die at config load. Point it at a file a stage authored " + "(e.g. a promoted tuned-GEMM CSV) or leave it unset to use the default config table." + ) + return None + + +# Sanitization for LLM-supplied overrides (benchmark_script / result_dir): reject path separators / shell +# metacharacters, raising ``ValueError`` instead of running an unsafe subprocess. +_SCRIPT_NAME_RE = re.compile(r"^[A-Za-z0-9_.\-]+\.sh$") +_RESULT_DIR_FORBID_RE = re.compile(r"[\s\"'`$;&|<>(){}\[\]\\*?!]") + + +def sanitize_script_name(value: Any) -> str | None: + """Return ``value`` if it's a safe Magpie benchmark script file name.""" + if value is None: + return None + text = str(value).strip() + if not text: + return None + if not _SCRIPT_NAME_RE.match(text): + raise ValueError( + f"benchmark_script={text!r} rejected: must be a bare *.sh " + "file name (no path separators, no shell metacharacters)" + ) + return text + + +def sanitize_result_dir(value: Any) -> str | None: + """Return ``value`` if it's a safe absolute (or workspace-relative) dir.""" + if value is None: + return None + text = str(value).strip() + if not text: + return None + if _RESULT_DIR_FORBID_RE.search(text): + raise ValueError( + f"result_dir={text!r} rejected: contains whitespace or shell " + "metacharacters; pass an absolute or workspace-relative path" + ) + return text + + +def _is_safe_path_entry(entry: str) -> bool: + """A single path entry is safe iff it has no ``:``, no ``..``, no control chars.""" + return ( + bool(entry) + and ":" not in entry + and ".." not in Path(entry).parts + and not any(c in entry for c in ("\n", "\r", "\x00")) + ) + + +# Reserved keys carried by dedicated override fields; never accepted via runtime_env. +_RUNTIME_ENV_RESERVED: frozenset[str] = frozenset({"PATH", "PYTHONPATH", "LD_LIBRARY_PATH"}) + + +def apply_runtime_override(envs: dict[str, str], override: dict[str, Any]) -> None: + """Inject an attempt runtime override into the materialized YAML envs dict.""" + if not override: + return + from ._workload_envs import prepend_launch_path + + path_prefix = str(override.get("path_prefix") or "").strip() + if path_prefix: + prepend_launch_path(envs, "PATH", path_prefix) + entrypoint_bin = str(override.get("entrypoint_bin_dir") or "").strip() + if entrypoint_bin: + if _is_safe_path_entry(entrypoint_bin): + prepend_launch_path(envs, "PATH", entrypoint_bin) + else: + log.warning("apply_runtime_override: dropping unsafe entrypoint_bin_dir %r", entrypoint_bin) + pp_prefix = str(override.get("pythonpath_prefix") or "").strip() + if pp_prefix: + if _is_safe_path_entry(pp_prefix): + prepend_launch_path(envs, "PYTHONPATH", pp_prefix) + else: + log.warning("apply_runtime_override: dropping unsafe pythonpath_prefix %r", pp_prefix) + # Multi-entry prefixes: prepend in order so the first entry wins, ahead of any single-dir pythonpath_prefix and + # the inherited value. + for entry in reversed(_coerce_prefix_list(override.get("pythonpath_prefixes"))): + if _is_safe_path_entry(entry): + prepend_launch_path(envs, "PYTHONPATH", entry) + else: + log.warning("apply_runtime_override: dropping unsafe pythonpath entry %r", entry) + # Native loader path: prepend attempt entries while preserving any inherited entries (e.g. /opt/rocm/lib). + for entry in reversed(_coerce_prefix_list(override.get("ld_library_path_prefix"))): + if _is_safe_path_entry(entry): + prepend_launch_path(envs, "LD_LIBRARY_PATH", entry) + else: + log.warning("apply_runtime_override: dropping unsafe ld_library_path entry %r", entry) + _runtime_env = override.get("runtime_env") + if isinstance(_runtime_env, dict): + for raw_k, raw_v in _runtime_env.items(): + key = str(raw_k) + if not _ENV_KEY_RE.match(key) or key in BLOCKED_CHILD_ENV_NAMES or key in _RUNTIME_ENV_RESERVED: + log.warning("apply_runtime_override: dropping unsafe runtime_env key %r", key) + continue + envs[key] = str(raw_v) + for key in ("framework_bin", "framework_python", "framework_venv_root"): + val = str(override.get(key) or "").strip() + if val: + envs[f"HYPERLOOM_{key.upper()}"] = val + # runtime_python_exe takes priority over framework_python as the launch interpreter. + rpe = str(override.get("runtime_python_exe") or "").strip() + if rpe: + envs["HYPERLOOM_FRAMEWORK_PYTHON"] = rpe + + +def _coerce_prefix_list(value: Any) -> list[str]: + """Normalize a runtime-prefix field (list/tuple/str) to a list of entries.""" + if value is None: + return [] + if isinstance(value, str): + return [value.strip()] if value.strip() else [] + if isinstance(value, (list, tuple)): + return [str(v).strip() for v in value if str(v).strip()] + return [] + + +def _build_variant_yaml( + base_yaml_path: Path, + base_extra_args: str, + variant: GridVariant, + *, + output_subdir: Path, + model_path: str | None = None, + gpu_type: str | None = None, + benchmark_script: str | None = None, + server_lifecycle: dict[str, Any] | None = None, + base_args_mode: str = "append", + base_extra_envs: dict[str, str] | None = None, + base_remove_args: list[str] | None = None, + base_unset_envs: list[str] | None = None, +) -> Path: + """Materialize a per-variant Magpie YAML on disk.""" + with base_yaml_path.open(encoding="utf-8") as f: + cfg = yaml.safe_load(f) + bench = cfg.setdefault("benchmark", {}) + replacing = str(base_args_mode).strip().lower() == "replace" + envs = apply_runtime_benchmark_overrides( + bench, + model_path=model_path, + gpu_type=gpu_type, + benchmark_script=benchmark_script, + conc=variant_conc(variant), + ) + extra_args_env = server_args_env_name(bench.get("framework")) + + variant_remove = to_str_list(getattr(variant, "remove_args", [])) + # A replacing base drops the inherited string wholesale, so only the + # variant's own removals still name flags that survive to be stripped. + effective_remove = ( + variant_remove if replacing else list(dict.fromkeys(to_str_list(base_remove_args) + variant_remove)) + ) + combined = compose_server_args( + inherited_args="" if replacing else str(envs.get(extra_args_env, "")), + base_extra_args=base_extra_args, + variant_extra_args=variant.extra_server_args, + remove_args=effective_remove, + args_mode=getattr(variant, "args_mode", "append"), + ) + # A grid variant never injects a MoE runner backend itself, but it does inherit one -- from the baseline recipe it + # was seeded with, or from an explicitly authored variant. + if combined and _SGLANG_MOE_RUNNER_BACKEND_RE.search(combined): + from ._workload_envs import _remove_moe_runner_backend_arg + + if extra_args_env == "EXTRA_SGLANG_ARGS" and moe_runner_requires_aiter(combined, model_path): + log.warning( + "grid: dropping inherited --moe-runner-backend for variant %s: " + "this checkpoint's MoE quant scheme is only implemented on the " + "aiter runner and would crash on the first forward pass.", + variant.name, + ) + combined = _remove_moe_runner_backend_arg(combined) + if combined: + envs[extra_args_env] = _shell_safe_dedupe(combined) + elif extra_args_env in envs or variant.args_mode == "replace" or base_args_mode == "replace": + envs[extra_args_env] = "" + # Composed base-then-variant, so a variant unsetting a key the base sets + # removes it: the last layer to name a key is the one that decides it. + for k in to_str_list(base_unset_envs): + if k.strip().upper() in BLOCKED_EXTERNAL_ENV_NAMES: + continue + envs.pop(k, None) + for k, v in (base_extra_envs or {}).items(): + envs[str(k)] = str(v) + for k in getattr(variant, "unset_envs", []) or []: + # Unsetting a pin retargets the benchmark rather than toggling a knob. + if str(k).strip().upper() in BLOCKED_EXTERNAL_ENV_NAMES: + log.warning("grid: refusing to unset pinned env %s for variant %s", k, variant.name) + continue + envs.pop(str(k), None) + for k, v in variant.extra_envs.items(): + envs[str(k)] = str(v) + from ._workload_envs import pin_mlperf_round_concurrency, prepend_launch_path + + pin_mlperf_round_concurrency(envs) + # The launcher re-exports these unconditionally, so a value carried here is + # one the run never used. + for k in launcher_overwritten_envs(bench) & envs.keys(): + log.warning("grid: dropping %s for variant %s; the recipe overwrites it", k, variant.name) + envs.pop(k, None) + # The three AgentX bounds took this rung's CONC through ``variant_conc`` above, not through this merge: raising + # the client's grace alone would make the round wait inside a cap that did not move with it. + _overlay = str(getattr(variant, "overlay_pythonpath", "") or "").strip() + if _overlay: + # Structural containment on the overlay dir before it is prepended to PYTHONPATH: a legitimate authored-kernel + # overlay is a single existing directory (never a ``:``-joined list, never a ``..`` traversal or control + # char). + _overlay_ok = ( + ":" not in _overlay + and ".." not in Path(_overlay).parts + and not any(c in _overlay for c in ("\n", "\r", "\x00")) + and Path(_overlay).is_dir() + ) + if _overlay_ok: + prepend_launch_path(envs, "PYTHONPATH", _overlay) + else: + log.warning( + "grid: dropping unsafe overlay_pythonpath %r (not a single " + "existing directory / contains separator or traversal)", + _overlay, + ) + + # Attempt runtime override: inject path_prefix/pythonpath_prefix/framework_bin etc. into benchmark.envs so the + # server subprocess resolves the attempt runtime. + _rt_override = getattr(variant, "runtime_override", None) or {} + if _rt_override: + apply_runtime_override(envs, _rt_override) + + # PATH guard: the xdit wrapper needs both `/venv/bin` (the `xdit` console script) and `/opt/rocm/bin` (`hipcc`); + # force-prepend both so an LLM-supplied PATH can't drop one. + if str(bench.get("framework", "")).strip().lower() == "xdit": + _cur_path = str(envs.get("PATH", "") or "") + _parts = [p for p in _cur_path.split(":") if p] + for _essential in ("/opt/rocm/bin", "/venv/bin"): + if _essential not in _parts: + _parts.insert(0, _essential) + envs["PATH"] = ":".join(_parts) + + if server_lifecycle is not None: + from ._server_lifecycle import inject_lifecycle + + inject_lifecycle( + bench, + cleanup=bool(server_lifecycle.get("cleanup", True)), + pid_dir=server_lifecycle["pid_dir"], + port=int(server_lifecycle["port"]), + ) + + # The final write to the argument env; nothing below may touch it. + seal_server_argv(envs, bench.get("framework")) + output_subdir.mkdir(parents=True, exist_ok=True) + out_path = output_subdir / "config.yaml" + with out_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(cfg, f, sort_keys=False) + return out_path + + +def _parse_report(workspace: Path) -> dict[str, Any] | None: + """Load ``benchmark_report.json`` from a benchmark workspace.""" + report = workspace / "benchmark_report.json" + if not report.exists(): + return None + try: + with report.open(encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + return None + return data if isinstance(data, dict) else None + + +# How long to keep re-reading ``benchmark_report.json`` when the report does not yet parse into a valid measurement. +REPORT_SETTLE_SECONDS = 30.0 +REPORT_SETTLE_POLL_SECONDS = 1.0 + + +async def _settled_measurement( + workspace: Path, + *, + subprocess_started_unix: float | None, + settle_seconds: float = REPORT_SETTLE_SECONDS, + poll_seconds: float = REPORT_SETTLE_POLL_SECONDS, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + """Read the benchmark report, re-reading briefly while it is still settling.""" + deadline = time.monotonic() + max(0.0, float(settle_seconds)) + attempts = 0 + while True: + report = _parse_report(workspace) + measurement = extract_benchmark_measurement( + report, + workspace=workspace, + subprocess_started_unix=subprocess_started_unix, + ) + attempts += 1 + if measurement.get("valid_measurement") or time.monotonic() >= deadline: + if attempts > 1 and measurement.get("valid_measurement"): + log.info( + "grid_runner: benchmark_report became valid after %d read(s); " + "the report was still being written when the subprocess was reaped", + attempts, + ) + # Harvest already tried this, but it runs before the report is guaranteed to exist: a report Magpie + # finishes writing after the subprocess is reaped would otherwise leave the round with no GPU artifact at + # all. Here the report is in hand, so write it from that rather than reading the file a second time. + if report is not None: + write_gpu_metrics_from_report(workspace, report, source="benchmark_report.json") + return report, measurement + await asyncio.sleep(max(0.01, float(poll_seconds))) + + +def _run_grid_warmup_enabled() -> bool: + """Whether ``run_grid`` should discard a cold warmup round when possible.""" + return env_flag("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", default=not os.environ.get("PYTEST_CURRENT_TEST")) + + +def _prepend_magpie_pythonpath(magpie_dir: str, current_pythonpath: str) -> str: + """Prepend Magpie's import root to PYTHONPATH, skipping package-root dirs.""" + if not magpie_dir or is_python_package_root(magpie_dir): + return current_pythonpath + return f"{magpie_dir}:{current_pythonpath}" if current_pythonpath else magpie_dir + + +def sync_benchmark_timeout(config_path: Path, timeout_sec: float) -> None: + """Give Magpie and bypass the same cap as the enclosing benchmark process.""" + cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) + bench = cfg["benchmark"] + bench["timeout_seconds"] = timeout_sec + if bench.get("server_lifecycle"): + bench["server_lifecycle"]["server_ready_timeout_s"] = timeout_sec + envs = bench.setdefault("envs", {}) + envs["PYTHONUNBUFFERED"] = "1" + sync_agentx_phase_wait(envs, timeout_sec) + config_path.write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8") + + +def _benchmark_server_log(config_path: Path, output_dir: Path) -> str | None: + """Scriptable benchmarks have no server whose readiness can arm silence.""" + cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) + framework = str(cfg["benchmark"].get("framework", "")).lower() + return None if framework in {"custom", "xdit"} else str(output_dir / "server.log") + + +def _run_magpie( + *, + magpie_python: str, + config_path: Path, + output_dir: Path, + timeout_sec: int, + cwd: str, + result_dir: str | None = None, + silence_timeout_sec: float | None = None, + server_already_ready: bool = False, + serving_lease: Any = None, + on_output: Callable[[], None] | None = None, + session_deadline_sec: float | None = None, + unset_envs: list[str] | None = None, +) -> tuple[int, str, str]: + """Blocking subprocess wrapper. Returns (rc, stdout, stderr).""" + sync_benchmark_timeout(config_path, timeout_sec) + server_log_path = _benchmark_server_log(config_path, output_dir) + env = scrub_benchmark_process_env(os.environ.copy()) + env["PYTHONUNBUFFERED"] = "1" + from ._workload_envs import resolve_reference_launch + + _args, _envs, reference_controls = resolve_reference_launch() + for name in [*(reference_controls.get("unset_envs") or []), *(unset_envs or [])]: + if name.strip().upper() not in BLOCKED_EXTERNAL_ENV_NAMES: + env.pop(name, None) + env["PATH"] = f"/opt/venv/bin:{env.get('PATH', '')}" + magpie_dir = os.environ.get("MAGPIE_PATH") or "" + if magpie_dir: + env["PYTHONPATH"] = _prepend_magpie_pythonpath(magpie_dir, env.get("PYTHONPATH", "")) + + # Multi-node: tell Magpie to skip its local-server launch and point benchmark_serving at the head pod's ClusterIP. + from ._multi_node_env import magpie_remote_env + + env.update(magpie_remote_env()) + + # Pin Magpie's InferenceX resolution to ``$INFERENCEX_PATH`` (its highest-precedence rung) so it loads the patched + # checkout, not a stale copy. + inferencex_path = os.environ.get("INFERENCEX_PATH", "").strip() + if inferencex_path: + env["MAGPIE_INFERENCEX_PATH"] = inferencex_path + # Baseline patches its own checkout, but explore / sweep never pass through that hook: re-assert here so a + # resumed session or a re-cloned checkout still emits the eval-start marker. + ensure_benchmark_lib_eval_start_patched(Path(inferencex_path)) + + # The generation bounds + pathology probe are asserted whether or not ``$INFERENCEX_PATH`` is set: unset falls + # back to the same env discovery the baseline arm uses ($MAGPIE_PATH/InferenceX). + probe_root = Path(inferencex_path) if inferencex_path else None + # Best-effort, unlike the probe below: a missing guard only costs the reason a failed round reports. + ensure_eval_unbound_outputs_patched(probe_root) + if not ensure_eval_probe_patched(probe_root) and not materialized_run_eval_disabled(config_path): + eval_bounds_msg = ( + "eval generation bounds + pathology probe are not installed " + "(utils/evals/patches/lm_eval_sitecustomize.py, inferencex=" + f"{inferencex_path or ''}, INFERENCEX_PATH=" + f"{os.environ.get('INFERENCEX_PATH', '') or ''}); this " + "variant runs eval, so it would be scored against a baseline that " + "truncated at a different point" + ) + if eval_probe_targets_exist(probe_root): + log.error("grid_runner: %s; failing this benchmark", eval_bounds_msg) + return (EVAL_PROBE_UNPATCHABLE_RETURNCODE, "", eval_bounds_msg) + log.warning("grid_runner: %s", eval_bounds_msg) + # AgentX: deploy the aiperf client into InferenceX ``benchmarks/`` + preflight aiperf right before Magpie runs it, + # via the shared helper (also used by the baseline/profile shell-out). + from ._workload_envs import prepare_agentx_runtime + + _agx_err = prepare_agentx_runtime(env=env, inferencex_path=inferencex_path, config_path=config_path) + if _agx_err: + log.error("%s; failing this benchmark", _agx_err) + return (AGENTX_PREFLIGHT_RETURNCODE, "", _agx_err) + # RESULT_DIR default; leaks are picked up by the salvage path. + env["RESULT_DIR"] = result_dir or str(output_dir) + # InferenceX ``run_lm_eval`` cleans ``$EVAL_RESULT_DIR`` after processing lm-eval output. + env["EVAL_RESULT_DIR"] = str(Path(env["RESULT_DIR"]) / "eval_output") + # Pin SERVER_LOG / GPU_METRICS_CSV per-task so logs land alongside ``benchmark_report.json``. + env["SERVER_LOG"] = str(output_dir / "server.log") + env["GPU_METRICS_CSV"] = str(output_dir / "gpu_metrics.csv") + + # Ray-managed GPU execution (§12 T1): route the round through the serving lease's actor, which holds ``num_gpus`` + # across every round sharing this server. + if serving_lease is not None: + from ._ray_backend import strip_visible_devices_from_config + + ray_config_path = strip_visible_devices_from_config(config_path) + cmd = build_benchmark_command( + python_exe=magpie_python, + config_path=ray_config_path, + output_dir=output_dir, + ) + # ``on_output`` cannot follow the round here: the benchmark runs inside a Ray actor in another process + # (potentially on another node) and only its final ``(rc, stdout, stderr)`` comes back, so there is nothing + # local to call per line. + return serving_lease.run_session_kill( + cmd, + env=env, + cwd=cwd, + timeout=timeout_sec, + silence_timeout_sec=silence_timeout_sec, + server_log_path=server_log_path, + server_already_ready=server_already_ready, + # Converted here, at the last moment before the process boundary: the actor cannot read this process's + # monotonic clock. + session_remaining_sec=session_deadline_to_remaining_sec(session_deadline_sec), + ) + + cmd = build_benchmark_command( + python_exe=magpie_python, + config_path=config_path, + output_dir=output_dir, + ) + # The launch puts Magpie in its own POSIX session and tears down the whole + # descendant tree on every exit path. + proc = run_with_session_kill( + cmd, + env=env, + cwd=cwd, + timeout=timeout_sec, + silence_timeout_sec=silence_timeout_sec, + server_log_path=server_log_path, + server_already_ready=server_already_ready, + on_output=on_output, + session_deadline_sec=session_deadline_sec, + ) + return proc.returncode, proc.stdout or "", proc.stderr or "" + + +def _num_gpus_for_config(config_path: Path) -> float: + """Read the tensor-parallel size (``TP``) from a materialized benchmark YAML.""" + try: + with Path(config_path).open(encoding="utf-8") as fp: + cfg = yaml.safe_load(fp) or {} + envs = (cfg.get("benchmark") or {}).get("envs") or {} + return float(int(envs.get("TP", 1) or 1)) + except Exception: # noqa: BLE001 — best-effort; default to 1 GPU + return 1.0 + + +def _mn_restart_env( + reference_envs: dict[str, str], + variant: Any, + unset_envs: list[str], +) -> dict[str, str]: + """Merge the reference server envs under one variant's own overrides.""" + merged = {k: v for k, v in reference_envs.items() if k not in unset_envs} + merged.update({str(k): str(v) for k, v in (variant.extra_envs or {}).items()}) + return merged + + +def _resolve_mn_effective_server_args( + cfg_path: Path, + base_yaml_path: Path, + variant: Any, + *, + base_extra_args: str, + base_args_mode: str, +) -> str: + """Resolve the multi-node server args for a variant restart; prefer the materialized variant YAML, falling back to a recompose from the base YAML.""" + try: + with cfg_path.open(encoding="utf-8") as _f: + _variant_cfg = yaml.safe_load(_f) or {} + _variant_bench = _variant_cfg.get("benchmark") or {} + _variant_envs = _variant_bench.get("envs") or {} + _variant_framework_env = server_args_env_name(_variant_bench.get("framework")) + return str(_variant_envs.get(_variant_framework_env) or "") + except Exception: + log.debug( + "grid_runner: failed to read materialized variant args from %s", + cfg_path, + exc_info=True, + ) + try: + with base_yaml_path.open(encoding="utf-8") as _f: + _base_cfg = yaml.safe_load(_f) or {} + _base_bench = _base_cfg.get("benchmark") or {} + _base_envs = _base_bench.get("envs") or {} + _base_framework_env = server_args_env_name(_base_bench.get("framework")) + _fallback_inherited_args = str(_base_envs.get(_base_framework_env) or "") + except Exception: # noqa: BLE001 - best-effort parity fallback + _fallback_inherited_args = "" + return _shell_safe_dedupe( + compose_server_args( + inherited_args="" if str(base_args_mode).strip().lower() == "replace" else _fallback_inherited_args, + base_extra_args=base_extra_args, + variant_extra_args=variant.extra_server_args, + remove_args=getattr(variant, "remove_args", []), + args_mode=getattr(variant, "args_mode", "append"), + ) + ) + + +def _variant_progress_note( + grid: list[GridVariant], + results: list[VariantResult], + idx: int, +) -> dict[str, Any]: + """Build the progress note for the variant at ``idx`` from that variant's own row.""" + landed = results[idx] if idx < len(results) else None + return { + "unit": "variant", + "label": grid[idx].name, + "index": idx + 1, + "total": len(grid), + "status": getattr(landed, "status", None), + "output_throughput": getattr(landed, "output_throughput", None), + } + + +# The returncode side of :mod:`...stop_attribution`: the same two causes, keyed by the sentinel a subprocess comes +# back with. +_STOPPED_BY_THE_RUN: dict[int, StoppedByTheRun] = { + SESSION_TIME_EXHAUSTED_RETURNCODE: STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + ORCHESTRATOR_CANCELLED_RETURNCODE: STOPPED_BY_THE_RUN[ORCHESTRATOR_CANCELLED_CLASS], +} + + +def stopped_by_the_run(returncode: int | None) -> StoppedByTheRun | None: + """Return how to record a round the run itself stopped, if it did.""" + if returncode is None: + return None + return _STOPPED_BY_THE_RUN.get(int(returncode)) + + +def variant_conc(variant: Any) -> int | None: + """The concurrency a variant will run at, or ``None`` when it names none.""" + raw = (getattr(variant, "extra_envs", None) or {}).get("CONC") + try: + conc = int(raw) + except (TypeError, ValueError): + return None + return conc if conc > 0 else None + + +def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: + """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call.""" + if shared_state is None: + return (None, None) + deadline_fn = getattr(shared_state, "grid_session_deadline_sec", None) + deadline = deadline_fn() if callable(deadline_fn) else None + variant_sec = _phase_state.one_more_measurement_sec(shared_state) + if variant_sec is None: + variant_sec = _phase_state.measured_seconds(shared_state, "baseline_runtime_sec") + return (deadline, variant_sec) + + +async def run_grid( + *, + base_yaml_path: Path, + base_extra_args: str, + grid: list[GridVariant], + output_root: Path, + magpie_python: str | None = None, + keep_going_on_failure: bool = True, + model_path: str | None = None, + gpu_type: str | None = None, + benchmark_script: str | None = None, + result_dir: str | None = None, + server_lifecycle: dict[str, Any] | None = None, + base_args_mode: str = "append", + base_extra_envs: dict[str, str] | None = None, + base_remove_args: list[str] | None = None, + base_unset_envs: list[str] | None = None, + warmup_before_measure: bool | None = None, + server_already_ready: bool = False, + ready_server_log: str | None = None, + serving_lease: Any = None, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, + deadline_stop: StoppedByTheRun = STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + lifecycle_boot_only: bool = False, +) -> list[VariantResult]: + """Execute variants; ``deadline_stop`` names the owner of the supplied deadline. + + ``ready_server_log`` is the log of the caller's ready server, measured when no variant log exists. + """ + silence_timeout_sec, benchmark_timeout_sec = resolve_benchmark_timeouts() + if not magpie_python: + # Backend-aware: bypass uses a plain python3, not Magpie's venv. + from .benchmark_backend import resolve_benchmark_interpreter + + magpie_python = resolve_benchmark_interpreter() + else: + magpie_python = _validate_magpie_python_override(magpie_python) + if warmup_before_measure is None: + warmup_before_measure = _run_grid_warmup_enabled() + auto_warmup_requested = bool(warmup_before_measure and server_lifecycle is None) + results: list[VariantResult] = [] + # This function names the working directory, so it creates it: callers and the per-variant config writer both + # happen to create it first today, and neither is a contract. + output_root.mkdir(parents=True, exist_ok=True) + cwd = str(output_root) + + # Reap orphaned aiter JIT build locks before booting any server. + try: + from ._aiter_jit import sweep_stale_aiter_locks_if_dead + + _lock_sweep = sweep_stale_aiter_locks_if_dead() + if _lock_sweep.get("deleted"): + log.warning( + "grid_runner: reaped %d orphaned aiter JIT lock(s) under %s before server launch (compiler_alive=%s)", + _lock_sweep.get("deleted"), + _lock_sweep.get("dir"), + _lock_sweep.get("compiler_alive"), + ) + except Exception as exc: # noqa: BLE001 — best-effort hygiene, never blocks the grid + log.debug("grid_runner: aiter lock sweep swallowed: %r", exc) + + # Multi-node: the reference recipe's server envs do not reach the variant restart through + # restart_server_for_round, which never reads them. + from ._multi_node_env import is_multi_node as _mn_is_multi_node + + _mn_ref_envs: dict[str, str] = {} + if _mn_is_multi_node(): + from ._workload_envs import resolve_reference_base + + _, _mn_ref_envs = resolve_reference_base() + + # Reported on entry, not on completion: ``_report_finished_variant`` only runs once a result has been appended, so + # a first variant that hangs — or a branch that raises before reaching it — would emit nothing at all, which is + # exactly the silence the heartbeat exists to break. + async def _unit_started(idx: int, label: str) -> None: + """Report that a unit of variant ``idx`` is about to start.""" + await report_progress( + unit="variant_step", + label=f"{grid[idx].name}:{label}", + index=idx + 1, + total=len(grid), + status="started", + ) + + async def _reported_magpie(idx: int, label: str, **kwargs: Any) -> tuple[int, str, str]: + """Run one Magpie pass, announced on entry and kept alive by its output.""" + await _unit_started(idx, label) + async with heartbeat_while_output_flows( + unit="variant_step", + label=f"{grid[idx].name}:{label}", + index=idx + 1, + total=len(grid), + ) as activity: + return await asyncio.to_thread( + _run_magpie, on_output=activity.note, unset_envs=grid[idx].unset_envs, **kwargs + ) + + # Variant boundary: a progress heartbeat so a grid that runs for hours is distinguishable from one that hung on + # its first variant. + async def _report_finished_variant(idx: int) -> None: + """Report the variant that just landed.""" + await report_progress(**_variant_progress_note(grid, results, idx)) + + def _record_round_stop( + stopped: StoppedByTheRun, + *, + idx: int, + variant: GridVariant, + slot: Path, + round_label: str, + returncode: int | None, + started_unix: float, + server_log: Path, + server_survivors: list[int], + ) -> bool: + """Record a round the run stopped and say whether the grid is over. + + ``server_survivors`` are the servers the variant booted that outlived its teardown. + """ + if returncode == SESSION_TIME_EXHAUSTED_RETURNCODE: + stopped = deadline_stop + runtime_sec = round(max(0.0, time.time() - started_unix), 2) + log.warning( + "grid_runner: variant %d/%d name=%s %s round reaped after %.1fs: %s; recorded as skipped, not failed", + idx + 1, + len(grid), + variant.name, + round_label, + runtime_sec, + stopped.interrupted, + ) + teardown = "tree reaped" + if server_survivors: + teardown += f", but server pid(s) {', '.join(map(str, server_survivors))} survived SIGKILL" + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=stopped.error_class, + error_summary=f"{stopped.interrupted}; {teardown}", + extra_args=variant.extra_server_args, + ) + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="skipped", + returncode=returncode, + runtime_sec=runtime_sec, + error=stopped.interrupted, + error_class=stopped.error_class, + server_log_path=_measurement_server_log_path(server_log, slot=slot), + note=variant.note, + nonfatal_warnings=[f"server_survived_teardown:{pid}" for pid in server_survivors], + ) + ) + if stopped.ends_the_batch: + results.extend(_not_run_skip_result(rest, stopped) for rest in grid[idx + 1 :]) + return True + return not keep_going_on_failure + + # How many full benchmark passes one variant costs. + _mn_warmup_rounds = 0 + if variant_expected_sec is not None: + from ._multi_node_env import ( + is_multi_node as _mn_is_multi_node, + mn_bench_warmup_enabled as _mn_bench_warmup_enabled, + ) + + _mn_warmup_rounds = 1 if (_mn_is_multi_node() and _mn_bench_warmup_enabled()) else 0 + variant_rounds = 1 + (1 if auto_warmup_requested else 0) + _mn_warmup_rounds + + def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant_rounds) -> bool: + """Skip variant ``idx`` and every one after it, or say the budget still fits.""" + if session_deadline_sec is None: + return False + remaining_sec = session_deadline_sec - time.monotonic() + required_sec = float(variant_expected_sec) * rounds_left if variant_expected_sec is not None else None + if remaining_sec > 0 and (required_sec is None or remaining_sec >= required_sec): + return False + log.warning( + "grid_runner: %.0fs left cannot admit this variant's %d remaining round(s) " + "(expected_sec=%s, spent on: %s); skipping %d remaining variant(s)", + max(0.0, remaining_sec), + rounds_left, + required_sec, + spent_on, + len(grid) - idx, + ) + results.extend(_not_run_skip_result(variant, deadline_stop) for variant in grid[idx:]) + return True + + for i, variant in enumerate(grid): + try: + # Session-budget stop: skip the remaining variants once the wall-clock deadline is reached or the remaining + # budget cannot fit another variant, so a timeout halts the grid instead of draining it (and the last variant + # cannot overrun the close window). + if _skip_rest_for_budget(i, spent_on="the variants before this one"): + break + await _unit_started(i, "variant") + slot = output_root / f"variant_{i:02d}_{_safe(variant.name)}" + server_log = slot / "server.log" + # Capability fast-fail: drop a variant whose env flag the build cannot honour before booting a doomed server, + # still recording the failure so the LLM learns not to re-pick it. + cap_reason = unsupported_capability_reason(variant) or ungrounded_env_file_reason(variant) + if cap_reason: + log.warning( + "grid_runner: variant %d/%d name=%s aborted: capability_unsupported: %s", + i + 1, + len(grid), + variant.name, + cap_reason, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="capability_unsupported", + error_summary=cap_reason, + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=cap_reason, + error_class="capability_unsupported", + note=variant.note, + ) + ) + lifecycle: dict[str, Any] = {"eligible": False} + auto_warmup = False + try: + cfg_path = _build_variant_yaml( + base_yaml_path, + base_extra_args, + variant, + output_subdir=slot, + model_path=model_path, + gpu_type=gpu_type, + benchmark_script=benchmark_script, + server_lifecycle=server_lifecycle, + base_args_mode=base_args_mode, + base_extra_envs=base_extra_envs, + base_remove_args=base_remove_args, + base_unset_envs=base_unset_envs, + ) + except Exception as exc: # noqa: BLE001 + build_error = "yaml_build_error" + log.warning( + "grid_runner: variant %d/%d name=%s aborted: %s: %r", + i + 1, + len(grid), + variant.name, + build_error, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=build_error, + error_summary=repr(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"{build_error}: {exc!r}", + error_class=build_error, + note=variant.note, + ) + ) + + _mn_effective_args = _resolve_mn_effective_server_args( + cfg_path, + base_yaml_path, + variant, + base_extra_args=base_extra_args, + base_args_mode=base_args_mode, + ) + + if auto_warmup_requested: + try: + from ._server_lifecycle import resolve_lifecycle_params + + lifecycle = resolve_lifecycle_params(cfg_path) + auto_warmup = bool(lifecycle.get("eligible")) + if auto_warmup: + cfg_path = _build_variant_yaml( + base_yaml_path, + base_extra_args, + variant, + output_subdir=slot, + model_path=model_path, + gpu_type=gpu_type, + benchmark_script=benchmark_script, + server_lifecycle={ + "cleanup": True, + "pid_dir": str(slot), + "port": int(lifecycle.get("port") or 0), + }, + base_args_mode=base_args_mode, + base_extra_envs=base_extra_envs, + base_remove_args=base_remove_args, + base_unset_envs=base_unset_envs, + ) + else: + log.info( + "grid_runner: warmup-before-measure not eligible (%s); running single measured round.", + lifecycle.get("reason") or "unknown", + ) + except Exception as exc: # noqa: BLE001 + log.warning( + "grid_runner: warmup-before-measure eligibility/materialization failed (%r); running single measured round.", + exc, + ) + auto_warmup = False + + warmup_tput: float | None = None + if auto_warmup: + warmup_slot = slot / "warmup_round" + warmup_server_log = warmup_slot / "server.log" + warmup_lifecycle = { + "cleanup": False, + "pid_dir": str(slot), + "port": int(lifecycle.get("port") or 0), + } + try: + warmup_cfg_path = _build_variant_yaml( + base_yaml_path, + base_extra_args, + variant, + output_subdir=warmup_slot, + model_path=model_path, + gpu_type=gpu_type, + benchmark_script=benchmark_script, + server_lifecycle=warmup_lifecycle, + base_args_mode=base_args_mode, + base_extra_envs=base_extra_envs, + base_remove_args=base_remove_args, + base_unset_envs=base_unset_envs, + ) + except Exception as exc: # noqa: BLE001 + log.warning( + "grid_runner: variant %d/%d name=%s aborted: warmup_yaml_build_error: %r", + i + 1, + len(grid), + variant.name, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="warmup_yaml_build_error", + error_summary=repr(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"warmup_yaml_build_error: {exc!r}", + error_class="warmup_yaml_build_error", + note=variant.note, + ) + ) + + warmup_workspaces_before = snapshot_workspaces(warmup_slot) + warmup_started_unix = time.time() + # Held in a local because the abort line below has to name the cap the round was actually granted: the + # declared one is a hang backstop, and a round killed at the reserved cap logged as a two-hour timeout + # reads as a variant that hangs rather than a budget that ran out. + warmup_cap_sec = benchmark_timeout_sec + try: + warmup_rc, warmup_stdout, warmup_stderr = await _reported_magpie( + i, + "warmup", + magpie_python=magpie_python, + config_path=warmup_cfg_path, + output_dir=warmup_slot, + timeout_sec=warmup_cap_sec, + silence_timeout_sec=silence_timeout_sec, + cwd=cwd, + result_dir=result_dir, + serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + ) + except subprocess.TimeoutExpired as exc: + _teardown_variant_server(slot, lifecycle) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: warmup timeout (timeout_sec=%d): %s", + i + 1, + len(grid), + variant.name, + warmup_cap_sec, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="warmup_magpie_timeout", + error_summary=str(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"warmup_timeout: {exc}", + error_class="warmup_magpie_timeout", + server_log_path=_existing_log_path(warmup_server_log), + note=variant.note, + runtime_sec=round(max(0.0, time.time() - warmup_started_unix), 2), + nonfatal_warnings=["run_grid_warmup_round_failed"], + ) + ) + + warmup_stopped = stopped_by_the_run(warmup_rc) + if warmup_stopped is not None: + warmup_survivors = _teardown_variant_server(slot, lifecycle) + grid_is_over = _record_round_stop( + warmup_stopped, + idx=i, + variant=variant, + slot=slot, + round_label="warmup", + returncode=warmup_rc, + started_unix=warmup_started_unix, + server_log=warmup_server_log, + server_survivors=warmup_survivors, + ) + await _report_finished_variant(i) + if grid_is_over: + break + continue + + warmup_run_ws = select_run_workspace(warmup_slot, known_before=warmup_workspaces_before) + warmup_workspace = warmup_run_ws if warmup_run_ws is not None else warmup_slot + warmup_harvested = harvest_leaked_artifacts( + warmup_workspace, + subprocess_started_unix=warmup_started_unix, + ) + if warmup_run_ws is not None: + _, warmup_measurement = await _settled_measurement( + warmup_workspace, + subprocess_started_unix=warmup_started_unix, + settle_seconds=REPORT_SETTLE_SECONDS if warmup_rc == 0 else 0.0, + ) + else: + warmup_measurement = extract_benchmark_measurement( + None, + workspace=warmup_workspace, + subprocess_started_unix=warmup_started_unix, + ) + if warmup_rc != 0 or not warmup_measurement.get("valid_measurement"): + _teardown_variant_server(slot, lifecycle) + warmup_error = ( + server_log_death_excerpt(str(warmup_server_log)) + or redact_secret_values((warmup_stderr or warmup_stdout)[-2000:]) + if warmup_rc != 0 + else "warmup benchmark_report missing valid throughput/completed requests" + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: warmup_round_failed (rc=%s): %s", + i + 1, + len(grid), + variant.name, + warmup_rc, + warmup_error[:200], + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="warmup_round_failed", + error_summary=warmup_error, + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + workspace=str(warmup_workspace) if warmup_run_ws is not None else None, + report_path=( + str(warmup_workspace / "benchmark_report.json") + if (warmup_workspace / "benchmark_report.json").exists() + else None + ), + raw_result_path=warmup_measurement.get("raw_result_path"), + reported_success=warmup_measurement.get("reported_success"), + returncode=warmup_rc, + error=warmup_error, + error_class="warmup_round_failed", + server_log_path=_existing_log_path(warmup_server_log), + note=variant.note, + runtime_sec=round(max(0.0, time.time() - warmup_started_unix), 2), + nonfatal_warnings=[ + "run_grid_warmup_round_failed", + *[f"harvested_leaked_artifact:{src}" for src, _ in warmup_harvested], + ], + ) + ) + warmup_tput = warmup_measurement.get("output_throughput") + log.info( + "grid_runner: variant %s warmup tput=%.1f tok/s discarded; measuring hot round next", + variant.name, + warmup_tput or 0.0, + ) + + from ._multi_node_env import log_mn_banner + + log_mn_banner( + "grid_runner", + log, + variant=f"{i + 1}/{len(grid)}:{variant.name}", + ) + log.info( + "grid_runner: variant %d/%d name=%s args=%s", + i + 1, + len(grid), + variant.name, + variant.extra_server_args, + ) + + # Multi-node only: restart sglang/vllm with this variant's flags so each row runs against a fresh server. + from ._multi_node_server_lifecycle import ( + ServerRestartFailed, + restart_server_for_round, + ) + + try: + # PD knobs auto-resolved from $PD_* env; PD config stays constant across variants within one run. + _variant_unset = [str(k) for k in getattr(variant, "unset_envs", []) or [] if str(k).strip()] + _restart_env = _mn_restart_env(_mn_ref_envs, variant, _variant_unset) + await restart_server_for_round( + extra_server_args=_mn_effective_args, + # Reference recipe envs under this variant's own overrides (e.g. MORI_* MoE-dispatch tuning) so + # server-side env knobs proposed by specialists take effect on the restarted sglang without dropping + # the envs the baseline was measured with. + extra_env=_restart_env, + unset_env=_variant_unset, + model_path=model_path, + ep=int(os.environ.get("EP") or 0) or None, + ) + except ServerRestartFailed as exc: + log.warning( + "grid_runner: variant %d/%d name=%s aborted: mn_server_restart_failed: %s", + i + 1, + len(grid), + variant.name, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="mn_server_restart_failed", + error_summary=str(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"mn_server_restart_failed: {exc}", + error_class="mn_server_restart_failed", + note=variant.note, + ) + ) + + # The restart above is a launch of its own and is under no cap, so the budget admitted at the top of the loop + # may no longer be there. + if _skip_rest_for_budget( + i, + spent_on="this variant's server restart", + rounds_left=1 + _mn_warmup_rounds, + ): + if auto_warmup: + _teardown_variant_server(slot, lifecycle) + break + + # Multi-node client warmup: one discarded benchmark pass against the just-restarted, persistent remote server + # to warm JIT / steady-state before the measured pass. + from ._multi_node_env import ( + is_multi_node as _mn_imn, + mn_bench_warmup_enabled as _mn_warm, + ) + + if _mn_imn() and _mn_warm(): + _mn_warm_slot = slot / "mn_warmup" + _mn_warm_started_unix = time.time() + # The measurement is discarded, but the returncode is not: a warmup the run stopped is the same stop as + # one in the measured round, and discarding it launches the measured round after the cancel. + _mn_warm_rc: int | None = None + _mn_warm_rc, _, _ = await _reported_magpie( + i, + "mn_warmup", + magpie_python=magpie_python, + config_path=cfg_path, + output_dir=_mn_warm_slot, + timeout_sec=benchmark_timeout_sec, + silence_timeout_sec=silence_timeout_sec, + server_already_ready=True, + cwd=cwd, + result_dir=None, + serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + ) + log.info( + "grid_runner: MN warmup pass done (discarded) %d/%d name=%s rc=%s", + i + 1, + len(grid), + variant.name, + _mn_warm_rc, + ) + _mn_warm_stopped = stopped_by_the_run(_mn_warm_rc) + if _mn_warm_stopped is not None: + _mn_warm_survivors = _teardown_variant_server(slot, lifecycle if auto_warmup else None) + grid_is_over = _record_round_stop( + _mn_warm_stopped, + idx=i, + variant=variant, + slot=slot, + round_label="mn_warmup", + returncode=_mn_warm_rc, + started_unix=_mn_warm_started_unix, + server_log=_mn_warm_slot / "server.log", + server_survivors=_mn_warm_survivors, + ) + await _report_finished_variant(i) + if grid_is_over: + break + continue + + # Snapshot wall-clock before launch so the salvage path can mtime-gate leak destinations per-variant. + slot_workspaces_before = snapshot_workspaces(slot) + variant_started_unix = time.time() + measure_cap_sec = benchmark_timeout_sec + try: + rc, stdout, stderr = await _reported_magpie( + i, + "benchmark", + magpie_python=magpie_python, + config_path=cfg_path, + output_dir=slot, + timeout_sec=measure_cap_sec, + cwd=cwd, + result_dir=result_dir, + silence_timeout_sec=silence_timeout_sec, + server_already_ready=(server_already_ready or auto_warmup or _mn_imn()), + serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + ) + except subprocess.TimeoutExpired as exc: + # Harvest pre-timeout leaks. + to_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot + to_harvested = harvest_leaked_artifacts( + to_destination, + subprocess_started_unix=variant_started_unix, + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: magpie timeout (timeout_sec=%d): %s", + i + 1, + len(grid), + variant.name, + measure_cap_sec, + exc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="magpie_timeout", + error_summary=str(exc), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + error=f"timeout: {exc}", + error_class="magpie_timeout", + server_log_path=_existing_log_path(server_log), + note=variant.note, + runtime_sec=round( + max(0.0, time.time() - variant_started_unix), + 2, + ), + nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in to_harvested], + ) + ) + finally: + round_survivors = _teardown_variant_server(slot, lifecycle if auto_warmup else None) + + # Eval bounds could not be installed for a variant that runs eval, so nothing launched. + if rc == EVAL_PROBE_UNPATCHABLE_RETURNCODE: + variant_runtime_sec = round(max(0.0, time.time() - variant_started_unix), 2) + log.error( + "grid_runner: variant %d/%d name=%s aborted: eval_probe_unpatchable: %s", + i + 1, + len(grid), + variant.name, + stderr, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="eval_probe_unpatchable", + error_summary=stderr, + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + returncode=rc, + runtime_sec=variant_runtime_sec, + error=stderr, + error_class="eval_probe_unpatchable", + note=variant.note, + ) + ) + + # Server-liveness watchdog fired: engine/worker bootstrap died but the parent hung. + if rc == SERVER_DEAD_RETURNCODE: + variant_runtime_sec = round( + max(0.0, time.time() - variant_started_unix), + 2, + ) + sd_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot + sd_harvested = harvest_leaked_artifacts( + sd_destination, + subprocess_started_unix=variant_started_unix, + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: " + "server_init_dead (engine/worker bootstrap failed; parent " + "hung) after %.1fs", + i + 1, + len(grid), + variant.name, + variant_runtime_sec, + ) + death_excerpt = server_log_death_excerpt(str(server_log)) or ( + "server engine/worker init failed; parent process hung and was reaped by the liveness watchdog" + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="server_init_dead", + error_summary=death_excerpt, + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + returncode=rc, + runtime_sec=variant_runtime_sec, + error=death_excerpt, + error_class="server_init_dead", + server_log_path=_existing_log_path(server_log), + note=variant.note, + nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in sd_harvested], + ) + ) + + # Detokenizer-stall watchdog fired: the server came up healthy but then went silent (hung engine / wedged + # detokenizer). + if rc == DETOKENIZER_STALL_RETURNCODE: + variant_runtime_sec = round( + max(0.0, time.time() - variant_started_unix), + 2, + ) + ds_destination = select_run_workspace(slot, known_before=slot_workspaces_before) or slot + ds_harvested = harvest_leaked_artifacts( + ds_destination, + subprocess_started_unix=variant_started_unix, + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: detokenizer_stall " + "(server ready but log went silent) after %.1fs", + i + 1, + len(grid), + variant.name, + variant_runtime_sec, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="detokenizer_stall", + error_summary=( + "server reported ready but emitted no log output (hung " + "engine / detokenizer stall); reaped by the " + "detokenizer-stall watchdog" + ), + extra_args=variant.extra_server_args, + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + returncode=rc, + runtime_sec=variant_runtime_sec, + error="detokenizer_stall: server ready but log went silent", + error_class="detokenizer_stall", + server_log_path=_existing_log_path(server_log), + note=variant.note, + nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in ds_harvested], + ) + ) + + stopped = stopped_by_the_run(rc) + if stopped is not None: + grid_is_over = _record_round_stop( + stopped, + idx=i, + variant=variant, + slot=slot, + round_label="measured", + returncode=rc, + started_unix=variant_started_unix, + server_log=server_log, + server_survivors=round_survivors, + ) + await _report_finished_variant(i) + if grid_is_over: + break + continue + + workspace = select_run_workspace(slot, known_before=slot_workspaces_before) + # Always-on artifact harvest so each slot keeps its server.log / gpu_metrics / profile relay for Robustness + # RCA. + harvest_destination = workspace if workspace is not None else slot + harvested = harvest_leaked_artifacts( + harvest_destination, + subprocess_started_unix=variant_started_unix, + ) + if harvested: + log.info( + "_grid_runner: variant=%s harvested %d leaked artifact(s): %s", + variant.name, + len(harvested), + ", ".join(src.name for src, _ in harvested), + ) + if workspace is None: + harvest_tags = [f"harvested_leaked_artifact:{src}" for src, _ in harvested] + # An AgentX preflight abort never reaches Magpie, so of course no workspace exists -- but calling it + # ``no_benchmark_workspace`` erases the one fact that decides what to do next. + agentx_preflight_abort = rc == AGENTX_PREFLIGHT_RETURNCODE + if agentx_preflight_abort: + # Same class the baseline path uses for the same abort, so both routes classify it identically. + no_ws_error_class = AGENTX_PREFLIGHT_ERROR_CLASS + no_ws_error_summary = ( + redact_secret_values((stderr or "").strip()[-2000:]) or "AgentX preflight aborted the round" + ) + else: + no_ws_error_class = "no_benchmark_workspace" + no_ws_error_summary = server_log_death_excerpt(str(server_log)) or ( + redact_secret_values((stderr or stdout)[-2000:]) + if rc != 0 + else "no benchmark_* workspace produced" + ) + log.warning( + "grid_runner: variant %d/%d name=%s aborted: %s (rc=%s)", + i + 1, + len(grid), + variant.name, + no_ws_error_class, + rc, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=no_ws_error_class, + error_summary=no_ws_error_summary, + extra_args=variant.extra_server_args, + ) + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + returncode=rc, + error=no_ws_error_summary, + error_class=no_ws_error_class, + server_log_path=_existing_log_path(server_log), + nonfatal_warnings=harvest_tags, + note=variant.note, + ) + ) + await _report_finished_variant(i) + if agentx_preflight_abort: + # Environment, not variant. + log.error( + "grid_runner: variant %d/%d name=%s aborted on the AgentX preflight; " + "abandoning the remaining %d point(s) -- the client is missing for the " + "whole grid, not for this variant: %s", + i + 1, + len(grid), + variant.name, + len(grid) - (i + 1), + no_ws_error_summary, + ) + # Recorded rather than dropped: a grid that just returns fewer points than it declared reads as a grid + # that was that size. + results.extend( + VariantResult( + name=rest.name, + extra_server_args=rest.extra_server_args, + extra_envs=dict(rest.extra_envs), + status="skipped", + error="AgentX preflight aborted the grid before this variant ran", + error_class=AGENTX_PREFLIGHT_ERROR_CLASS, + note=rest.note, + ) + for rest in grid[i + 1 :] + ) + break + if rc != 0 and not keep_going_on_failure: + break + continue + report_path = workspace / "benchmark_report.json" + # The report is written during shutdown and the reader runs the moment the subprocess is reaped, so the read is + # worth waiting on however the process exited: a non-zero exit can still be a round that served its whole + # protocol, and that verdict is taken from this one read. + report, measurement = await _settled_measurement( + workspace, + subprocess_started_unix=variant_started_unix, + settle_seconds=REPORT_SETTLE_SECONDS, + ) + nonzero_kept_error: str | None = None + warnings = list(measurement.pop("nonfatal_warnings", []) or []) + for leak_src, _ in harvested: + warnings.append(f"harvested_leaked_artifact:{leak_src}") + if warmup_tput is not None: + warnings.append("run_grid_warmup_discarded_first") + warnings.append(f"warmup_round_tput:{float(warmup_tput):.1f}") + + if not measurement.get("valid_measurement"): + if lifecycle_boot_only and rc == 0: + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="succeeded", + workspace=str(workspace), + report_path=str(report_path) if report_path.exists() else None, + returncode=rc, + nonfatal_warnings=warnings, + note="server_lifecycle_boot_only", + ) + ) + await _report_finished_variant(i) + continue + death_excerpt = server_log_death_excerpt(str(server_log)) + if rc != 0: + error = death_excerpt or redact_secret_values((stderr or stdout)[-2000:]) + # The bypass/scriptable path runs the customer body in a child whose stderr is redirected to + # benchmark_stderr.log, not the parent pipe — so `stderr`/`stdout` here are often empty and the real + # diagnostic (e.g. argparse "unrecognized arguments") is only on disk. + if not error.strip(): + error = redact_secret_values(_on_disk_stderr_tail(workspace, slot)) + # Last resort: report.errors when the pipe and on-disk logs are all empty. + if not error.strip(): + error = redact_secret_values(_report_errors_summary(report)) + invalid_class = "magpie_nonzero_invalid_measurement" + elif not report: + error = death_excerpt or "benchmark_report missing" + invalid_class = "benchmark_report_missing" + else: + error = death_excerpt or "benchmark_report missing valid throughput/completed requests" + invalid_class = "benchmark_report_invalid_metric" + log.warning( + "grid_runner: variant %d/%d name=%s aborted: %s (rc=%s): %s", + i + 1, + len(grid), + variant.name, + invalid_class, + rc, + error[:200], + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=invalid_class, + error_summary=error, + extra_args=variant.extra_server_args, + ) + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + workspace=str(workspace), + report_path=str(report_path) if report_path.exists() else None, + raw_result_path=measurement.get("raw_result_path"), + reported_success=measurement.get("reported_success"), + returncode=rc, + nonfatal_warnings=warnings, + error=error, + error_class=invalid_class, + server_log_path=_existing_log_path(server_log), + note=variant.note, + ) + ) + await _report_finished_variant(i) + if rc != 0 and not keep_going_on_failure: + break + continue + + if rc != 0: + nonzero_error = redact_secret_values((stderr or stdout)[-2000:]) + if served_complete_protocol(measurement): + # Every request the client recorded as requested was served, so + # the exit code came from something the round had already + # finished with. The cause stays on the result: a reader + # comparing this point to its neighbours is owed the reason it + # is not a clean zero. + warnings.append(f"nonzero_rc_after_complete_protocol:{rc}") + nonzero_kept_error = nonzero_error + log.warning( + "grid_runner: variant %s exited %d after serving its whole protocol " + "(%s of %s requests); keeping the measurement: %s", + variant.name, + rc, + measurement.get("completed_requests"), + measurement.get("requested_requests"), + nonzero_error, + ) + else: + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class="magpie_nonzero_after_valid_measurement", + error_summary=nonzero_error, + extra_args=variant.extra_server_args, + ) + log.warning( + "grid_runner: variant %s aborted: magpie_nonzero_after_valid_measurement " + "(rc=%d, served %s of %s requests)", + variant.name, + rc, + measurement.get("completed_requests"), + measurement.get("requested_requests"), + ) + raise _VariantAborted( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="failed", + workspace=str(workspace), + report_path=str(report_path) if report_path.exists() else None, + raw_result_path=measurement.get("raw_result_path"), + reported_success=measurement.get("reported_success"), + returncode=rc, + nonfatal_warnings=warnings, + error=nonzero_error, + error_class="magpie_nonzero_after_valid_measurement", + server_log_path=None, + note=variant.note, + ) + ) + + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="succeeded", + output_throughput=measurement.get("output_throughput"), + request_throughput=measurement.get("request_throughput"), + total_token_throughput=measurement.get("total_token_throughput"), + completed_requests=measurement.get("completed_requests"), + duration_seconds=measurement.get("duration_seconds"), + ttft_mean_ms=measurement.get("ttft_mean_ms"), + **constraint_fields(measurement), + tpot_mean_ms=measurement.get("tpot_mean_ms"), + input_throughput=measurement.get("input_throughput"), + tpot_p90_ms=measurement.get("tpot_p90_ms"), + intvty_p90=measurement.get("e2e_norm_intvty_p90"), + intvty_p50=measurement.get("e2e_norm_intvty_p50"), + request_error_rate=measurement.get("request_error_rate"), + workspace=str(workspace), + report_path=str(report_path) if report_path.exists() else None, + raw_result_path=measurement.get("raw_result_path"), + reported_success=measurement.get("reported_success"), + returncode=rc, + nonfatal_warnings=warnings, + error=nonzero_kept_error, + server_log_path=None, + note=variant.note, + runtime_sec=round( + max(0.0, time.time() - variant_started_unix), + 2, + ), + ) + ) + log.info( + "grid_runner: variant %s tput=%.1f tok/s", + variant.name, + results[-1].output_throughput or 0.0, + ) + await _report_finished_variant(i) + except _VariantAborted as abort: + results.append(abort.result) + await _report_finished_variant(i) + if not keep_going_on_failure: + break + await asyncio.to_thread( + _attach_grid_launch_evidence, + results, + grid=grid, + output_root=output_root, + caller_reused_ready_server=server_already_ready, + ready_server_log=ready_server_log, + ) + return results + + +def _teardown_variant_server(slot: Path, lifecycle: dict[str, Any] | None) -> list[int]: + """Stop every server this variant booted for its own rounds; return the pids still alive after SIGKILL. + + ``lifecycle`` names the persistent server the rounds shared, ``None`` when they shared none. + """ + from ._server_lifecycle import stop_agentx_servers, teardown_lifecycle_server + + if lifecycle is not None: + teardown_lifecycle_server( + pid_dir=slot, + framework=str(lifecycle.get("framework") or ""), + port=int(lifecycle.get("port") or 0), + ) + return stop_agentx_servers(slot) + + +def _not_run_skip_result(variant: GridVariant, stopped: StoppedByTheRun) -> VariantResult: + """Build the ``skipped`` result for a variant the run never got to.""" + return VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="skipped", + error=stopped.never_started, + error_class=stopped.error_class, + note=variant.note, + ) + + +def _existing_log_path(path: Path) -> str | None: + """Return ``path`` as a string when it exists, else ``None``.""" + return str(path) if path.exists() else None + + +def _measurement_server_log_path( + server_log: Path, + workspace: Path | None = None, + *, + slot: Path | None = None, +) -> str | None: + """Return the server log attributable to this variant's measurement.""" + owning_slot = slot or server_log.parent + direct = _existing_log_path(server_log) + if not direct and workspace is not None: + direct = _existing_log_path(workspace / "server.log") + if direct: + return direct + warmup_dir = owning_slot / "warmup_round" + try: + candidates = [path for path in warmup_dir.glob("*/server.log") if path.is_file()] + except OSError: + candidates = [] + if not candidates: + return None + try: + return str(max(candidates, key=lambda path: path.stat().st_mtime)) + except OSError: + return None + + +def _attach_grid_launch_evidence( + results: list[VariantResult], + *, + grid: list[GridVariant], + output_root: Path, + caller_reused_ready_server: bool, + ready_server_log: str | None = None, +) -> None: + """Persist declared and observed launch evidence for each grid result.""" + from ._workload_envs import VLLM_SOURCE_ROOT_ENVS + + for idx, result in enumerate(results): + if idx >= len(grid): + break + # A skipped variant never launched a measurement. + if result.status == "skipped" or result.error_class in { + "capability_unsupported", + "yaml_build_error", + "warmup_yaml_build_error", + }: + continue + slot = output_root / f"variant_{idx:02d}_{_safe(grid[idx].name)}" + config_path = slot / "config.yaml" + workspace = Path(result.workspace) if result.workspace else None + primary_log = Path(result.server_log_path) if result.server_log_path else slot / "server.log" + actual_log = _measurement_server_log_path(primary_log, workspace, slot=slot) + if not actual_log and ready_server_log: + actual_log = _existing_log_path(Path(ready_server_log)) + result.server_log_path = actual_log + evidence = build_launch_evidence( + config_path=config_path, + actual_server_log=actual_log, + framework=os.environ.get("FRAMEWORK", ""), + slot=slot, + caller_reused_ready_server=caller_reused_ready_server, + ) + result.launch_evidence = evidence + result.launch_evidence_path = persist_launch_evidence(evidence, slot=slot) + declared_env = evidence["requested_server_env"].keys() - set(VLLM_SOURCE_ROOT_ENVS) + report = scan_server_log(actual_log, evidence["framework"], declared_env=declared_env) + persist_runtime_findings(report, slot=slot) + + +def _safe(name: str) -> str: + """Filesystem-safe slug for variant directory names.""" + return "".join(c if c.isalnum() or c in "-_." else "_" for c in name)[:60] + + +def _write_variant_abort_marker( + slot: Path, + *, + variant_name: str, + error_class: str, + error_summary: str, + extra_args: str = "", +) -> None: + """Write ``abort_reason.json`` into the variant slot directory.""" + try: + slot.mkdir(parents=True, exist_ok=True) + marker = { + "variant": variant_name, + "error_class": error_class, + "error": (error_summary or "")[:2000], + "extra_args": extra_args, + "aborted_at_utc": time.strftime( + "%Y-%m-%dT%H:%M:%SZ", + time.gmtime(), + ), + } + (slot / "abort_reason.json").write_text( + json.dumps(marker, indent=2, sort_keys=True), + encoding="utf-8", + ) + except OSError as exc: + log.warning( + "_grid_runner: failed to write abort_reason.json at %s: %s", + slot, + exc, + ) + + +def _report_errors_summary(report: dict[str, Any] | None, limit: int = 2000) -> str: + """Join ``benchmark_report.json`` ``errors`` into a single diagnostic.""" + if not isinstance(report, dict): + return "" + errors = report.get("errors") + if not isinstance(errors, list): + return "" + text = "; ".join(str(item).strip() for item in errors if str(item).strip()) + return text[-limit:] if text else "" + + +def _on_disk_stderr_tail(*dirs: Path, limit: int = 2000) -> str: + """Return the tail of the first non-empty on-disk benchmark log.""" + for d in dirs: + if not d: + continue + for name in ("benchmark_stderr.log", "benchmark_stdout.log"): + try: + p = d / name + if not p.is_file(): + continue + text = p.read_text(encoding="utf-8", errors="replace").strip() + if text: + return f"[{name}] {text[-limit:]}" + except OSError: + continue + return "" + + +__all__ = [ + "DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC", + "GridVariant", + "ORCHESTRATOR_CANCELLED_CLASS", + "SESSION_TIME_EXHAUSTED_CLASS", + "StoppedByTheRun", + "SGLANG_WATCHDOG_TIMEOUT_ENV", + "VariantResult", + "apply_multi_node_invalid_variants", + "apply_aiter_moe_pin_filter", + "apply_runtime_benchmark_overrides", + "reorder_grid_for_multi_node", + "inject_sglang_attention_backend", + "inject_sglang_context_length", + "inject_sglang_watchdog_timeout", + "merge_server_args", + "remove_server_args", + "resolve_sglang_watchdog_timeout", + "run_grid", + "sanitize_result_dir", + "sanitize_script_name", + "server_args_env_name", + "session_grid_bounds", + "stopped_by_the_run", + # Re-exported from the sibling modules to keep the namespace intact. + "_resolve_magpie_python", + "_resolve_probe_python", + "coerce_extra_envs", + "compact_json_server_args", + "_SPACE_VALUE_FLAGS", + "_MULTI_VALUE_FLAGS", + "_VLLM_SINGLE_VALUE_FLAGS", + "dedup_vllm_server_args", + "_SGLANG_WATCHDOG_FLAG", + "_SGLANG_WATCHDOG_RE", + "DEFAULT_SGLANG_CONTEXT_HEADROOM_TOKENS", + "DEFAULT_SGLANG_CONTEXT_FLOOR_TOKENS", + "SGLANG_CONTEXT_HEADROOM_ENV", + "SGLANG_CONTEXT_FLOOR_ENV", + "_SGLANG_CONTEXT_LENGTH_FLAG", + "_SGLANG_CONTEXT_LENGTH_RE", + "_SGLANG_ATTN_BACKEND_FLAG", + "_SGLANG_ATTN_BACKEND_RE", + "_SGLANG_DUAL_CHUNK_BACKEND", + "_resolve_nonneg_int_env", + "resolve_sglang_context_cap", + "_resolve_dual_chunk_backend", + "_SGLANG_MOE_RUNNER_BACKEND_FLAG", + "_SGLANG_MOE_RUNNER_BACKEND_RE", + "moe_runner_requires_aiter", + "resolve_skip_spec", + "_parse_skip_spec", + "_RE_CUDA_GRAPH_MAX_BS", + "_MN_PARAMS_PRIORITY", + "_MN_BACKENDS_PRIORITY", + "_mn_priority_index", + "_COMPATIBILITY_FLAG_RULES", + "_XDIT_ENV_BLACKLIST", + "_XDIT_ENV_COMBO_BLACKLIST", + "xdit_blacklist_reason", + "_detect_model_class", + "apply_compatibility_filter", + "apply_user_skip_list", +] From 5459375f725042b90ba15b959e4726e68eafa659 Mon Sep 17 00:00:00 2001 From: luochen Date: Tue, 20 Oct 2026 11:06:58 +0800 Subject: [PATCH 4/4] Ground AITER_CONFIG_* env files where a specialist's proposals are admitted, not only in run_grid --- scripts/code_metrics_baseline.txt | 12 ++-- src/hyperloom/common/env_safety.py | 43 ++++++++++++ .../test_coordinator_async_batch2_unit.py | 19 ++++++ .../test_grid_runner_helpers_coverage_unit.py | 19 ++++-- .../test_specialist_patch_safety_unit.py | 44 ++++++++++++ .../actions/executors/_grid_runner.py | 56 ++-------------- .../orchestrator/specialists/dispatch.py | 24 ++++--- .../orchestrator/specialists/patch_safety.py | 67 +++++++++++++++++++ .../orchestrator/specialists/runner.py | 24 +------ 9 files changed, 215 insertions(+), 93 deletions(-) diff --git a/scripts/code_metrics_baseline.txt b/scripts/code_metrics_baseline.txt index 22cffac0ba..b6124dc410 100644 --- a/scripts/code_metrics_baseline.txt +++ b/scripts/code_metrics_baseline.txt @@ -171,7 +171,7 @@ cognitive-complexity src/hyperloom/orchestrator/specialists/dispatch.py::Special cognitive-complexity src/hyperloom/orchestrator/specialists/dispatch.py::SpecialistDispatchCollaborator.maybe_force_stalled_domain_specialist 39 cognitive-complexity src/hyperloom/orchestrator/specialists/dispatch.py::SpecialistDispatchCollaborator.warm_specialist_params 102 cognitive-complexity src/hyperloom/orchestrator/specialists/patch_safety.py::resolve_patch_apply_root 40 -cognitive-complexity src/hyperloom/orchestrator/specialists/runner.py::SpecialistRunner._finalize 49 +cognitive-complexity src/hyperloom/orchestrator/specialists/runner.py::SpecialistRunner._finalize 41 cognitive-complexity src/hyperloom/orchestrator/specialists/subprocess_.py::SpecialistSubprocessDispatcher._reap_loop 47 cognitive-complexity src/hyperloom/orchestrator/specialists/subprocess_.py::SpecialistSubprocessDispatcher.run 97 cognitive-complexity src/hyperloom/orchestrator/specialists/subprocess_.py::_codex_mcp_config 99 @@ -316,7 +316,7 @@ cyclomatic-complexity src/hyperloom/orchestrator/phases/prelude.py::PreludePhase cyclomatic-complexity src/hyperloom/orchestrator/phases/prelude.py::PreludePhase.maybe_enqueue_warm_replay 48 cyclomatic-complexity src/hyperloom/orchestrator/specialists/dispatch.py::SpecialistDispatchCollaborator.warm_specialist_params 41 cyclomatic-complexity src/hyperloom/orchestrator/specialists/patch_safety.py::resolve_patch_apply_root 21 -cyclomatic-complexity src/hyperloom/orchestrator/specialists/runner.py::SpecialistRunner._finalize 25 +cyclomatic-complexity src/hyperloom/orchestrator/specialists/runner.py::SpecialistRunner._finalize 22 cyclomatic-complexity src/hyperloom/orchestrator/specialists/subprocess_.py::SpecialistSubprocessDispatcher.run 39 cyclomatic-complexity src/hyperloom/orchestrator/specialists/subprocess_.py::_codex_mcp_config 28 cyclomatic-complexity src/hyperloom/orchestrator/trace_analysis/_bypass_trace_reader.py::_finalize 35 @@ -576,7 +576,7 @@ function-lines src/hyperloom/orchestrator/actions/executors/_geak_sweep.py::swee function-lines src/hyperloom/orchestrator/actions/executors/_gpu_pin.py::resolve_gpu_pin 105 function-lines src/hyperloom/orchestrator/actions/executors/_grid_runner.py::_build_variant_yaml 135 function-lines src/hyperloom/orchestrator/actions/executors/_grid_runner.py::_run_magpie 93 -function-lines src/hyperloom/orchestrator/actions/executors/_grid_runner.py::run_grid 1040 +function-lines src/hyperloom/orchestrator/actions/executors/_grid_runner.py::run_grid 1038 function-lines src/hyperloom/orchestrator/actions/executors/_kernel_agent_tool.py::materialize_unified_patch_snapshot 94 function-lines src/hyperloom/orchestrator/actions/executors/_kv_metrics.py::_read_aiperf_aggregate_json 101 function-lines src/hyperloom/orchestrator/actions/executors/_kv_metrics.py::sample_from_families 100 @@ -764,12 +764,12 @@ function-lines src/hyperloom/orchestrator/roles/critic_agent.py::CriticAgentBack function-lines src/hyperloom/orchestrator/specialists/dispatch.py::SpecialistDispatchCollaborator.build_specialist_round_entry 94 function-lines src/hyperloom/orchestrator/specialists/dispatch.py::SpecialistDispatchCollaborator.maybe_auto_retry_specialist 134 function-lines src/hyperloom/orchestrator/specialists/dispatch.py::SpecialistDispatchCollaborator.maybe_force_stalled_domain_specialist 114 -function-lines src/hyperloom/orchestrator/specialists/dispatch.py::SpecialistDispatchCollaborator.record_specialist_result 130 +function-lines src/hyperloom/orchestrator/specialists/dispatch.py::SpecialistDispatchCollaborator.record_specialist_result 128 function-lines src/hyperloom/orchestrator/specialists/dispatch.py::SpecialistDispatchCollaborator.warm_specialist_params 179 function-lines src/hyperloom/orchestrator/specialists/patch_safety.py::resolve_patch_apply_root 104 function-lines src/hyperloom/orchestrator/specialists/patch_safety.py::vet_patches 107 function-lines src/hyperloom/orchestrator/specialists/rebench.py::run_specialist_rebench 86 -function-lines src/hyperloom/orchestrator/specialists/runner.py::SpecialistRunner._finalize 183 +function-lines src/hyperloom/orchestrator/specialists/runner.py::SpecialistRunner._finalize 163 function-lines src/hyperloom/orchestrator/specialists/runner.py::SpecialistRunner._prepare 192 function-lines src/hyperloom/orchestrator/specialists/runner.py::SpecialistRunner._run_via_subprocess 173 function-lines src/hyperloom/orchestrator/specialists/subprocess_.py::SpecialistSubprocessDispatcher._build_codex_launch 97 @@ -953,7 +953,7 @@ module-lines src/hyperloom/orchestrator/phases/prelude.py 2640 module-lines src/hyperloom/orchestrator/policy/gate.py 1588 module-lines src/hyperloom/orchestrator/prompts/prompt_builder.py 1205 module-lines src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py 2571 -module-lines src/hyperloom/orchestrator/specialists/runner.py 1520 +module-lines src/hyperloom/orchestrator/specialists/runner.py 1505 module-lines src/hyperloom/orchestrator/specialists/subprocess_.py 1747 module-lines src/hyperloom/orchestrator/state/shared_state.py 2639 module-lines src/hyperloom/orchestrator/trace_analysis/_bypass_report.py 1231 diff --git a/src/hyperloom/common/env_safety.py b/src/hyperloom/common/env_safety.py index 2719406d68..ab615e85a6 100644 --- a/src/hyperloom/common/env_safety.py +++ b/src/hyperloom/common/env_safety.py @@ -12,6 +12,7 @@ import os import re from collections.abc import Mapping +from pathlib import Path log = logging.getLogger(__name__) @@ -422,6 +423,46 @@ def filter_untrusted_env_mapping( return allowed, dropped +# Env vars whose value names a config file the serving stack reads unconditionally: aiter's tuned-GEMM loader skips +# its merge default and pd.read_csv's exactly the AITER_CONFIG_* path when one is set, so a missing file kills the +# server at config load instead of falling back to the default table. Values that are not file paths (e.g. geak's +# JSON-payload AITER_CONFIG) do not carry the trailing underscore and stay untouched. +FILE_REFERENCING_ENV_PREFIXES: tuple[str, ...] = ("AITER_CONFIG_",) + + +def _is_file(path: str) -> bool: + # Path.is_file() raises for an unreadable parent or an overlong name (before Python 3.14); such a path is not a + # file the server can read either, and one proposal must not crash what checks it. + try: + return Path(path).is_file() + except OSError: + return False + + +def ungrounded_env_file_reason(envs: Mapping[str, object] | None) -> str | None: + """Return why ``envs`` cannot be served if one names a config file that does not exist, else ``None``. + + Grounds file-like env values the way patch targets are grounded: a proposal may point a config env at an + artifact only a later (possibly rejected) authoring step would create, and the server would die at config load. + The check is existence-only; anything already on disk (a promoted, durable artifact) passes. A set value is a + ``:``-joined list of paths (see ``csvs_aiter_will_load``), so every non-empty element must exist, not only the + first. + """ + for raw_key, raw_value in (envs or {}).items(): + key = str(raw_key or "").strip().upper() + value = str(raw_value or "").strip() + if not key.startswith(FILE_REFERENCING_ENV_PREFIXES) or not value: + continue + missing = [p for p in (part.strip() for part in value.split(":")) if p and not _is_file(p)] + if missing: + return ( + f"{key} points at {value}, whose element(s) {':'.join(missing)} do not exist; the server would read it " + "unconditionally and die at config load. Point it at a file a stage wrote (the gemm_tuning action " + "writes tuned-GEMM CSVs) or leave it unset to use the default config table." + ) + return None + + OPERATOR_EXTRA_ENV_VAR = "INFERENCE_OPTIMIZER_EXTRA_ENV" @@ -517,6 +558,7 @@ def redact_file_in_place(path: os.PathLike[str] | str, *, mode: int = 0o600) -> "BLOCKED_EXTERNAL_ENV_NAMES", "BLOCKED_UNTRUSTED_ENV_NAMES", "BLOCKED_VARIANT_ENV_NAMES", + "FILE_REFERENCING_ENV_PREFIXES", "GPU_MASK_ENV_NAMES", "OPERATOR_EXTRA_ENV_VAR", "SENSITIVE_ENV_NAME_MARKERS", @@ -534,5 +576,6 @@ def redact_file_in_place(path: os.PathLike[str] | str, *, mode: int = 0o600) -> "redact_secret_values", "scrub_benchmark_process_env", "scrub_child_process_env", + "ungrounded_env_file_reason", "valid_env_key", ] diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py index 8ea201bc18..8ed53cba92 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py @@ -2106,6 +2106,25 @@ async def test_record_specialist_result_logs_ungrounded_patches(coord: Coordinat assert "vllm/nope.py" in failures[0]["error_excerpt"] +@pytest.mark.asyncio +async def test_record_specialist_result_logs_proposals_whose_env_files_are_missing(coord: Coordinator) -> None: + """A proposal dropped at admission for an env file no stage wrote reaches the failure log the same way.""" + task = _ptask("rec-spec-env", "specialist") + await coord.specialist_dispatch.record_specialist_result( + task=task, + done_payload={ + "domain": "candidate_discovery_specialist", + "gap_canonical_id": "g1", + "proposal_set": [], + "proposals_env_ungrounded": ["proposal 'tuned-gemm': AITER_CONFIG_GEMM_BF16 points at /tmp/t.csv"], + }, + source="specialist:rec-spec-env", + ) + failures = [f for f in coord.shared_state.last_action_failures if f["task_id"] == "rec-spec-env"] + assert [f["error_class"] for f in failures] == ["proposal_env_file_missing"] + assert "AITER_CONFIG_GEMM_BF16" in failures[0]["error_excerpt"] + + @pytest.mark.asyncio async def test_record_specialist_result_no_dead_research_evidence_log( coord: Coordinator, diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py index e81006e5be..34d2db7dab 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py @@ -66,7 +66,7 @@ def test_ungrounded_env_file_reason_drops_missing_config_path(tmp_path) -> None: # A proposal may point a config env at an artifact no stage authored; the server would read it # unconditionally and die at config load, so the variant must be dropped with a reason. v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(tmp_path / "nope.csv")}) - reason = gr.ungrounded_env_file_reason(v) + reason = gr.ungrounded_env_file_reason(v.extra_envs) assert reason and "AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE" in reason assert str(tmp_path / "nope.csv") in reason @@ -75,7 +75,7 @@ def test_ungrounded_env_file_reason_passes_authored_file(tmp_path) -> None: csv = tmp_path / "tuned.csv" csv.write_text("gfx,M,N,K,splitK\ngfx950,64,5120,5120,2\n", encoding="utf-8") v = _variant("gemm", envs={"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": str(csv)}) - assert gr.ungrounded_env_file_reason(v) is None + assert gr.ungrounded_env_file_reason(v.extra_envs) is None def test_ungrounded_env_file_reason_accepts_colon_joined_existing_files(tmp_path) -> None: @@ -86,7 +86,7 @@ def test_ungrounded_env_file_reason_accepts_colon_joined_existing_files(tmp_path b = tmp_path / "t2.csv" b.write_text("gfx,M,N,K\ngfx950,4,5,6\n", encoding="utf-8") v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{b}"}) - assert gr.ungrounded_env_file_reason(v) is None + assert gr.ungrounded_env_file_reason(v.extra_envs) is None def test_ungrounded_env_file_reason_drops_colon_joined_missing_element(tmp_path) -> None: @@ -94,7 +94,7 @@ def test_ungrounded_env_file_reason_drops_colon_joined_missing_element(tmp_path) a.write_text("gfx,M,N,K\ngfx950,1,2,3\n", encoding="utf-8") missing = str(tmp_path / "never_authored.csv") v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": f"{a}:{missing}"}) - reason = gr.ungrounded_env_file_reason(v) + reason = gr.ungrounded_env_file_reason(v.extra_envs) assert reason and missing in reason and str(a) not in reason.split("element(s)")[1].split(" do not exist")[0] @@ -105,10 +105,17 @@ def raise_eacces(self): monkeypatch.setattr(gr.Path, "is_file", raise_eacces) v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": "/root/private/tuned.csv"}) - reason = gr.ungrounded_env_file_reason(v) + reason = gr.ungrounded_env_file_reason(v.extra_envs) assert reason and "/root/private/tuned.csv" in reason +def test_run_grid_fast_fail_drops_a_variant_whose_config_file_is_missing(tmp_path, monkeypatch) -> None: + # The backstop of the admission check: run_grid's fast-fail drops it before boot, whatever the framework. + monkeypatch.setenv("FRAMEWORK", "sglang") + v = _variant("gemm", envs={"AITER_CONFIG_GEMM_BF16": str(tmp_path / "never_written.csv")}) + assert "AITER_CONFIG_GEMM_BF16" in (gr.unsupported_capability_reason(v) or "") + + def test_ungrounded_env_file_reason_noop_without_config_envs(tmp_path) -> None: # Non-file envs (even under aiter-ish prefixes like the JSON-payload AITER_CONFIG) must stay untouched. v = _variant( @@ -119,7 +126,7 @@ def test_ungrounded_env_file_reason_noop_without_config_envs(tmp_path) -> None: "AITER_CONFIG_GEMM_A8W8": "", }, ) - assert gr.ungrounded_env_file_reason(v) is None + assert gr.ungrounded_env_file_reason(v.extra_envs) is None # -- unsupported_capability_reason (env-flag build probe) ----------------- diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_patch_safety_unit.py b/src/hyperloom/inference_optimizer/tests/test_specialist_patch_safety_unit.py index 37dc3b2b06..9ffc2ca871 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_patch_safety_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_patch_safety_unit.py @@ -199,6 +199,50 @@ def test_patch_safety_report_notes_empty(): assert ps.PatchSafetyReport().notes() == [] +# ---- proposals whose env files are not on disk ------------------------------- +def test_a_proposal_pointing_an_env_at_an_unwritten_file_is_dropped_at_admission(tmp_path): + """Qwen3-14B-FP8 session 20261014T024938Z: two-step proposals ("author the CSV, then point the env at it") whose + first step nothing ran; benched as config arms, their servers died at config load.""" + written = tmp_path / "tuned.csv" + written.write_text("gfx,M,N,K\n", encoding="utf-8") + missing = str(tmp_path / "aiter_configs" / "never_written.csv") + payload = { + "proposal_set": [ + {"name": "tuned-gemm", "extra_envs": {"AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE": missing}}, + {"name": "written", "extra_envs": {"AITER_CONFIG_GEMM_BF16": str(written)}}, + {"name": "flag-only", "extra_args": "--max-num-seqs 64", "extra_envs": {"VLLM_USE_V1": "1"}}, + ] + } + dropped = ps.drop_env_ungrounded_proposals(payload) + assert [p["name"] for p in payload["proposal_set"]] == ["written", "flag-only"] + assert len(dropped) == 1 and dropped[0].startswith("proposal 'tuned-gemm': ") + assert missing in dropped[0] and "gemm_tuning" in dropped[0] + assert ps.drop_env_ungrounded_proposals({"proposal_set": "not a list"}) == [] + + +def test_admit_payload_books_the_gate_and_reports_the_env_drops(tmp_path): + missing = str(tmp_path / "never_written.csv") + payload = { + "summary": "s", + "expected_gain_pct": 12, + "proposal_set": [{"name": "tuned-gemm", "extra_envs": {"AITER_CONFIG_GEMM_BF16": missing}}], + } + vetted = ( + ["a.patch"], + [{"path": "b.patch", "verdict": ps.GROUND_MISSING_TARGET, "detail": "missing_target: b"}], + {"a.patch": ps.GROUND_APPLIES}, + False, + ) + report = ps.admit_payload(payload, vetted, {"a.patch": "/src/vllm", "b.patch": "/src/vllm"}) + assert payload["proposal_set"] == [] and "expected_gain_pct" not in payload + assert payload["patches_written"] == ["a.patch"] and payload["patch_roots"] == {"a.patch": "/src/vllm"} + assert payload["patches_ungrounded"] == ["missing_target: b"] + assert len(payload["proposals_env_ungrounded"]) == 1 and "patches_span_multiple_roots" not in payload + assert any("expected_gain_pct" in f for f in report.forbidden_fields) + assert report.env_ungrounded == payload["proposals_env_ungrounded"] + assert any(n.startswith("proposal_env_file_missing:") for n in report.notes()) + + # ---- scan_numeric_claims --------------------------------------------------- def test_scan_numeric_claims(): payload = { diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 6039ed0109..ee7dc41575 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -27,6 +27,7 @@ is_python_package_root, redact_secret_values, scrub_benchmark_process_env, + ungrounded_env_file_reason, ) from ...phases import machine_state as _phase_state @@ -236,59 +237,18 @@ def _probe_vllm_aiter_shared_expert_unsupported() -> str | None: def unsupported_capability_reason(variant: "GridVariant") -> str | None: - """Return a drop reason if a variant sets an env flag the installed framework build cannot honour, else ``None``.""" - fw = (os.environ.get("FRAMEWORK", "") or "sglang").strip().lower() - if fw != "vllm": - return None + """Return why a variant cannot boot (a config file not on disk, an env flag the build lacks), else ``None``.""" envs = {str(k): str(v) for k, v in (getattr(variant, "extra_envs", None) or {}).items()} + if missing_file := ungrounded_env_file_reason(envs): + return missing_file + if (os.environ.get("FRAMEWORK", "") or "sglang").strip().lower() != "vllm": + return None val = envs.get("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS") if val is None or not is_truthy(val, default=True): return None return _probe_vllm_aiter_shared_expert_unsupported() -# Env vars whose value names a config file the serving stack reads unconditionally: aiter's tuned-GEMM loader skips -# its merge default and pd.read_csv's exactly the AITER_CONFIG_* path when one is set, so a missing file kills the -# server at config load instead of falling back to the default table. Values that are not file paths (e.g. geak's -# JSON-payload AITER_CONFIG) do not carry the trailing underscore and stay untouched. -_FILE_REFERENCING_ENV_PREFIXES: tuple[str, ...] = ("AITER_CONFIG_",) - - -def _is_file(path: str) -> bool: - # Path.is_file() raises for an unreadable parent or an overlong name (before Python 3.14); such a path is not a - # file the server can read either, and one variant's proposal must not crash the whole grid. - try: - return Path(path).is_file() - except OSError: - return False - - -def ungrounded_env_file_reason(variant: "GridVariant") -> str | None: - """Return a drop reason if an extra_envs value names a config file that does not exist, else ``None``. - - Grounds file-like env values the same way ``vet_patches`` grounds patch targets: a proposal may point a config - env at an artifact only a later (possibly rejected) authoring step would create, and the benchmark would boot a - server that dies at config load. The check is existence-only; anything already on disk (a promoted, durable - artifact) passes. A set value is a ``:``-joined list of paths (see ``csvs_aiter_will_load``), so every - non-empty element must exist, not only the first. - """ - for raw_key, raw_value in (getattr(variant, "extra_envs", None) or {}).items(): - key = str(raw_key or "").strip().upper() - if not key.startswith(_FILE_REFERENCING_ENV_PREFIXES): - continue - value = str(raw_value or "").strip() - if not value: - continue - missing = [p for p in (part.strip() for part in value.split(":")) if p and not _is_file(p)] - if missing: - return ( - f"{key} points at {value}, whose element(s) {':'.join(missing)} do not exist; the server would read it " - "unconditionally and die at config load. Point it at a file a stage authored " - "(e.g. a promoted tuned-GEMM CSV) or leave it unset to use the default config table." - ) - return None - - # Sanitization for LLM-supplied overrides (benchmark_script / result_dir): reject path separators / shell # metacharacters, raising ``ValueError`` instead of running an unsafe subprocess. _SCRIPT_NAME_RE = re.compile(r"^[A-Za-z0-9_.\-]+\.sh$") @@ -986,9 +946,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant await _unit_started(i, "variant") slot = output_root / f"variant_{i:02d}_{_safe(variant.name)}" server_log = slot / "server.log" - # Capability fast-fail: drop a variant whose env flag the build cannot honour before booting a doomed server, - # still recording the failure so the LLM learns not to re-pick it. - cap_reason = unsupported_capability_reason(variant) or ungrounded_env_file_reason(variant) + cap_reason = unsupported_capability_reason(variant) if cap_reason: log.warning( "grid_runner: variant %d/%d name=%s aborted: capability_unsupported: %s", diff --git a/src/hyperloom/orchestrator/specialists/dispatch.py b/src/hyperloom/orchestrator/specialists/dispatch.py index 4d3c6d8032..2c3f14fe82 100644 --- a/src/hyperloom/orchestrator/specialists/dispatch.py +++ b/src/hyperloom/orchestrator/specialists/dispatch.py @@ -35,6 +35,12 @@ __all__ = ["SpecialistDispatchCollaborator"] _SOURCE_PATCH_FAMILY = "source_patch" +#: ``(specialist_done key, failure class)`` for what the patch-safety gate dropped at admission: each reaches the +#: durable failure log, which every SEED turn renders. +_ADMISSION_DROPS: tuple[tuple[str, str], ...] = ( + ("patches_ungrounded", "patch_targets_ungrounded"), + ("proposals_env_ungrounded", "proposal_env_file_missing"), +) #: ``(SharedState field, param key, conversion)`` for the values a specialist is warmed with. The converted value #: decides: an empty string, an empty container and a zero are not exported, because an absent key is how the @@ -880,16 +886,14 @@ async def record_specialist_result( ) # Specialist notes reach the prompt only through this task's one inbox # line; ``last_action_failures`` is rendered every SEED turn. - ungrounded = done_payload.get("patches_ungrounded") - if isinstance(ungrounded, list) and ungrounded: - self.shared_state.record_action_failure( - action="specialist", - task_id=task.task_id, - result={ - "error_class": "patch_targets_ungrounded", - "error": "; ".join(str(d) for d in ungrounded[:4]), - }, - ) + for dropped_key, error_class in _ADMISSION_DROPS: + dropped = done_payload.get(dropped_key) + if isinstance(dropped, list) and dropped: + self.shared_state.record_action_failure( + action="specialist", + task_id=task.task_id, + result={"error_class": error_class, "error": "; ".join(str(d) for d in dropped[:4])}, + ) # Advisory multi-model scoring of the proposal_set; informational only, gates nothing. if self._proposal_scorer is not None and proposals: scores = await self._proposal_scorer.score( diff --git a/src/hyperloom/orchestrator/specialists/patch_safety.py b/src/hyperloom/orchestrator/specialists/patch_safety.py index dbc736dfa6..fee7c5540a 100644 --- a/src/hyperloom/orchestrator/specialists/patch_safety.py +++ b/src/hyperloom/orchestrator/specialists/patch_safety.py @@ -31,6 +31,7 @@ from pathlib import Path, PurePosixPath from typing import Any +from hyperloom.common.env_safety import ungrounded_env_file_reason from hyperloom.common.unified_diff import strip_diff_path, strip_path_components @@ -753,6 +754,7 @@ class PatchSafetyReport: grounding: dict[str, str] = field(default_factory=dict) numeric_warnings: list[str] = field(default_factory=list) forbidden_fields: list[str] = field(default_factory=list) + env_ungrounded: list[str] = field(default_factory=list) def notes(self) -> list[str]: """Render audit notes for SpecialistRunResult / session_breakdown. @@ -790,6 +792,8 @@ def notes(self) -> list[str]: out.append("patch_safety_numeric:" + ",".join(self.numeric_warnings[:8])) if self.forbidden_fields: out.append("patch_safety_forbidden_fields:" + ",".join(self.forbidden_fields[:8])) + if self.env_ungrounded: + out.append("proposal_env_file_missing:" + "; ".join(self.env_ungrounded[:4])) return out @@ -1068,6 +1072,67 @@ def vet_patches( return kept, ungrounded, grounding, len(resolved_roots) > 1 +def drop_env_ungrounded_proposals(payload: dict[str, Any]) -> list[str]: + """Remove the ``proposal_set`` entries whose ``extra_envs`` name a config file that is not on disk; why, per entry. + + The patch gate grounds what a proposal changes; this grounds the files its envs make the server read. A proposal + premised on an artifact some later step would write ("author the CSV, then point the env at it") is refused here, + where the orchestrator hears why, rather than benched as a config arm whose server dies at config load. + ``run_grid`` keeps the same check as the backstop. + """ + proposals = payload.get("proposal_set") + if not isinstance(proposals, list): + return [] + kept: list[Any] = [] + dropped: list[str] = [] + for proposal in proposals: + reason = ungrounded_env_file_reason(proposal.get("extra_envs")) if isinstance(proposal, dict) else None + if reason is None: + kept.append(proposal) + continue + dropped.append(f"proposal {str(proposal.get('name') or '').strip() or ''!r}: {reason}") + if dropped: + payload["proposal_set"] = kept + return dropped + + +def admit_payload( + payload: dict[str, Any], + vetted: tuple[list[str], list[dict[str, str]], dict[str, str], bool], + patch_roots: dict[str, str], +) -> PatchSafetyReport: + """Apply the patch-safety gate to a specialist_done ``payload`` in place; the report of what it found. + + It books what :func:`vet_patches` kept, scans the numeric claims, strips the forbidden fields and drops the + proposals whose env files are missing. + """ + kept, ungrounded, grounding, spans_roots = vetted + numeric_warnings = scan_numeric_claims(payload) + # Strip, do not forward: the Critic is instructed to reject the whole + # proposal_set over these fields, which costs the round every idea the + # specialist produced. The audit note records what the strip took. + forbidden_fields = strip_forbidden_proposal_fields(payload) + env_ungrounded = drop_env_ungrounded_proposals(payload) + payload["patches_written"] = kept + payload["patch_grounding"] = grounding + if patch_roots: + payload["patch_roots"] = {p: r for p, r in patch_roots.items() if p in kept} + if ungrounded: + payload["patches_ungrounded"] = [d["detail"] for d in ungrounded[:8]] + if env_ungrounded: + payload["proposals_env_ungrounded"] = env_ungrounded[:8] + if spans_roots: + payload["patches_span_multiple_roots"] = True + return PatchSafetyReport( + kept_patches=kept, + ungrounded=ungrounded, + grounding=grounding, + numeric_warnings=numeric_warnings, + forbidden_fields=forbidden_fields, + env_ungrounded=env_ungrounded, + ) + + __all__ = [ "ADVISE_VERDICT", "ADVISORY_RULE_PROPOSAL_KINDS", @@ -1089,9 +1154,11 @@ def vet_patches( "QUANTITATIVE_CLAIM_REASON_CODE", "SCOPE_DOMAINS_LITERAL", "SPECIALIST_WORK_ARTIFACT_PATHSPECS", + "admit_payload", "advisory_only_reason_codes", "advisory_rules_govern", "cross_domain_rule_descriptors", + "drop_env_ungrounded_proposals", "ground_patch_text", "is_unified_diff", "numeric_claims", diff --git a/src/hyperloom/orchestrator/specialists/runner.py b/src/hyperloom/orchestrator/specialists/runner.py index 2ced6bab94..877dd62ee5 100644 --- a/src/hyperloom/orchestrator/specialists/runner.py +++ b/src/hyperloom/orchestrator/specialists/runner.py @@ -1211,33 +1211,13 @@ def _finalize( patches=deduped, patch_roots=collected_roots, ) - kept, ungrounded, grounding, spans_roots = _patch_safety.vet_patches( + vetted = _patch_safety.vet_patches( deduped, base_checkout=base_checkout, candidate_roots=candidate_roots, explicit_root=explicit_root, ) - numeric_warnings = _patch_safety.scan_numeric_claims(done_payload) - # Strip, do not forward: the Critic is instructed to reject the whole - # proposal_set over these fields, which costs the round every idea the - # specialist produced. The audit note below records what the strip took. - forbidden_fields = _patch_safety.strip_forbidden_proposal_fields(done_payload) - safety = _patch_safety.PatchSafetyReport( - kept_patches=kept, - ungrounded=ungrounded, - grounding=grounding, - numeric_warnings=numeric_warnings, - forbidden_fields=forbidden_fields, - ) - done_payload["patches_written"] = kept - done_payload["patch_grounding"] = grounding - if collected_roots: - done_payload["patch_roots"] = {p: r for p, r in collected_roots.items() if p in kept} - if ungrounded: - done_payload["patches_ungrounded"] = [d["detail"] for d in ungrounded[:8]] - if spans_roots: - done_payload["patches_span_multiple_roots"] = True - notes.extend(safety.notes()) + notes.extend(_patch_safety.admit_payload(done_payload, vetted, collected_roots).notes()) recovered = bool(done_payload.get("_recovered_from_partial")) # ``partial`` keeps an infra failure visible without making the attempt # retry-eligible, which would discard whatever was salvaged.