diff --git a/cvs/lib/conf_lib/AGENTS.md b/cvs/lib/conf_lib/AGENTS.md index 83a4eed4c..feccc3597 100644 --- a/cvs/lib/conf_lib/AGENTS.md +++ b/cvs/lib/conf_lib/AGENTS.md @@ -1,6 +1,8 @@ # cvs/lib/conf_lib — config blocks and `load_config` -Suite schemas are built from `config_blocks.py` and loaded with `loader.load_config`. +Suite schemas are built from `config_blocks.py`, live in `schemas/` (one module per suite, such +as `schemas/vllm.py`), and are loaded with `loader.load_config`. A suite with a schema here calls +`load_config` directly; it has no loader module of its own. Substitution (`substitute_config`) and `BaseVariantConfig` stay in `cvs/lib/utils/config_loader.py`, which older suite loaders import. diff --git a/cvs/lib/conf_lib/schemas/__init__.py b/cvs/lib/conf_lib/schemas/__init__.py new file mode 100644 index 000000000..a73a51e90 --- /dev/null +++ b/cvs/lib/conf_lib/schemas/__init__.py @@ -0,0 +1,4 @@ +''' +Copyright 2026 Advanced Micro Devices, Inc. +All rights reserved. +''' diff --git a/cvs/lib/conf_lib/schemas/unittests/__init__.py b/cvs/lib/conf_lib/schemas/unittests/__init__.py new file mode 100644 index 000000000..a73a51e90 --- /dev/null +++ b/cvs/lib/conf_lib/schemas/unittests/__init__.py @@ -0,0 +1,4 @@ +''' +Copyright 2026 Advanced Micro Devices, Inc. +All rights reserved. +''' diff --git a/cvs/lib/conf_lib/schemas/unittests/test_vllm.py b/cvs/lib/conf_lib/schemas/unittests/test_vllm.py new file mode 100644 index 000000000..002e45e9b --- /dev/null +++ b/cvs/lib/conf_lib/schemas/unittests/test_vllm.py @@ -0,0 +1,366 @@ +'''Unit tests for the vLLM config schema in cvs.lib.conf_lib.schemas.vllm.''' + +import json +import tempfile +import unittest +from pathlib import Path + +from pydantic import ValidationError + +from cvs.lib.conf_lib.loader import ConfigError, load_config +from cvs.lib.conf_lib.schemas.vllm import ( + ContainerEnvTokenError, + RunCell, + VariantConfig, + serialize_cli_options, +) +from cvs.unittests.support import FAKE_CRED, write_filled_config + + +CELL = "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16" +FAKE_CRED_FILE = "/home/test/.hf_token" +CONFIG_DIR = Path(__file__).resolve().parents[4] / "input" / "config_file" / "inference" / "vllm" +DISTRIBUTED_TEMPLATE = CONFIG_DIR / "mi3xx_vllm_llama33-70b_fp8_distributed.json" +MERGE_CELL = "ISL=1024,OSL=1024,TP=8,PP=1,CONC=16" + + +def _config(**overrides): + value = { + "enforce_thresholds": False, + "threshold_json": "threshold.json", + "ib_hca_devices": "auto", + "ib_netdev": "eth0", + "paths": { + "shared_fs": "/home/test", + "models_dir": "/models", + "log_dir": "/logs", + "hf_token_file": FAKE_CRED_FILE, + }, + "container": {"name": "vllm", "image": "image", "runtime": {"name": "docker", "args": {}}}, + "server_params": { + "model": "/models/DeepSeek-R1-0528-FP8", + "tensor_parallel_size": 8, + "pipeline_parallel_size": 2, + }, + "benchmark_params": {"num_prompts": 25}, + "sweeps": {CELL: {"num_prompts": 50}}, + "runs": [CELL], + "thresholds": {}, + } + value.update(overrides) + return value + + +def _with_args(args): + config = _config() + config["container"]["runtime"]["args"] = args + return config + + +class TestRunCell(unittest.TestCase): + def test_parses_only_canonical_cell_keys(self): + cell = RunCell.parse(CELL) + self.assertEqual((cell.isl, cell.osl, cell.tp, cell.pp, cell.concurrency), (1024, 1024, 8, 2, 16)) + for invalid in ( + "ISL=1024,OSL=1024,TP=8,CONC=16", + "OSL=1024,ISL=1024,TP=8,PP=2,CONC=16", + "ISL=1024, OSL=1024,TP=8,PP=2,CONC=16", + "ISL=0,OSL=1024,TP=8,PP=2,CONC=16", + ): + with self.subTest(invalid=invalid): + with self.assertRaises(ValueError): + RunCell.parse(invalid) + + +class TestRuns(unittest.TestCase): + def test_resolves_selected_run_with_cell_overrides(self): + variant = VariantConfig(**_config()) + (run,) = variant.resolved_runs() + self.assertEqual(run.cell.key, CELL) + self.assertEqual(run.benchmark_params["num_prompts"], 50) + self.assertEqual(run.benchmark_params["random_input_len"], 1024) + self.assertEqual(run.benchmark_params["random_output_len"], 1024) + self.assertEqual(run.benchmark_params["max_concurrency"], 16) + + def test_rejects_unknown_root_and_legacy_shape(self): + for kwargs in ({"unexpected": 1}, {"schema_version": 1}, {"params": {}}): + with self.subTest(kwargs=kwargs): + with self.assertRaises(ValidationError): + VariantConfig(**_config(**kwargs)) + + def test_requires_nonempty_explicit_runs(self): + with self.assertRaisesRegex(ValidationError, "nonempty explicit"): + VariantConfig(**_config(runs=[])) + + def test_rejects_mismatched_parallelism_and_missing_enforced_threshold(self): + wrong_cell = "ISL=1024,OSL=1024,TP=4,PP=2,CONC=16" + with self.assertRaisesRegex(ValidationError, "conflicts"): + VariantConfig(**_config(sweeps={wrong_cell: {}}, runs=[wrong_cell])) + with self.assertRaisesRegex(ValidationError, "missing threshold"): + VariantConfig(**_config(enforce_thresholds=True)) + + +class TestContainer(unittest.TestCase): + def test_accepts_complete_container_network_environment(self): + config = _config() + config["container"]["env"] = { + "NCCL_IB_HCA": "rdma0", + "NCCL_SOCKET_IFNAME": "eno0", + "GLOO_SOCKET_IFNAME": "eno0", + "TP_SOCKET_IFNAME": "eno0", + } + variant = VariantConfig(**config) + self.assertEqual(variant.container.nccl_ib_hcas, ["rdma0"]) + self.assertTrue(variant.container.socket_env_configured) + + def test_rejects_incomplete_container_socket_environment(self): + names = ("NCCL_SOCKET_IFNAME", "GLOO_SOCKET_IFNAME", "TP_SOCKET_IFNAME") + for missing in names: + with self.subTest(missing=missing): + config = _config() + config["container"]["env"] = {name: "eno0" for name in names if name != missing} + with self.assertRaisesRegex(ValidationError, "must set all socket interface"): + VariantConfig(**config) + + def test_rejects_empty_container_hca(self): + config = _config() + config["container"]["env"] = {"NCCL_IB_HCA": " , "} + with self.assertRaisesRegex(ValidationError, "must name at least one device"): + VariantConfig(**config) + + def test_rejects_hf_token_in_container_env_without_echoing_value(self): + for name in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "hf_token", "Hf_Token"): + for extra in ({}, {"NCCL_SOCKET_IFNAME": "eno0"}): + with self.subTest(name=name, extra=extra): + config = _config() + config["container"]["env"] = {**extra, name: FAKE_CRED} + with self.assertRaises(ContainerEnvTokenError) as ctx: + VariantConfig(**config) + self.assertIn(f"container.env must not set ['{name}']", str(ctx.exception)) + self.assertIn("paths.hf_token_file", str(ctx.exception)) + self.assertNotIn(FAKE_CRED, repr(ctx.exception.args)) + + def test_runtime_args_accept_docker_keys_but_not_env(self): + variant = VariantConfig(**_with_args({"shm_size": "64g", "cap_add": ["SYS_PTRACE"], "ulimit": ["memlock=-1"]})) + self.assertEqual(variant.container.model_dump()["runtime"]["args"]["shm_size"], "64g") + for args in ({"env": {"NCCL_DEBUG": "INFO"}}, {"unknown_flag": True}): + with self.subTest(args=args): + with self.assertRaises(ValidationError): + VariantConfig(**_with_args(args)) + + +class TestCommentKeysAndOptions(unittest.TestCase): + def test_comment_keys_are_ignored_but_option_payloads_keep_theirs(self): + config = _config( + _comment="root", + server_params={ + "_comment": "server", + "model": "/models/x", + "tensor_parallel_size": 8, + "pipeline_parallel_size": 2, + "json_arg": {"_comment": "upstream payload"}, + }, + sweeps={"_comment": "cells", CELL: {"_comment": "cell", "json_arg": {"_comment": "upstream payload"}}}, + accuracy={"_comment": "accuracy", "tasks": [{"_comment": "task", "id": "gsm8k", "task": "gsm8k"}]}, + ) + config["container"]["runtime"]["args"] = {"_comment_mounts": "notes", "volumes": ["/data:/data"]} + variant = VariantConfig(**config) + self.assertEqual(variant.server_params.extra_options(), {"json_arg": {"_comment": "upstream payload"}}) + self.assertEqual(variant.sweeps, {CELL: {"json_arg": {"_comment": "upstream payload"}}}) + self.assertEqual(variant.accuracy.tasks[0].id, "gsm8k") + + def test_serializes_documented_value_shapes(self): + options = { + "trust_remote_code": True, + "max_model_len": 8192, + "served_model_name": ["first", "second"], + "structured_outputs": {"json": {"type": "object"}}, + "optional": None, + } + self.assertEqual( + serialize_cli_options(options), + [ + "--trust-remote-code", + "--max-model-len", + "8192", + "--served-model-name", + "first", + "second", + "--structured-outputs", + '{"json":{"type":"object"}}', + ], + ) + + def test_rejects_ambiguous_false_and_reserved_names(self): + with self.assertRaises(ValueError): + serialize_cli_options({"enforce_eager": False}) + with self.assertRaisesRegex(ValidationError, "cannot override harness"): + VariantConfig( + **_config( + server_params={ + "model": "/models/x", + "tensor_parallel_size": 8, + "pipeline_parallel_size": 2, + "nnodes": 2, + } + ) + ) + + with self.assertRaisesRegex(ValidationError, "benchmark_params cannot override harness"): + VariantConfig(**_config(benchmark_params={"num_prompts": 25, "percentile_metrics": "ttft"})) + + +class TestThresholds(unittest.TestCase): + def _variant(self, thresholds, enforce=False): + return VariantConfig(**_config(enforce_thresholds=enforce, thresholds=thresholds)) + + def test_partial_and_empty_cells_load_with_enforcement_off_and_on(self): + for thresholds in ({CELL: {"output_throughput": {"kind": "min", "value": 0}}}, {CELL: {}}): + for enforce in (False, True): + with self.subTest(thresholds=thresholds, enforce=enforce): + self.assertEqual(self._variant(thresholds, enforce=enforce).thresholds, thresholds) + + def test_enforcement_still_requires_selected_run_cell(self): + self._variant({}, enforce=False) + with self.assertRaisesRegex(ValidationError, "missing threshold coverage"): + self._variant({}, enforce=True) + + def test_comment_metrics_are_ignored_and_other_underscore_keys_rejected(self): + spec = {"kind": "min", "value": 0} + variant = self._variant({CELL: {"_comment": "calibrate", "_example_floor": spec, "output_throughput": spec}}) + self.assertEqual(variant.thresholds, {CELL: {"output_throughput": spec}}) + for cell in ( + {"notes": spec}, + {"_output_throughput": spec}, + {"_": spec}, + {"output_throughput": {**spec, "_note": "calibrated"}}, + ): + with self.subTest(cell=cell): + with self.assertRaisesRegex(ValidationError, "unknown (vLLM threshold metric|threshold fields)"): + self._variant({CELL: cell}) + + def test_rejects_specs_outside_the_vllm_direction_rule(self): + for metric, spec in ( + ("output_throughput", {"kind": "max", "value": 1}), + ("mean_ttft_ms", {"kind": "min", "value": 1}), + ("output_throughput", {"kind": "min_tok_s", "value": 1}), + ("output_throughput", {"kind": "within", "value": 1, "tolerance_pct": 5}), + ("output_throughput", {"kind": "min", "value": 1, "optional": False}), + ("output_throughput", {"kind": "min", "value": 1, "unit": "tok/s"}), + ("output_throughput", {"kind": "min", "value": "1"}), + ("client.output_throughput", {"kind": "min", "value": 1}), + ): + with self.subTest(metric=metric, spec=spec): + with self.assertRaises(ValidationError): + self._variant({CELL: {metric: spec}}) + + def test_accuracy_thresholds_are_not_metric_cells(self): + accuracy = {"gsm8k": {"gsm8k.exact_match__strict-match": {"kind": "min", "value": 0, "tolerance_pct": 5}}} + cell = {"output_throughput": {"kind": "min", "value": 0}} + variant = self._variant({CELL: cell, "accuracy": accuracy}, enforce=True) + self.assertEqual(variant.thresholds["accuracy"], accuracy) + with self.assertRaisesRegex(ValidationError, "threshold cells"): + self._variant({CELL: cell, "accuracy": accuracy, "acuracy": {}}, enforce=True) + + +def _templates(): + return sorted(path for path in CONFIG_DIR.glob("*.json") if not path.name.endswith("threshold.json")) + + +class TestShippedTemplates(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.tmp = Path(self._tmp.name) + + def test_every_template_loads_once_its_placeholders_are_filled(self): + templates = _templates() + self.assertTrue(templates) + for template in templates: + with self.subTest(config=template.name): + load_config(VariantConfig, write_filled_config(template, self.tmp), {"username": "test"}) + + def test_unfilled_distributed_template_names_the_placeholders_it_checks(self): + with self.assertRaises(ConfigError) as ctx: + load_config(VariantConfig, DISTRIBUTED_TEMPLATE, {"username": "test"}) + self.assertEqual( + sorted(problem.split(":", 1)[0] for problem in ctx.exception.problems), + [ + "container.env.GLOO_SOCKET_IFNAME", + "container.env.NCCL_IB_GID_INDEX", + "container.env.NCCL_SOCKET_IFNAME", + "container.env.TP_SOCKET_IFNAME", + "container.image", + "container.runtime.args.volumes.1", + "container.runtime.args.volumes.3", + "server_params.model", + ], + ) + + def test_placeholder_in_an_option_value_fails_the_load(self): + filled = write_filled_config(DISTRIBUTED_TEMPLATE, self.tmp) + config = json.loads(filled.read_text()) + config["server_params"]["served_model_name"] = "" + (cell,) = config["runs"] + config["sweeps"][cell]["tokenizer"] = "" + filled.write_text(json.dumps(config)) + with self.assertRaises(ConfigError) as ctx: + load_config(VariantConfig, filled, {"username": "test"}) + self.assertEqual( + sorted(problem.split(":", 1)[0] for problem in ctx.exception.problems), + ["server_params.served_model_name", f"sweeps.{cell}.tokenizer"], + ) + + +class TestContainerForOrchestrator(unittest.TestCase): + def test_cluster_env_volumes_and_shm_size_merge_without_aliasing(self): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + (tmp / "threshold.json").write_text("{}") + config = { + "enforce_thresholds": False, + "threshold_json": str(tmp / "threshold.json"), + "paths": {"shared_fs": "/home/x", "models_dir": "/models", "log_dir": "/logs", "hf_token_file": "/t"}, + "container": { + "name": "vllm", + "image": "rocm/vllm:1", + "env": {"B": "config"}, + "runtime": {"name": "docker", "args": {"volumes": ["/config:/config"]}}, + }, + "server_params": {"model": "/models/m", "tensor_parallel_size": 8}, + "sweeps": {MERGE_CELL: {}}, + "runs": [MERGE_CELL], + } + (tmp / "config.json").write_text(json.dumps(config)) + cluster = { + "container": { + "name": "from-cluster", + "env": {"A": "cluster", "B": "cluster"}, + "runtime": { + "name": "docker", + "args": { + "volumes": ["/cluster:/cluster"], + "shm_size": "64g", + "registry": {"username": "user", "password_file": "/home/x/registry"}, + }, + }, + } + } + loaded = load_config(VariantConfig, tmp / "config.json", cluster) + self.assertEqual(loaded.container["name"], "vllm") + self.assertEqual(loaded.container["env"], {"A": "cluster", "B": "config"}) + self.assertEqual(loaded.container["runtime"]["args"]["volumes"], ["/config:/config"]) + self.assertEqual(loaded.container["runtime"]["args"]["shm_size"], "64g") + loaded.container["env"]["A"] = "mutated" + loaded.container["runtime"]["args"]["volumes"].append("/extra") + loaded.container["runtime"]["args"]["shm_size"] = "1g" + loaded.container["runtime"]["args"]["registry"]["username"] = "mutated" + cluster_container = loaded.cluster["container"] + self.assertEqual(cluster_container["env"]["A"], "cluster") + self.assertEqual(cluster_container["runtime"]["args"]["volumes"], ["/cluster:/cluster"]) + self.assertEqual(cluster_container["runtime"]["args"]["shm_size"], "64g") + self.assertEqual(cluster_container["runtime"]["args"]["registry"]["username"], "user") + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/utils/vllm_config_loader.py b/cvs/lib/conf_lib/schemas/vllm.py similarity index 72% rename from cvs/lib/inference/utils/vllm_config_loader.py rename to cvs/lib/conf_lib/schemas/vllm.py index 07278663d..e65098bfc 100644 --- a/cvs/lib/inference/utils/vllm_config_loader.py +++ b/cvs/lib/conf_lib/schemas/vllm.py @@ -5,70 +5,115 @@ import json import re from dataclasses import dataclass -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Annotated, Any, Dict, List, Literal, Optional, Union -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, create_model, model_validator +from pydantic import AfterValidator, ConfigDict, Field, PrivateAttr, field_validator, model_validator from cvs.lib.inference.utils.accuracy_config import AccuracyConfig -from cvs.lib.inference.utils.vllm_metrics import METRIC_REGISTRY, validate_threshold_spec -from cvs.lib.utils.config_loader import substitute_config +from cvs.lib.inference.utils.vllm_metrics import validate_threshold_spec +from cvs.lib.conf_lib.config_blocks import ( + PLACEHOLDER_MESSAGE, + ConfigStr, + ContainerSpec, + Paths, + RuntimeArgs, + RuntimeSpec, + ThresholdFile, + _Allow, + _Forbid, + has_placeholder, + placeholders_rejected, +) -GATED_GPU_METRICS = {definition.name for definition in METRIC_REGISTRY if definition.datasource == "gpu"} -GATED_PROM_METRICS = {definition.name for definition in METRIC_REGISTRY if definition.datasource == "prometheus"} _CELL_RE = re.compile( r"^ISL=(?P[1-9]\d*),OSL=(?P[1-9]\d*),TP=(?P[1-9]\d*),PP=(?P[1-9]\d*),CONC=(?P[1-9]\d*)$" ) -_METADATA_PREFIXES = ("_comment", "_example") _SOCKET_ENV = {"NCCL_SOCKET_IFNAME", "GLOO_SOCKET_IFNAME", "TP_SOCKET_IFNAME"} _HF_TOKEN_ENV = {"HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"} _SERVER_RESERVED = {"master_addr", "master_port", "nnodes", "node_rank", "headless"} +_METADATA_PREFIXES = ("_comment", "_example") -class _Forbid(BaseModel): - model_config = ConfigDict(extra="forbid") +def _reject_option_placeholder(value, info): + if not placeholders_rejected(info): + return value + text = value if isinstance(value, str) else json.dumps(value, default=str) + if has_placeholder(text): + raise ValueError(PLACEHOLDER_MESSAGE) + return value -class _Options(BaseModel): - """Strict at the section boundary, flexible only for upstream CLI options.""" +OptionValue = Annotated[Any, AfterValidator(_reject_option_placeholder)] - model_config = ConfigDict(extra="allow") - def extra_options(self) -> Dict[str, Any]: - return dict(self.model_extra or {}) +class _Options(_Allow): + """Strict at the section boundary, flexible only for upstream CLI options.""" + __pydantic_extra__: Dict[str, OptionValue] -VllmThresholdSpec = create_model( - "VllmThresholdSpec", - __base__=_Forbid, - kind=(Any, ...), - value=(Any, ...), -) + def extra_options(self): + return dict(self.model_extra or {}) -def _validate_vllm_thresholds(thresholds): +def _reject_underscore_metric_keys(thresholds): + # A misspelled metric such as _output_throughput would otherwise be dropped + # as a comment, and enforcement only checks that the cell exists. for cell_key, cell in thresholds.items(): - if cell_key == "accuracy": + if cell_key == "accuracy" or cell_key.startswith("_") or not isinstance(cell, dict): continue - if not isinstance(cell, dict): - raise ValueError(f"vLLM threshold cell {cell_key!r} must be an object") - for metric, raw_spec in cell.items(): + for metric, spec in cell.items(): if metric.startswith(_METADATA_PREFIXES): continue - spec = VllmThresholdSpec.model_validate(raw_spec).model_dump() - validate_threshold_spec(metric, spec) + if metric.startswith("_"): + raise ValueError(f"unknown vLLM threshold metric {metric!r}") + extra = sorted(key for key in spec if key.startswith("_")) if isinstance(spec, dict) else [] + if extra: + raise ValueError(f"{cell_key}.{metric}: unknown threshold fields {extra}") + + +def _validate_vllm_thresholds(thresholds): + """Check every cell spec and return the thresholds without comment keys. + + The report code reads ``variant.thresholds`` as plain dicts, so the specs + are returned as written rather than as ``ThresholdSpec`` models. + """ + _reject_underscore_metric_keys(thresholds) + cells = ThresholdFile.model_validate(thresholds).cells() + for cell_key, cell in cells.items(): + for metric, spec in cell.items(): + if "optional" in spec.model_fields_set: + raise ValueError(f"{cell_key}.{metric}: vLLM thresholds do not support optional") + validate_threshold_spec(metric, spec.model_dump()) + cleaned = {cell_key: {metric: thresholds[cell_key][metric] for metric in cell} for cell_key, cell in cells.items()} + if "accuracy" in thresholds: + cleaned["accuracy"] = thresholds["accuracy"] + return cleaned -class RuntimeArgs(_Forbid): +class VllmRuntimeArgs(RuntimeArgs): + model_config = ConfigDict(extra="forbid") + network: str = "host" ipc: str = "host" privileged: bool = True - volumes: List[str] = Field(default_factory=list) + volumes: List[ConfigStr] = Field(default_factory=list) devices: List[str] = Field(default_factory=list) + # container.env is the one env block, so the Hugging Face token check + # cannot be bypassed through docker run -e arguments built from here. + @model_validator(mode="before") + @classmethod + def _reject_env(cls, data): + if isinstance(data, dict) and "env" in data: + raise ValueError("vLLM does not read runtime.args.env; set these variables in container.env") + return data + + +class VllmRuntime(RuntimeSpec): + model_config = ConfigDict(extra="forbid") -class Runtime(_Forbid): name: Literal["docker"] = "docker" - args: RuntimeArgs = Field(default_factory=RuntimeArgs) + args: VllmRuntimeArgs = Field(default_factory=VllmRuntimeArgs) class ContainerEnvTokenError(Exception): @@ -79,12 +124,8 @@ class ContainerEnvTokenError(Exception): """ -class ContainerConfig(_Forbid): - lifetime: Literal["no_launch", "per_run", "persistent"] = "per_run" - name: str - image: str - env: Dict[str, str] = Field(default_factory=dict) - runtime: Runtime = Field(default_factory=Runtime) +class VllmContainerSpec(ContainerSpec): + runtime: VllmRuntime = Field(default_factory=VllmRuntime) @model_validator(mode="before") @classmethod @@ -123,18 +164,11 @@ def socket_env_configured(self): return bool(_SOCKET_ENV & set(self.env)) -class Paths(_Forbid): - shared_fs: str - models_dir: str - log_dir: str - hf_token_file: str - - class ServerParams(_Options): """Harness fields plus arbitrary vLLM ``serve`` flags in snake_case.""" backend: Literal["vllm"] = "vllm" - model: str + model: ConfigStr tensor_parallel_size: int pipeline_parallel_size: int = 1 port: int = 8888 @@ -197,7 +231,7 @@ def _validate_upstream_options(self): _OPTION_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*$") -def _validate_cli_option_map(options: Dict[str, Any], *, section: str) -> None: +def _validate_cli_option_map(options, *, section): for name, value in options.items(): if not _OPTION_NAME_RE.fullmatch(name): raise ValueError(f"{section} option names must be snake_case: {name!r}") @@ -212,7 +246,7 @@ def _validate_cli_option_map(options: Dict[str, Any], *, section: str) -> None: raise ValueError(f"{section}.{name} must be JSON serializable") from exc -def serialize_cli_options(options: Dict[str, Any]) -> List[str]: +def serialize_cli_options(options): """Serialize generic snake-case option maps for vLLM command lines.""" _validate_cli_option_map(options, section="option") argv = [] @@ -241,7 +275,7 @@ class RunCell: concurrency: int @classmethod - def parse(cls, value: str) -> RunCell: + def parse(cls, value): match = _CELL_RE.fullmatch(value) if not match: raise ValueError("run cell must be canonical ISL=,OSL=,TP=,PP=,CONC=") @@ -255,60 +289,42 @@ class ResolvedRun: benchmark_params: Dict[str, Any] -def _strip_metadata(value: Any) -> Any: - """Remove local comments at schema boundaries without touching option payloads.""" - if not isinstance(value, dict): - return value - cleaned = {key: item for key, item in value.items() if not key.startswith(_METADATA_PREFIXES)} - for section in ("paths", "container", "runtime", "args", "accuracy"): - if isinstance(cleaned.get(section), dict): - cleaned[section] = _strip_metadata(cleaned[section]) - if isinstance(cleaned.get("tasks"), list): - cleaned["tasks"] = [ - {key: item for key, item in task.items() if not key.startswith(_METADATA_PREFIXES)} - if isinstance(task, dict) - else task - for task in cleaned["tasks"] - ] - for section in ("server_params", "benchmark_params"): - if isinstance(cleaned.get(section), dict): - cleaned[section] = { - key: item for key, item in cleaned[section].items() if not key.startswith(_METADATA_PREFIXES) - } - if isinstance(cleaned.get("sweeps"), dict): - cleaned["sweeps"] = { - key: { - option: option_value - for option, option_value in values.items() - if not option.startswith(_METADATA_PREFIXES) - } - if isinstance(values, dict) - else values - for key, values in cleaned["sweeps"].items() - if not key.startswith(_METADATA_PREFIXES) - } - return cleaned - - class VariantConfig(_Forbid): enforce_thresholds: bool = True threshold_json: str ib_hca_devices: Union[Literal["auto"], List[str], None] = "auto" ib_netdev: Optional[str] = None paths: Paths - container: ContainerConfig + container: VllmContainerSpec server_params: ServerParams benchmark_params: BenchmarkParams = Field(default_factory=BenchmarkParams) - sweeps: Dict[str, Dict[str, Any]] + sweeps: Dict[str, Dict[str, OptionValue]] runs: List[str] thresholds: Dict[str, Dict[str, Any]] = Field(default_factory=dict) accuracy: AccuracyConfig = Field(default_factory=AccuracyConfig) _effective_topology: Any = PrivateAttr(default=None) _run_cells: Dict[str, RunCell] = PrivateAttr(default_factory=dict) + @field_validator("sweeps", mode="before") + @classmethod + def _drop_sweep_comments(cls, value): + if not isinstance(value, dict): + return value + return { + key: {name: item for name, item in entry.items() if not name.startswith("_")} + if isinstance(entry, dict) + else entry + for key, entry in value.items() + if not key.startswith("_") + } + + @field_validator("thresholds") + @classmethod + def _check_thresholds(cls, value): + return _validate_vllm_thresholds(value) + @model_validator(mode="after") def _validate_runs_and_thresholds(self): - _validate_vllm_thresholds(self.thresholds) if not self.runs: raise ValueError("runs must be a nonempty explicit list") parsed = {key: RunCell.parse(key) for key in self.sweeps} @@ -347,25 +363,25 @@ def _validate_runs_and_thresholds(self): return self @property - def model_id(self) -> str: + def model_id(self): return self.server_params.model - def bind_effective_topology(self, topology) -> None: + def bind_effective_topology(self, topology): self._effective_topology = topology - def cell(self, key: str) -> RunCell: + def cell(self, key): return self._run_cells[key] - def expected_cells(self) -> List[str]: + def expected_cells(self): return list(self.runs) - def cell_key(self, isl, osl, concurrency, **_unused) -> str: + def cell_key(self, isl, osl, concurrency, **_unused): return ( f"ISL={isl},OSL={osl},TP={self.server_params.tensor_parallel_size}," f"PP={self.server_params.pipeline_parallel_size},CONC={concurrency}" ) - def resolved_runs(self) -> List[ResolvedRun]: + def resolved_runs(self): base = self.benchmark_params.model_dump() extras = self.benchmark_params.extra_options() resolved = [] @@ -380,11 +396,3 @@ def resolved_runs(self) -> List[ResolvedRun]: ) resolved.append(ResolvedRun(cell=self.cell(key), benchmark_params=values)) return resolved - - -def load_variant(config_path, cluster_dict): - """Load a vLLM config, threshold file, placeholders, and selected runs.""" - raw, thresholds = substitute_config(config_path, cluster_dict) - raw = _strip_metadata(raw) - raw["thresholds"] = thresholds - return VariantConfig(**raw) diff --git a/cvs/lib/inference/unittests/test_vllm_config_contract.py b/cvs/lib/inference/unittests/test_vllm_config_contract.py deleted file mode 100644 index 01085d1a1..000000000 --- a/cvs/lib/inference/unittests/test_vllm_config_contract.py +++ /dev/null @@ -1,238 +0,0 @@ -import json -import unittest -from pathlib import Path - -from pydantic import ValidationError - -from cvs.lib.inference.utils.vllm_config_loader import ( - ContainerEnvTokenError, - RunCell, - VariantConfig, - _strip_metadata, - load_variant, - serialize_cli_options, -) -from cvs.unittests.support import FAKE_CRED - - -CELL = "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16" -FAKE_CRED_FILE = "/home/test/.hf_token" - - -def _packaged_configs(): - root = Path(__file__).resolve().parents[3] / "input" / "config_file" / "inference" / "vllm" - return sorted(path for path in root.glob("*.json") if not path.name.endswith("threshold.json")) - - -def _config(**overrides): - value = { - "enforce_thresholds": False, - "threshold_json": "threshold.json", - "ib_hca_devices": "auto", - "ib_netdev": "eth0", - "paths": { - "shared_fs": "/home/test", - "models_dir": "/models", - "log_dir": "/logs", - "hf_token_file": FAKE_CRED_FILE, - }, - "container": {"name": "vllm", "image": "image", "runtime": {"name": "docker", "args": {}}}, - "server_params": { - "model": "/models/DeepSeek-R1-0528-FP8", - "tensor_parallel_size": 8, - "pipeline_parallel_size": 2, - }, - "benchmark_params": {"num_prompts": 25}, - "sweeps": {CELL: {"num_prompts": 50}}, - "runs": [CELL], - "thresholds": {}, - } - value.update(overrides) - return value - - -class TestRunCell(unittest.TestCase): - def test_parses_only_canonical_cell_keys(self): - cell = RunCell.parse(CELL) - self.assertEqual((cell.isl, cell.osl, cell.tp, cell.pp, cell.concurrency), (1024, 1024, 8, 2, 16)) - for invalid in ( - "ISL=1024,OSL=1024,TP=8,CONC=16", - "OSL=1024,ISL=1024,TP=8,PP=2,CONC=16", - "ISL=1024, OSL=1024,TP=8,PP=2,CONC=16", - "ISL=0,OSL=1024,TP=8,PP=2,CONC=16", - ): - with self.subTest(invalid=invalid): - with self.assertRaises(ValueError): - RunCell.parse(invalid) - - -class TestVllmConfigContract(unittest.TestCase): - def test_resolves_selected_run_with_cell_overrides(self): - variant = VariantConfig(**_config()) - (run,) = variant.resolved_runs() - self.assertEqual(run.cell.key, CELL) - self.assertEqual(run.benchmark_params["num_prompts"], 50) - self.assertEqual(run.benchmark_params["random_input_len"], 1024) - self.assertEqual(run.benchmark_params["random_output_len"], 1024) - self.assertEqual(run.benchmark_params["max_concurrency"], 16) - - def test_rejects_unknown_root_and_legacy_shape(self): - for kwargs in ({"unexpected": 1}, {"schema_version": 1}, {"params": {}}): - with self.subTest(kwargs=kwargs): - with self.assertRaises(ValidationError): - VariantConfig(**_config(**kwargs)) - - def test_requires_nonempty_explicit_runs(self): - with self.assertRaisesRegex(ValidationError, "nonempty explicit"): - VariantConfig(**_config(runs=[])) - - def test_rejects_mismatched_parallelism_and_missing_enforced_threshold(self): - wrong_cell = "ISL=1024,OSL=1024,TP=4,PP=2,CONC=16" - with self.assertRaisesRegex(ValidationError, "conflicts"): - VariantConfig(**_config(sweeps={wrong_cell: {}}, runs=[wrong_cell])) - with self.assertRaisesRegex(ValidationError, "missing threshold"): - VariantConfig(**_config(enforce_thresholds=True)) - - def test_record_only_mode_allows_unmeasured_selected_cells(self): - self.assertEqual(VariantConfig(**_config()).expected_cells(), [CELL]) - - def test_accepts_complete_container_network_environment(self): - config = _config() - config["container"]["env"] = { - "NCCL_IB_HCA": "rdma0", - "NCCL_SOCKET_IFNAME": "eno0", - "GLOO_SOCKET_IFNAME": "eno0", - "TP_SOCKET_IFNAME": "eno0", - } - variant = VariantConfig(**config) - self.assertEqual(variant.container.nccl_ib_hcas, ["rdma0"]) - self.assertTrue(variant.container.socket_env_configured) - - def test_rejects_incomplete_container_socket_environment(self): - names = ("NCCL_SOCKET_IFNAME", "GLOO_SOCKET_IFNAME", "TP_SOCKET_IFNAME") - for missing in names: - with self.subTest(missing=missing): - config = _config() - config["container"]["env"] = {name: "eno0" for name in names if name != missing} - with self.assertRaisesRegex(ValidationError, "must set all socket interface"): - VariantConfig(**config) - - def test_rejects_empty_container_hca(self): - config = _config() - config["container"]["env"] = {"NCCL_IB_HCA": " , "} - with self.assertRaisesRegex(ValidationError, "must name at least one device"): - VariantConfig(**config) - - def test_rejects_hf_token_in_container_env_without_echoing_value(self): - for name in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "hf_token", "Hf_Token"): - for extra in ({}, {"NCCL_SOCKET_IFNAME": "eno0"}): - with self.subTest(name=name, extra=extra): - config = _config() - config["container"]["env"] = {**extra, name: FAKE_CRED} - with self.assertRaises(ContainerEnvTokenError) as ctx: - VariantConfig(**config) - self.assertIn(f"container.env must not set ['{name}']", str(ctx.exception)) - self.assertIn("paths.hf_token_file", str(ctx.exception)) - self.assertNotIn(FAKE_CRED, repr(ctx.exception.args)) - - def test_other_container_errors_keep_their_input(self): - config = _config() - config["container"]["env"] = {"NCCL_SOCKET_IFNAME": "eno0"} - with self.assertRaises(ValidationError) as ctx: - VariantConfig(**config) - self.assertIn("input_value", str(ctx.exception)) - - def test_preserves_runtime_environment_without_model_policy(self): - for env in ( - {"VLLM_ROCM_USE_AITER": "0", "VLLM_ROCM_USE_AITER_MHA": "0"}, - {"VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "0"}, - {"CUSTOM_RUNTIME_OPTION": "custom value"}, - ): - with self.subTest(env=env): - config = _config() - config["container"]["env"] = env - variant = VariantConfig(**config) - self.assertEqual(variant.container.env, env) - - -class TestMetadataAndOptionSerialization(unittest.TestCase): - def test_strips_local_comments_but_preserves_option_payload(self): - value = _strip_metadata( - { - "_comment": "root", - "server_params": { - "_comment": "server", - "json_arg": {"_comment": "upstream payload"}, - }, - "sweeps": {CELL: {"_comment": "cell", "json_arg": {"_comment": "upstream payload"}}}, - "accuracy": {"tasks": [{"_comment": "task", "id": "gsm8k", "task": "gsm8k"}]}, - } - ) - self.assertNotIn("_comment", value) - self.assertEqual(value["server_params"]["json_arg"], {"_comment": "upstream payload"}) - self.assertEqual(value["sweeps"][CELL]["json_arg"], {"_comment": "upstream payload"}) - self.assertNotIn("_comment", value["accuracy"]["tasks"][0]) - - def test_serializes_documented_value_shapes(self): - options = { - "trust_remote_code": True, - "max_model_len": 8192, - "served_model_name": ["first", "second"], - "structured_outputs": {"json": {"type": "object"}}, - "optional": None, - } - self.assertEqual( - serialize_cli_options(options), - [ - "--trust-remote-code", - "--max-model-len", - "8192", - "--served-model-name", - "first", - "second", - "--structured-outputs", - '{"json":{"type":"object"}}', - ], - ) - - def test_rejects_ambiguous_false_and_reserved_names(self): - with self.assertRaises(ValueError): - serialize_cli_options({"enforce_eager": False}) - with self.assertRaisesRegex(ValidationError, "cannot override harness"): - VariantConfig( - **_config( - server_params={ - "model": "/models/x", - "tensor_parallel_size": 8, - "pipeline_parallel_size": 2, - "nnodes": 2, - } - ) - ) - - with self.assertRaisesRegex(ValidationError, "benchmark_params cannot override harness"): - VariantConfig(**_config(benchmark_params={"num_prompts": 25, "percentile_metrics": "ttft"})) - - -class TestPackagedVllmCatalog(unittest.TestCase): - def test_shipped_configs_resolve_their_declared_contract(self): - configs = _packaged_configs() - self.assertTrue(configs, "No packaged vLLM configs found") - for path in configs: - with self.subTest(config=path.name): - raw = json.loads(path.read_text()) - threshold_path = path.parent / raw["threshold_json"] - self.assertTrue(threshold_path.is_file(), f"Missing packaged threshold file: {threshold_path}") - variant = load_variant(path, {"username": "test"}) - runs = variant.resolved_runs() - self.assertTrue(runs) - self.assertEqual([run.cell.key for run in runs], variant.runs) - self.assertTrue(set(variant.runs) <= set(variant.sweeps)) - self.assertTrue(set(variant.thresholds) - {"accuracy"} <= set(variant.sweeps)) - for run in runs: - self.assertEqual(run.cell.tp, variant.server_params.tensor_parallel_size) - self.assertEqual(run.cell.pp, variant.server_params.pipeline_parallel_size) - - -if __name__ == "__main__": - unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_config_loader.py b/cvs/lib/inference/unittests/test_vllm_config_loader.py deleted file mode 100644 index d1b83c031..000000000 --- a/cvs/lib/inference/unittests/test_vllm_config_loader.py +++ /dev/null @@ -1,176 +0,0 @@ -'''Unit tests for strict vLLM threshold schema validation.''' - -import math -import unittest - -from pydantic import ValidationError - -from cvs.lib.inference.utils.vllm_config_loader import ( - GATED_GPU_METRICS, - GATED_PROM_METRICS, - VariantConfig, -) - - -CELL = "ISL=128,OSL=2048,TP=8,PP=1,CONC=16" -FAKE_CRED_FILE = "/home/x/.hf" - - -def _variant(thresholds, enforce=False): - return VariantConfig( - enforce_thresholds=enforce, - threshold_json="threshold.json", - paths={ - "shared_fs": "/home/x", - "models_dir": "/home/x/models", - "log_dir": "/home/x/LOGS", - "hf_token_file": FAKE_CRED_FILE, - }, - container={"name": "test", "image": "test", "runtime": {"name": "docker", "args": {}}}, - server_params={"model": "amd/model", "tensor_parallel_size": 8}, - sweeps={CELL: {}}, - runs=[CELL], - thresholds=thresholds, - ) - - -class TestPartialThresholdCells(unittest.TestCase): - def test_one_spec_cell_loads_with_enforcement_off_and_on(self): - thresholds = {CELL: {"output_throughput": {"kind": "min", "value": 0}}} - for enforce in (False, True): - with self.subTest(enforce=enforce): - variant = _variant(thresholds, enforce=enforce) - self.assertEqual(variant.thresholds, thresholds) - - def test_empty_cell_loads_with_enforcement_off_and_on(self): - for enforce in (False, True): - with self.subTest(enforce=enforce): - variant = _variant({CELL: {}}, enforce=enforce) - self.assertEqual(variant.thresholds[CELL], {}) - - def test_cell_metadata_loads_with_enforcement_off_and_on(self): - thresholds = { - CELL: { - "_comment": "calibrate before enforcing", - "_example": {"output_throughput": {"kind": "min", "value": 1}}, - "output_throughput": {"kind": "min", "value": 0}, - } - } - for enforce in (False, True): - with self.subTest(enforce=enforce): - variant = _variant(thresholds, enforce=enforce) - self.assertEqual(variant.thresholds, thresholds) - - def test_non_metadata_unknown_key_is_rejected(self): - for enforce in (False, True): - for metric in ("notes", "_typo", "_metadata", "_"): - with self.subTest(enforce=enforce, metric=metric): - with self.assertRaisesRegex( - ValidationError, - f"unknown vLLM threshold metric {metric!r}", - ): - _variant( - {CELL: {metric: {"kind": "min", "value": 0}}}, - enforce=enforce, - ) - - def test_enforcement_still_requires_selected_run_cell(self): - _variant({}, enforce=False) - with self.assertRaisesRegex(ValidationError, "missing threshold coverage"): - _variant({}, enforce=True) - - def test_accuracy_is_exempt_from_vllm_metric_validation(self): - accuracy = { - "gsm8k": { - "gsm8k.exact_match__strict-match": { - "kind": "min", - "value": 0, - "tolerance_pct": 5, - } - } - } - variant = _variant({CELL: {}, "accuracy": accuracy}, enforce=True) - self.assertEqual(variant.thresholds["accuracy"], accuracy) - - -class TestStrictThresholdSpecs(unittest.TestCase): - def assertRejected(self, metric, spec): - with self.assertRaises(ValidationError): - _variant({CELL: {metric: spec}}) - - def test_rejects_nonfinite_or_non_builtin_values(self): - for value in (True, "1", None, [], {}, math.nan, math.inf, -math.inf): - with self.subTest(value=value): - self.assertRejected( - "output_throughput", - {"kind": "min", "value": value}, - ) - - def test_rejects_missing_unknown_and_legacy_kinds(self): - for spec in ( - {"value": 1}, - {"kind": "min"}, - {"kind": "info", "value": 1}, - {"kind": "min_tok_s", "value": 1}, - {"kind": "max_ms", "value": 1}, - {"kind": "within", "value": 1}, - {"kind": "min_ratio", "value": 1}, - {"kind": "other", "value": 1}, - ): - with self.subTest(spec=spec): - self.assertRejected("output_throughput", spec) - - def test_rejects_direction_mismatch(self): - self.assertRejected("output_throughput", {"kind": "max", "value": 1}) - self.assertRejected("mean_ttft_ms", {"kind": "min", "value": 1}) - - def test_rejects_prefixed_and_unknown_metric_names(self): - for metric in ("client.output_throughput", "gpu.gpu_compute_util_pct", "prom.queue_time_p50_ms", "new_metric"): - with self.subTest(metric=metric): - self.assertRejected(metric, {"kind": "min", "value": 1}) - - def test_rejects_extra_spec_fields(self): - for field, value in ( - ("unit", "tok/s"), - ("reference", "total_token_throughput"), - ("tolerance_pct", 5), - ("extra", 1), - ): - with self.subTest(field=field): - self.assertRejected( - "output_throughput", - {"kind": "min", "value": 1, field: value}, - ) - - def test_accepts_finite_int_float_zero_and_negative(self): - for value in (0, -1, 1.5): - with self.subTest(value=value): - variant = _variant({CELL: {"output_throughput": {"kind": "min", "value": value}}}) - self.assertEqual(variant.thresholds[CELL]["output_throughput"]["value"], value) - - -class TestDatasourceViews(unittest.TestCase): - def test_gpu_and_prometheus_names_are_bare_and_complete(self): - self.assertEqual( - GATED_GPU_METRICS, - { - "peak_gpu_memory_mb", - "model_load_memory_mb", - "model_load_s", - "gpu_bandwidth_util_pct", - "gpu_compute_util_pct", - }, - ) - self.assertEqual( - GATED_PROM_METRICS, - { - "queue_time_p50_ms", - "queue_time_p95_ms", - "prefill_time_p50_ms", - "prefill_time_p95_ms", - }, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_config_loader_accuracy.py b/cvs/lib/inference/unittests/test_vllm_config_loader_accuracy.py deleted file mode 100644 index dd2f94cea..000000000 --- a/cvs/lib/inference/unittests/test_vllm_config_loader_accuracy.py +++ /dev/null @@ -1,103 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. - -Unit tests for the `accuracy` field wired onto vllm's VariantConfig (step 5 of -the accuracy-harness plan). AccuracyConfig itself is fully covered by -test_accuracy_config.py; these tests only pin the wiring: default/opt-in -behavior and pass-through of an explicit accuracy block. -''' - -import unittest - -from cvs.lib.inference.utils.accuracy_config import AccuracyConfig -from cvs.lib.inference.utils.vllm_config_loader import VariantConfig - -FAKE_CRED_FILE = "/home/x/.hf" - - -def _base_kwargs(**overrides): - cell = "ISL=1024,OSL=1024,TP=8,PP=1,CONC=16" - kwargs = dict( - enforce_thresholds=False, - threshold_json="threshold.json", - paths={ - "shared_fs": "/home/x", - "models_dir": "/home/x/models", - "log_dir": "/home/x/LOGS", - "hf_token_file": FAKE_CRED_FILE, - }, - container={"name": "test", "image": "test", "runtime": {"name": "docker", "args": {}}}, - server_params={"model": "/models/test-model", "tensor_parallel_size": 8}, - sweeps={cell: {}}, - runs=[cell], - thresholds={}, - ) - kwargs.update(overrides) - return kwargs - - -class TestVariantConfigAccuracyField(unittest.TestCase): - def test_defaults_to_empty_accuracy_config_when_omitted(self): - vc = VariantConfig(**_base_kwargs()) - self.assertIsInstance(vc.accuracy, AccuracyConfig) - self.assertEqual(vc.accuracy.tasks, []) - - def test_accepts_explicit_accuracy_tasks(self): - vc = VariantConfig(**_base_kwargs(accuracy={"tasks": [{"id": "mmlu", "task": "mmlu", "num_fewshot": 5}]})) - self.assertEqual(len(vc.accuracy.tasks), 1) - self.assertEqual(vc.accuracy.tasks[0].id, "mmlu") - self.assertEqual(vc.accuracy.tasks[0].num_fewshot, 5) - - def test_duplicate_task_ids_rejected_through_variant_config(self): - with self.assertRaises(ValueError): - VariantConfig( - **_base_kwargs( - accuracy={ - "tasks": [ - {"id": "mmlu", "task": "mmlu"}, - {"id": "mmlu", "task": "mmlu"}, - ] - } - ) - ) - - -class TestAccuracyThresholdKeyDoesNotTripSweepCoverage(unittest.TestCase): - """The top-level "accuracy" threshold key must not be flagged as an - unrecognized sweep-cell key by _check_thresholds_cover_sweep, now that it - delegates to the shared validate_thresholds_cover_sweep.""" - - _CELL = "ISL=1024,OSL=1024,TP=8,PP=1,CONC=16" - - def _full_gated_specs(self): - return {"output_throughput": {"kind": "min", "value": 0}} - - def test_accuracy_key_alongside_full_sweep_coverage_constructs(self): - vc = VariantConfig( - **_base_kwargs( - enforce_thresholds=True, - thresholds={ - self._CELL: self._full_gated_specs(), - "accuracy": {"mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.5}}}, - }, - ) - ) - self.assertIn("accuracy", vc.thresholds) - - def test_typo_key_is_rejected_at_config_load(self): - with self.assertRaisesRegex(ValueError, "threshold cells"): - VariantConfig( - **_base_kwargs( - enforce_thresholds=True, - thresholds={ - self._CELL: self._full_gated_specs(), - "accuracy": {"mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.5}}}, - "acuracy": {}, - }, - ) - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_deck_profile.py b/cvs/lib/inference/unittests/test_vllm_deck_profile.py index 978587f47..d6711ffe6 100644 --- a/cvs/lib/inference/unittests/test_vllm_deck_profile.py +++ b/cvs/lib/inference/unittests/test_vllm_deck_profile.py @@ -3,11 +3,13 @@ All rights reserved. ''' +import tempfile import unittest from pathlib import Path from cvs.lib.inference.vllm_topology import EffectiveVllmTopology -from cvs.lib.inference.utils.vllm_config_loader import load_variant +from cvs.lib.conf_lib.loader import load_config +from cvs.lib.conf_lib.schemas.vllm import VariantConfig from cvs.lib.inference.utils.vllm_metrics import ( CLIENT_METRICS, METRIC_CATEGORIES, @@ -20,6 +22,7 @@ from cvs.lib.report.registry import get_resolved_profile, resolve_suite_report_config from cvs.lib.report.rundeck.config_adapter import build_inference_config_from_profile from cvs.lib.report.inference import build_inference_report_payload +from cvs.unittests.support import write_filled_config class TestVllmDeckProfile(unittest.TestCase): @@ -121,10 +124,11 @@ def test_split_suite_stems_reuse_vllm_profile(self): def test_distributed_cell_uses_bound_threshold_key(self): root = Path(__file__).resolve().parents[3] - variant = load_variant( - root / "input/config_file/inference/vllm/mi3xx_vllm_llama33-70b_fp8_distributed.json", - {"username": "test"}, - ) + with tempfile.TemporaryDirectory() as tmp: + config = write_filled_config( + root / "input/config_file/inference/vllm/mi3xx_vllm_llama33-70b_fp8_distributed.json", tmp + ) + variant = load_config(VariantConfig, config, {"username": "test"}).config variant.bind_effective_topology(EffectiveVllmTopology("distributed", ("node0", "node1"), 2)) cell_id = variant.expected_cells()[0] variant.enforce_thresholds = True diff --git a/cvs/lib/inference/unittests/test_vllm_job.py b/cvs/lib/inference/unittests/test_vllm_job.py index 308001308..d2445f2fe 100644 --- a/cvs/lib/inference/unittests/test_vllm_job.py +++ b/cvs/lib/inference/unittests/test_vllm_job.py @@ -16,7 +16,7 @@ from unittest import mock from cvs.lib.inference.unittests.fake_orch import FakeOrch -from cvs.lib.inference.utils.vllm_config_loader import VariantConfig +from cvs.lib.conf_lib.schemas.vllm import VariantConfig from cvs.lib.inference.vllm_job import VllmJob from cvs.unittests.support import FAKE_CRED, TMP, run_trusted diff --git a/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py b/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py index 89620ccf5..5c9b1016e 100644 --- a/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py +++ b/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py @@ -4,7 +4,7 @@ Unit tests for the Ray distributed-executor-backend support added to cvs.lib.inference.vllm_job.VllmJob and -cvs.lib.inference.utils.vllm_config_loader.VariantConfig. +cvs.lib.conf_lib.schemas.vllm.VariantConfig. These tests were originally derived from the Ray execution contract before implementation and retain coverage of its observable behavior. @@ -60,7 +60,7 @@ from pydantic import ValidationError from cvs.lib.inference.vllm_topology import EffectiveVllmTopology -from cvs.lib.inference.utils.vllm_config_loader import VariantConfig, serialize_cli_options +from cvs.lib.conf_lib.schemas.vllm import VariantConfig, serialize_cli_options from cvs.lib.inference.vllm_job import VllmJob RAY = {"distributed-executor-backend": "ray"} diff --git a/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py b/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py index 963617f85..888be3dcb 100644 --- a/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py +++ b/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py @@ -14,7 +14,7 @@ import json import unittest import unittest.mock as mock -from cvs.lib.inference.utils.vllm_config_loader import VariantConfig, serialize_cli_options +from cvs.lib.conf_lib.schemas.vllm import VariantConfig, serialize_cli_options from cvs.lib.inference.vllm_job import VllmJob from cvs.unittests.support import TMP diff --git a/cvs/lib/inference/vllm_job.py b/cvs/lib/inference/vllm_job.py index dc9d822d9..37bc997a1 100644 --- a/cvs/lib/inference/vllm_job.py +++ b/cvs/lib/inference/vllm_job.py @@ -41,7 +41,7 @@ from typing import Optional from cvs.lib import globals -from cvs.lib.inference.utils.vllm_config_loader import serialize_cli_options +from cvs.lib.conf_lib.schemas.vllm import serialize_cli_options from cvs.lib.inference.utils.vllm_metrics import project_vllm_metrics from cvs.lib.utils.model_query_lib import OpenAIProbe diff --git a/cvs/tests/inference/vllm/_common.py b/cvs/tests/inference/vllm/_common.py index 7c43ae1c9..800f6b46b 100644 --- a/cvs/tests/inference/vllm/_common.py +++ b/cvs/tests/inference/vllm/_common.py @@ -8,7 +8,8 @@ import pytest from cvs.lib import globals -from cvs.lib.inference.utils.vllm_config_loader import load_variant +from cvs.lib.conf_lib.loader import load_config +from cvs.lib.conf_lib.schemas.vllm import VariantConfig from cvs.lib.inference.utils.vllm_metrics import ( METRIC_REGISTRY, VLLM_RESULTS_COLUMNS, @@ -42,7 +43,7 @@ def pytest_generate_tests(metafunc): config_file = metafunc.config.getoption("config_file") if not config_file: return - variant = load_variant(config_file, {}) + variant = load_config(VariantConfig, config_file, {}).config runs = variant.resolved_runs() ids = [run.cell.key for run in runs] if "accuracy_task" in metafunc.fixturenames: diff --git a/cvs/tests/inference/vllm/conftest.py b/cvs/tests/inference/vllm/conftest.py index 927c88c57..e217717db 100644 --- a/cvs/tests/inference/vllm/conftest.py +++ b/cvs/tests/inference/vllm/conftest.py @@ -3,7 +3,6 @@ All rights reserved. ''' -import json from pathlib import Path import pytest @@ -11,8 +10,9 @@ from cvs.core.orchestrators.factory import OrchestratorConfig, OrchestratorFactory from cvs.lib import globals from cvs.lib.inference.vllm_topology import resolve_vllm_topology, scope_vllm_cluster -from cvs.lib.inference.utils.vllm_config_loader import load_variant from cvs.lib.inference.utils.vllm_metrics import VLLM_RESULTS_COLUMNS +from cvs.lib.conf_lib.loader import load_config +from cvs.lib.conf_lib.schemas.vllm import VariantConfig from cvs.lib.report.benchmark_metric_registry import ( benchmark_metric_columns_for_nodeid, benchmark_metric_rows_from_item, @@ -26,7 +26,6 @@ render_benchmark_metrics_html, ) from cvs.lib.report.subtest_reports import called_from_subtest_context, is_subtest_report -from cvs.lib.utils_lib import resolve_cluster_config_placeholders from cvs.tests.inference.vllm._shared import validate_vllm_execution_mode log = globals.log @@ -68,39 +67,20 @@ def _attach_metric_panel(report, rows) -> None: stamp_benchmark_metric_rows_on_report(report, rows) -def _deep_merge(base, override): - """Recursively merge `override` onto `base` (dicts merged key-wise, scalars/lists replaced). - - Protects cluster-set SCALAR and DICT container keys (e.g. shm_size, an env - map) from being wiped by a top-level replace: they survive unless the variant - overrides that same key. List keys (e.g. runtime.args, volume mounts) are - REPLACED here, not unioned -- the cluster's list values are recombined with - the variant's additively further downstream, in container.py's getters. - """ - if not (isinstance(base, dict) and isinstance(override, dict)): - return override - out = dict(base) - for k, v in override.items(): - out[k] = _deep_merge(base[k], v) if k in base else v - return out - - @pytest.fixture(scope="module") -def cluster_dict(pytestconfig): +def loaded_config(pytestconfig): cluster_file = pytestconfig.getoption("cluster_file") if not cluster_file: pytest.fail("--cluster_file is required") - with open(cluster_file) as fp: - d = json.load(fp) - return resolve_cluster_config_placeholders(d) - - -@pytest.fixture(scope="module") -def variant_config(pytestconfig, cluster_dict): config_file = pytestconfig.getoption("config_file") if not config_file: pytest.fail("--config_file is required") - return load_variant(config_file, cluster_dict) + return load_config(VariantConfig, config_file, cluster_file) + + +@pytest.fixture(scope="module") +def variant_config(loaded_config): + return loaded_config.config class _Lifecycle: @@ -134,7 +114,7 @@ def lifecycle(): @pytest.fixture(scope="module") -def orch(cluster_dict, variant_config, lifecycle, vllm_mode): +def orch(loaded_config, lifecycle, vllm_mode): """Construct a ContainerOrchestrator and own ONLY its teardown safety net. The actual launch/sshd happen in test_launch_container / test_setup_sshd @@ -143,19 +123,12 @@ def orch(cluster_dict, variant_config, lifecycle, vllm_mode): the container is still torn down here. When test_teardown ran successfully it sets lifecycle.torn_down, so the finalizer no-ops (no double teardown). """ - # OrchestratorConfig.from_configs does a top-level dict.update, so a bare variant - # container block would wipe the cluster file's container settings. Deep-merge the - # variant ONTO the cluster block so cluster-set scalar/dict keys survive, with the - # variant winning on conflicting keys. (List keys like runtime.args are replaced - # here but recombined additively downstream in container.py's getters.) - suite_cluster = scope_vllm_cluster(vllm_mode, cluster_dict) - container_block = _deep_merge( - suite_cluster.get("container", {}), - variant_config.container.model_dump(), - ) + # from_configs replaces top-level keys, so the container block passed here must + # already hold the cluster's container settings; load_config merged them. + suite_cluster = scope_vllm_cluster(vllm_mode, loaded_config.cluster) testsuite_config = { "orchestrator": "container", - "container": container_block, + "container": loaded_config.container, } cfg = OrchestratorConfig.from_configs(suite_cluster, testsuite_config) o = OrchestratorFactory.create_orchestrator(log, cfg) diff --git a/cvs/unittests/support.py b/cvs/unittests/support.py index 4f3c33787..bcb099289 100644 --- a/cvs/unittests/support.py +++ b/cvs/unittests/support.py @@ -10,8 +10,11 @@ Tests also use ``FAKE_CRED`` instead of credential-looking literals. ''' +import json +import re import subprocess # nosec B404 - test helper; callers pass argv lists of commands the test itself builds import xml.etree.ElementTree as ElementTree # nosec B405 - parses only files the calling test generated +from pathlib import Path TMP = "/tmp" # nosec B108 - CVS uses /tmp as its shared remote scratch dir; tests assert the real paths FAKE_CRED = "fake-cred" @@ -32,3 +35,30 @@ def run_trusted(argv, **kwargs): def parse_trusted_xml(path): """Parse an XML file the calling test generated and return the ElementTree.""" return ElementTree.parse(path) # nosec B314 - input is a file the calling test generated + + +_PLACEHOLDER_RE = re.compile(r"]*)?>", re.IGNORECASE) + + +def write_filled_config(template_path, out_dir): + """Write a copy of a shipped config with every ```` value filled in. + + Comment keys are copied unchanged. ``threshold_json`` is made absolute so + the copy still reads the shipped threshold file. Returns the copy's path. + """ + template_path = Path(template_path) + + def fill(node): + if isinstance(node, dict): + return {key: item if key.startswith("_") else fill(item) for key, item in node.items()} + if isinstance(node, list): + return [fill(item) for item in node] + if isinstance(node, str): + return _PLACEHOLDER_RE.sub("filled", node) + return node + + config = fill(json.loads(template_path.read_text())) + config["threshold_json"] = str(template_path.parent / config["threshold_json"]) + out_path = Path(out_dir) / template_path.name + out_path.write_text(json.dumps(config)) + return out_path diff --git a/docs/how-to/test-suites/inference/vllm.rst b/docs/how-to/test-suites/inference/vllm.rst index 02b5529dc..484c11038 100644 --- a/docs/how-to/test-suites/inference/vllm.rst +++ b/docs/how-to/test-suites/inference/vllm.rst @@ -68,7 +68,7 @@ Step 2: Fill in the placeholders Replace every ```` before the run. ``{user-id}`` is not one of those: it becomes the login name of the ``cvs run`` process, unless the cluster file sets ``username``. -In the **cluster file**, set your SSH user, private key path, and node addresses. Unresolved ```` strings in the cluster file stop startup. The vLLM loader does not apply that same hard-exit to the workload file, except when ``threshold_json`` itself still contains ````. An unreplaced image name or mount fails later, at pull or ``docker run``. See :doc:`/how-to/run-with-containers` for the cluster-file walkthrough. +In the **cluster file**, set your SSH user, private key path, and node addresses. Unresolved ```` strings in the cluster file stop startup. A ```` or ```` left in a checked workload-file field fails test collection, and the error names every such field. :ref:`vllm-config-structure` lists the fields that are checked. See :doc:`/how-to/run-with-containers` for the cluster-file walkthrough. In the **Llama single-node file**, replace only these strings: @@ -313,15 +313,15 @@ CVS then bootstraps a Ray cluster before serving: ``ray start --head --port=1 unless distributed-executor-backend=ray``. You have two or more hosts on ``mp`` without pipeline parallelism. Raise ``pipeline_parallel_size``, or set ``distributed_executor_backend`` to ``"ray"``. **"vllm_distributed requires container.env.NCCL_SOCKET_IFNAME on multi-host clusters".** Set the socket interface. If you set any of ``NCCL_SOCKET_IFNAME``, ``GLOO_SOCKET_IFNAME``, or ``TP_SOCKET_IFNAME``, set all three. -**"Container image not specified in config".** ``container.image`` is empty. The orchestrator logs that line, then ``test_launch_container`` fails because ``setup_containers()`` returned false. The vLLM schema requires ``image``, so omitting the key fails validation instead. An empty string overwrites a cluster-file image. The packaged Llama files use ````, which is not empty and is not a real image until you replace it. +**"Container image not specified in config".** ``container.image`` is empty. The orchestrator logs that line, then ``test_launch_container`` fails because ``setup_containers()`` returned false. The vLLM schema requires ``image``, so omitting the key fails validation instead. An empty string overwrites a cluster-file image. -**"env" under "container.runtime.args".** ``RuntimeArgs`` rejects unknown keys, so ``env`` there fails config load. Put ``env`` on ``container``. +**"env" under "container.runtime.args".** ``env`` under ``runtime.args`` fails config load. Put ``env`` on ``container``. **A threshold fails with "actual must be a finite built-in int or float".** The gated datasource did not produce a valid value. Inspect the raw benchmark artifact, GPU telemetry, or server metrics for that cell. The suite owns benchmark percentile collection; workload configuration cannot override it. diff --git a/docs/reference/configuration-files/inference/vllm.rst b/docs/reference/configuration-files/inference/vllm.rst index 8e8805086..9919a7279 100644 --- a/docs/reference/configuration-files/inference/vllm.rst +++ b/docs/reference/configuration-files/inference/vllm.rst @@ -73,6 +73,8 @@ Each stage of the run is an independent test, so every stage becomes its own tim If a stage fails, later stages are skipped rather than cascading into confusing downstream errors. The container is still torn down by a leak-guard even when a mid-sweep test fails. +.. _vllm-config-structure: + Configuration file structure ============================ @@ -127,6 +129,13 @@ A vLLM configuration file has these top-level keys: ``false`` is rejected on those extra options. The harness booleans ``benchmark_params.ignore_eos`` and ``trust_remote_code`` do accept ``false``. Use an option's negative form instead of ``false`` for any other flag. + Keys that start with ``_`` are comments and are ignored, except inside + ``container.env`` and option values. In a threshold cell, only metric names + starting with ``_comment`` or ``_example`` are comments. A ```` or + ```` fails config load, and the error names every such field, + when it is left in a ``paths`` value, ``container.name``, ``container.image``, + a ``container.env`` value, a ``runtime.args`` volume, ``server_params.model``, + or an extra option or sweep value. Other values are not checked for placeholders. Placeholder substitution ------------------------ @@ -289,7 +298,7 @@ The vLLM ``container`` block selects the image, the container name, the lifetime Container block keys -------------------- -``ContainerConfig`` accepts only these keys. Anything else, including ``image_tar``, is a validation error. +``container`` accepts only these keys. Anything else, including ``image_tar``, is a validation error. .. list-table:: :widths: 3 2 5 @@ -312,21 +321,23 @@ Container block keys - The only value the vLLM schema accepts. * - ``runtime.args`` - see below - - Only ``network``, ``ipc``, ``privileged``, ``volumes``, and ``devices``. + - The Docker keys under :ref:`vllm-runtime-args`. ``env`` is rejected. * - ``env`` - ``{}`` - - Environment variables passed as ``docker run -e``. **On** ``container``, not under ``runtime.args``. ``HF_TOKEN`` and ``HUGGING_FACE_HUB_TOKEN`` are rejected; set ``paths.hf_token_file`` instead. + - Environment variables passed as ``docker run -e``. **On** ``container``, not under ``runtime.args``. ``HF_TOKEN`` and ``HUGGING_FACE_HUB_TOKEN`` are rejected in the vLLM file; set ``paths.hf_token_file`` instead. A token in the cluster file's ``container.env`` is passed to the container, and logs redact it. -``image_tar``, ``runtime.args.registry``, ``runtime.args.shm_size``, and ``runtime.args.cap_add`` are not fields of this model. Putting them in the vLLM file fails config load. Set those on the cluster file's ``container`` block. The vLLM conftest deep-merges the variant onto that block and keeps cluster keys the variant does not set. The orchestrator still emits them: ``image_tar`` loads an image, ``registry`` logs in before a pull, ``shm_size`` becomes ``--shm-size``, and ``cap_add`` appends to the orchestrator's default capabilities. See :doc:`/reference/cluster/cluster-file` for the orchestrator container keys that page documents. +``image_tar`` is not a field of this model; set it on the cluster file's ``container`` block. ``runtime.args`` keys such as ``registry``, ``shm_size``, and ``cap_add`` can be set in either file. CVS deep-merges the vLLM ``container`` block onto the cluster file's block and keeps cluster keys the vLLM file does not set. The orchestrator emits them: ``image_tar`` loads an image, ``registry`` logs in before a pull, ``shm_size`` becomes ``--shm-size``, and ``cap_add`` appends to the orchestrator's default capabilities. See :doc:`/reference/cluster/cluster-file` for the orchestrator container keys that page documents. .. warning:: - Put ``env`` on ``container``. ``RuntimeArgs`` forbids unknown keys, so ``env`` under ``runtime.args`` is rejected before launch. + Put ``env`` on ``container``. ``env`` under ``runtime.args`` is rejected at config load. + +.. _vllm-runtime-args: Runtime arguments ----------------- -``RuntimeArgs`` accepts only the keys below. Schema defaults are ``network`` ``host``, ``ipc`` ``host``, ``privileged`` true, and empty ``volumes`` and ``devices``. The orchestrator then appends its own mounts and devices; you cannot drop those defaults from the vLLM file. +``runtime.args`` accepts only the keys below. ``volumes``, ``devices``, ``network``, ``ipc`` and ``privileged`` come from the vLLM file or its schema defaults. ``shm_size``, ``cap_add`` and ``registry`` keep a cluster value when the vLLM file leaves them out. The orchestrator then appends its own mounts and devices; you cannot drop those defaults from the vLLM file. .. list-table:: :widths: 3 7 @@ -344,6 +355,14 @@ Runtime arguments - ``--ipc``. Schema default ``host``. * - ``privileged`` - ``--privileged`` when true. Schema default true. + * - ``cap_add``, ``security_opt``, ``group_add``, ``ulimit`` + - Lists appended to the orchestrator's default capabilities, security options, groups, and ulimits. + * - ``user`` + - ``--user``. + * - ``shm_size`` + - ``--shm-size``. + * - ``registry`` + - Registry login before a pull. See `Registry authentication`_. The assembled command is: @@ -355,7 +374,7 @@ The container is a long-lived sidecar; every workload command runs through ``doc .. note:: - ``--gpus`` is deliberately never emitted. GPU access on AMD hardware comes from the ``/dev/kfd`` and ``/dev/dri`` device mounts plus the ``video`` group, which the orchestrator adds. Capabilities, security options, groups, ulimits, shared-memory size, registry login, and image-tar load are orchestrator behavior. Set them in the cluster file, not in the vLLM schema. ``runtime.args.shm_size`` in the vLLM file is rejected; the same key on the cluster file is emitted as ``--shm-size``. + ``--gpus`` is deliberately never emitted. GPU access on AMD hardware comes from the ``/dev/kfd`` and ``/dev/dri`` device mounts plus the ``video`` group, which the orchestrator adds. Image-tar load is orchestrator behavior; set ``image_tar`` in the cluster file. Container lifetime ------------------ @@ -386,7 +405,7 @@ The ``lifetime`` key controls when the container is started and stopped relative Registry authentication ----------------------- -``runtime.args.registry`` and top-level ``image_tar`` are rejected in the vLLM file. Set them on the cluster file's ``container`` block if you need a private pull or a ``docker load``. The orchestrator then does the following. ``username`` and ``password_file`` are required; ``server`` defaults to Docker Hub. The password is read from a file on each remote host. There is no inline password key, and the login is kept out of the logs. Login is skipped when ``image_tar`` is set, because a tar load does not pull. +Set ``runtime.args.registry`` in either file, and ``image_tar`` on the cluster file's ``container`` block, if you need a private pull or a ``docker load``. The orchestrator then does the following. ``username`` and ``password_file`` are required; ``server`` defaults to Docker Hub. The password is read from a file on each remote host. There is no inline password key, and the login is kept out of the logs. Login is skipped when ``image_tar`` is set, because a tar load does not pull. Image resolution order at launch: @@ -406,7 +425,7 @@ The variant's ``container`` block is deep-merged **onto** the cluster file's blo .. warning:: - A loaded vLLM ``container`` block always contributes ``lifetime``, ``name``, and ``image``. ``name`` and ``image`` are required, so omitting them fails validation. An empty ``image`` string overwrites a cluster-file image. The orchestrator then logs ``Container image not specified in config`` and ``test_launch_container`` fails because ``setup_containers()`` returned false. Set ``image`` in the vLLM file. The packaged Llama files use ```` there. + A loaded vLLM ``container`` block always contributes ``lifetime``, ``name``, and ``image``. ``name`` and ``image`` are required, so omitting them fails validation. An empty ``image`` string overwrites a cluster-file image. The orchestrator then logs ``Container image not specified in config`` and ``test_launch_container`` fails because ``setup_containers()`` returned false. Set ``image`` in the vLLM file. The packaged files ship ```` there, which fails config load until you replace it. .. _vllm-paths: @@ -775,12 +794,11 @@ Sweep specs are strict at load time regardless of enforcement: - ``kind`` must equal the registry direction, exactly ``min`` or ``max``. - ``value`` must be a finite JSON number. Booleans, strings, null, arrays, objects, NaN, and infinity are rejected. -- Cell-level keys beginning with ``_comment`` or ``_example`` are metadata and - are ignored by metric validation; all other keys must name a registered - metric. +- Cell-level keys beginning with ``_comment`` or ``_example`` are comments and + are ignored; all other keys must name a registered metric. - Prefixed names (``client.*``, ``gpu.*``, ``prom.*``), unknown names, legacy kinds (including ``min_tok_s`` and ``max_ms``), references, tolerances, - units, ``info``, and extra fields are rejected. + units, ``info``, ``optional``, and extra fields are rejected. ``min`` fails below its value and passes at or above it. ``max`` fails above its value and passes at or below it. A gated actual that is missing, null, @@ -1107,13 +1125,15 @@ Troubleshooting * - ``duplicate task id(s)`` - Two ``accuracy.tasks`` entries share an ``id``. * - ``unknown vLLM threshold metric ''`` - - The name is not one of the 56 registry metrics. ``_comment`` and ``_example`` keys are ignored. + - The name is not one of the 56 registry metrics. Names beginning with ``_comment`` or ``_example`` are ignored. * - `` threshold kind must be '', got ''`` - ``kind`` does not match the registry direction. Use the required ``min`` or ``max`` value shown in the message. * - ``: actual must be a finite built-in int or float, got `` - The gated datasource did not produce a valid value. Inspect the benchmark artifact, GPU telemetry, or server metrics for that cell; percentile collection is harness-owned. * - Unknown key under ``container`` or ``runtime.args`` - - ``ContainerConfig`` and ``RuntimeArgs`` forbid extra keys. ``env`` belongs on ``container``. ``image_tar``, ``registry``, ``shm_size``, and ``cap_add`` belong on the cluster file. See :doc:`/reference/cluster/cluster-file`. + - ``container`` and ``runtime.args`` forbid keys they do not define. ``env`` belongs on ``container``, and ``image_tar`` on the cluster file. See :doc:`/reference/cluster/cluster-file`. + * - ``: still a placeholder; replace it with the value for your setup`` + - A template value was not replaced. The error lists every such field. * - ``runtime.name`` is not ``docker`` - The vLLM schema accepts only ``docker``.