diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4d4792..d441be7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,4 @@ jobs: - name: Install run: pip install -e packages/python - name: Validate all registry configs - run: | - for f in registry/**/*.yaml; do - python -m servingcard validate "$f" - done + run: python -m servingcard validate registry/ diff --git a/packages/python/servingcard/cli.py b/packages/python/servingcard/cli.py index 72a9f86..903b8f9 100644 --- a/packages/python/servingcard/cli.py +++ b/packages/python/servingcard/cli.py @@ -218,17 +218,48 @@ def _fetch_remote_card(url: str) -> ServingCard: @app.command() def validate( - path: Path = typer.Argument(..., help="Path to a servingcard YAML file"), + paths: list[Path] = typer.Argument( + ..., + help="One or more servingcard YAML files (or directories to recurse)", + exists=True, + file_okay=True, + dir_okay=True, + readable=True, + ), ) -> None: - """Validate a servingcard YAML file.""" - errors = validate_card(path) - if errors: - typer.echo(f"INVALID: {path}") - for err in errors: - typer.echo(f" - {err}") + """Validate one or more servingcard YAML files. + + Accepts files or directories. Directories are recursed for *.yaml and + *.yml. Exits 1 on any invalid file. Always loud about what was found — + silent success on a typo'd path is the failure mode this command is + built to prevent (CI used to silently iterate a literal glob). + """ + targets: list[Path] = [] + for p in paths: + if p.is_dir(): + targets.extend(sorted(p.rglob("*.yaml"))) + targets.extend(sorted(p.rglob("*.yml"))) + else: + targets.append(p) + + if not targets: + typer.echo("ERROR: no .yaml or .yml files found in given paths", err=True) + raise typer.Exit(code=2) + + invalid = 0 + for target in targets: + errors = validate_card(target) + if errors: + invalid += 1 + typer.echo(f"INVALID: {target}") + for err in errors: + typer.echo(f" - {err}") + else: + typer.echo(f"VALID: {target}") + + typer.echo(f"\n{len(targets) - invalid}/{len(targets)} valid") + if invalid: raise typer.Exit(code=1) - else: - typer.echo(f"VALID: {path}") # --------------------------------------------------------------------------- diff --git a/packages/python/tests/test_validate.py b/packages/python/tests/test_validate.py index 30a6411..1e0bc61 100644 --- a/packages/python/tests/test_validate.py +++ b/packages/python/tests/test_validate.py @@ -5,9 +5,13 @@ from pathlib import Path import yaml +from typer.testing import CliRunner +from servingcard.cli import app as _app from servingcard.validate import validate_card +_runner = CliRunner() + # --------------------------------------------------------------------------- # 1. Valid config returns empty errors list # --------------------------------------------------------------------------- @@ -167,3 +171,37 @@ def test_speculative_without_benchmark(tmp_path: Path, minimal_card_dict: dict) p.write_text(yaml.dump(d)) errors = validate_card(p) assert any("speculative" in e.lower() for e in errors) + + +# --------------------------------------------------------------------------- +# Spec 009 follow-up: validate CLI must fail loudly on missing paths and +# accept directories. Regression for the "silent literal-glob" CI failure. +# --------------------------------------------------------------------------- + + +def test_validate_cli_fails_on_missing_path() -> None: + result = _runner.invoke(_app, ["validate", "/no/such/path/at/all.yaml"]) + assert result.exit_code != 0 + + +def test_validate_cli_recurses_directory(tmp_path: Path, tmp_valid_yaml: Path) -> None: + nested = tmp_path / "nested" / "dir" + nested.mkdir(parents=True) + (nested / "card.yaml").write_text(tmp_valid_yaml.read_text()) + result = _runner.invoke(_app, ["validate", str(tmp_path)]) + assert result.exit_code == 0, result.stdout + assert "VALID" in result.stdout + assert "2/2 valid" in result.stdout + + +def test_validate_cli_directory_with_no_yaml_errors(tmp_path: Path) -> None: + result = _runner.invoke(_app, ["validate", str(tmp_path)]) + assert result.exit_code == 2 # distinct from validation failure (1) + + +def test_validate_cli_accepts_multiple_paths(tmp_path: Path, tmp_valid_yaml: Path) -> None: + p2 = tmp_path / "second.yaml" + p2.write_text(tmp_valid_yaml.read_text()) + result = _runner.invoke(_app, ["validate", str(tmp_valid_yaml), str(p2)]) + assert result.exit_code == 0, result.stdout + assert "2/2 valid" in result.stdout diff --git a/registry/deepseek-coder-v2-lite/gb10-fp8-baseline.yaml b/registry/deepseek-coder-v2-lite/gb10-fp8-baseline.yaml index f27da2c..dc957a5 100644 --- a/registry/deepseek-coder-v2-lite/gb10-fp8-baseline.yaml +++ b/registry/deepseek-coder-v2-lite/gb10-fp8-baseline.yaml @@ -15,7 +15,7 @@ huggingface: method_detail: PawBench full suite (PawStyle scenarios, 2 runs, concurrency 1/2/4/8) serving: engine_args: - model: /home/vvladescu/models/deepseek-coder-v2-lite-fp8 + model: ${MODEL_DIR}/deepseek-coder-v2-lite-fp8 quantization: fp8 gpu_memory_utilization: 0.8 max_model_len: 65536 diff --git a/registry/devstral-small-24b/gb10-baseline.yaml b/registry/devstral-small-24b/gb10-baseline.yaml index c233742..40045dc 100644 --- a/registry/devstral-small-24b/gb10-baseline.yaml +++ b/registry/devstral-small-24b/gb10-baseline.yaml @@ -15,7 +15,7 @@ huggingface: method_detail: PawBench full suite (PawStyle scenarios, 2 runs, concurrency 1/2/4/8) serving: engine_args: - model: /home/vvladescu/models/devstral-small-24b + model: ${MODEL_DIR}/devstral-small-24b quantization: bf16 gpu_memory_utilization: 0.8 max_model_len: 65536 diff --git a/registry/qwen3-coder/gb10-fp8-baseline.yaml b/registry/qwen3-coder/gb10-fp8-baseline.yaml index 94f2eef..81a7ff9 100644 --- a/registry/qwen3-coder/gb10-fp8-baseline.yaml +++ b/registry/qwen3-coder/gb10-fp8-baseline.yaml @@ -16,7 +16,7 @@ huggingface: url: https://huggingface.co/Qwen/Qwen3-Coder-Next serving: engine_args: - model: /home/vvladescu/models/qwen3-coder-next-fp8 + model: ${MODEL_DIR}/qwen3-coder-next-fp8 quantization: fp8 gpu_memory_utilization: 0.8 max_model_len: 131072 diff --git a/registry/qwen3-coder/gb10-fp8-eagle3-spec3.yaml b/registry/qwen3-coder/gb10-fp8-eagle3-spec3.yaml index fb2719b..2576241 100644 --- a/registry/qwen3-coder/gb10-fp8-eagle3-spec3.yaml +++ b/registry/qwen3-coder/gb10-fp8-eagle3-spec3.yaml @@ -16,7 +16,7 @@ huggingface: url: https://huggingface.co/Qwen/Qwen3-Coder-Next serving: engine_args: - model: /home/vvladescu/models/qwen3-coder-next-fp8 + model: ${MODEL_DIR}/qwen3-coder-next-fp8 quantization: fp8 gpu_memory_utilization: 0.8 max_model_len: 131072