Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,4 @@ jobs:
- name: Install
run: pip install -e packages/python
- name: Validate all registry configs
run: |
for f in registry/**/*.yaml; do
python -m servingcard validate "$f"
done
run: python -m servingcard validate registry/
49 changes: 40 additions & 9 deletions packages/python/servingcard/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,17 +218,48 @@ def _fetch_remote_card(url: str) -> ServingCard:

@app.command()
def validate(
path: Path = typer.Argument(..., help="Path to a servingcard YAML file"),
paths: list[Path] = typer.Argument(
...,
help="One or more servingcard YAML files (or directories to recurse)",
exists=True,
file_okay=True,
dir_okay=True,
readable=True,
),
) -> None:
"""Validate a servingcard YAML file."""
errors = validate_card(path)
if errors:
typer.echo(f"INVALID: {path}")
for err in errors:
typer.echo(f" - {err}")
"""Validate one or more servingcard YAML files.

Accepts files or directories. Directories are recursed for *.yaml and
*.yml. Exits 1 on any invalid file. Always loud about what was found —
silent success on a typo'd path is the failure mode this command is
built to prevent (CI used to silently iterate a literal glob).
"""
targets: list[Path] = []
for p in paths:
if p.is_dir():
targets.extend(sorted(p.rglob("*.yaml")))
targets.extend(sorted(p.rglob("*.yml")))
else:
targets.append(p)

if not targets:
typer.echo("ERROR: no .yaml or .yml files found in given paths", err=True)
raise typer.Exit(code=2)

invalid = 0
for target in targets:
errors = validate_card(target)
if errors:
invalid += 1
typer.echo(f"INVALID: {target}")
for err in errors:
typer.echo(f" - {err}")
else:
typer.echo(f"VALID: {target}")

typer.echo(f"\n{len(targets) - invalid}/{len(targets)} valid")
if invalid:
raise typer.Exit(code=1)
else:
typer.echo(f"VALID: {path}")


# ---------------------------------------------------------------------------
Expand Down
38 changes: 38 additions & 0 deletions packages/python/tests/test_validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@
from pathlib import Path

import yaml
from typer.testing import CliRunner

from servingcard.cli import app as _app
from servingcard.validate import validate_card

_runner = CliRunner()

# ---------------------------------------------------------------------------
# 1. Valid config returns empty errors list
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -167,3 +171,37 @@ def test_speculative_without_benchmark(tmp_path: Path, minimal_card_dict: dict)
p.write_text(yaml.dump(d))
errors = validate_card(p)
assert any("speculative" in e.lower() for e in errors)


# ---------------------------------------------------------------------------
# Spec 009 follow-up: validate CLI must fail loudly on missing paths and
# accept directories. Regression for the "silent literal-glob" CI failure.
# ---------------------------------------------------------------------------


def test_validate_cli_fails_on_missing_path() -> None:
result = _runner.invoke(_app, ["validate", "/no/such/path/at/all.yaml"])
assert result.exit_code != 0


def test_validate_cli_recurses_directory(tmp_path: Path, tmp_valid_yaml: Path) -> None:
nested = tmp_path / "nested" / "dir"
nested.mkdir(parents=True)
(nested / "card.yaml").write_text(tmp_valid_yaml.read_text())
result = _runner.invoke(_app, ["validate", str(tmp_path)])
assert result.exit_code == 0, result.stdout
assert "VALID" in result.stdout
assert "2/2 valid" in result.stdout


def test_validate_cli_directory_with_no_yaml_errors(tmp_path: Path) -> None:
result = _runner.invoke(_app, ["validate", str(tmp_path)])
assert result.exit_code == 2 # distinct from validation failure (1)


def test_validate_cli_accepts_multiple_paths(tmp_path: Path, tmp_valid_yaml: Path) -> None:
p2 = tmp_path / "second.yaml"
p2.write_text(tmp_valid_yaml.read_text())
result = _runner.invoke(_app, ["validate", str(tmp_valid_yaml), str(p2)])
assert result.exit_code == 0, result.stdout
assert "2/2 valid" in result.stdout
2 changes: 1 addition & 1 deletion registry/deepseek-coder-v2-lite/gb10-fp8-baseline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ huggingface:
method_detail: PawBench full suite (PawStyle scenarios, 2 runs, concurrency 1/2/4/8)
serving:
engine_args:
model: /home/vvladescu/models/deepseek-coder-v2-lite-fp8
model: ${MODEL_DIR}/deepseek-coder-v2-lite-fp8
quantization: fp8
gpu_memory_utilization: 0.8
max_model_len: 65536
Expand Down
2 changes: 1 addition & 1 deletion registry/devstral-small-24b/gb10-baseline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ huggingface:
method_detail: PawBench full suite (PawStyle scenarios, 2 runs, concurrency 1/2/4/8)
serving:
engine_args:
model: /home/vvladescu/models/devstral-small-24b
model: ${MODEL_DIR}/devstral-small-24b
quantization: bf16
gpu_memory_utilization: 0.8
max_model_len: 65536
Expand Down
2 changes: 1 addition & 1 deletion registry/qwen3-coder/gb10-fp8-baseline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ huggingface:
url: https://huggingface.co/Qwen/Qwen3-Coder-Next
serving:
engine_args:
model: /home/vvladescu/models/qwen3-coder-next-fp8
model: ${MODEL_DIR}/qwen3-coder-next-fp8
quantization: fp8
gpu_memory_utilization: 0.8
max_model_len: 131072
Expand Down
2 changes: 1 addition & 1 deletion registry/qwen3-coder/gb10-fp8-eagle3-spec3.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ huggingface:
url: https://huggingface.co/Qwen/Qwen3-Coder-Next
serving:
engine_args:
model: /home/vvladescu/models/qwen3-coder-next-fp8
model: ${MODEL_DIR}/qwen3-coder-next-fp8
quantization: fp8
gpu_memory_utilization: 0.8
max_model_len: 131072
Expand Down
Loading