From c0be98ef261afee4b01d1c1996e797ef913be949 Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Thu, 29 Oct 2026 15:48:21 -0700 Subject: [PATCH 1/4] feat(whisper): integrate the MLPerf speech2text workload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmarks whisper-large-v3 over LibriSpeech against an OpenAI-compatible transcription endpoint and scores Word Error Rate, following the framework's existing extension points. The one framework change: `HttpRequestTemplate` folded `Content-Type: application/json` into its two pre-rendered prefixes, and `extra_headers` can only append (a per-request header would duplicate it). The media type now comes from `HttpRequestAdapter.CONTENT_TYPE`, which the worker passes to the template once at setup — still pre-encoded, no hot-path cost. `/v1/audio/transcriptions` needs this because the OpenAI Audio API is a `multipart/form-data` file upload with no JSON variant, and it is the only vLLM route that serves `WhisperForConditionalGeneration`. - `api_type: openai_transcriptions` -> `openai/transcriptions_adapter.py`. The dataset row carries the clip's *path*; the worker reads the bytes in `encode_query`, so the corpus never crosses the issue thread or the ZMQ frame. Non-streaming (OpenAI does not support `stream` for Whisper), served by a shared `NonStreamingAccumulator`. - `whisper_librispeech` consumes the MLCommons-prepared `dev-all-repack.json` manifest. The ~30 s repackaging stays upstream: different clip boundaries change the WER, so a client-side rebuild would not be comparable to the reference. - `eval_method: wer` reports `100 - WER%` corpus-wide. `evaluation/wer.py` ports the benchmark's concatenation-aware metric (a generic WER library does not reproduce the reference number, which credits joined words across spans of up to three) and pairs it with `transformers`' `EnglishTextNormalizer`. Verified against upstream: 20k randomized alignment trials and 3k normalizer cases, zero divergence. No new dependencies. - A tokenizer with no chat template (Whisper and other ASR tokenizers) no longer logs an ERROR stack trace for its zero chat-template baseline. - The corpus is fetched on first run: `download_from_r2` pulls the published MLCommons artifact (pinned downloader commit, host allowlist, MD5-verified) into the dataset cache, so `generate_params.manifest` is now an override for a local copy rather than a prerequisite. - Manifest `fname` entries keep the upstream pipeline's `./data/` prefix, which the hosted artifact's layout does not reproduce. The loader probes the first clip and drops leading components that are absent under the root, so both layouts resolve with no configuration. Reference accuracy is 97.9329% (WER 2.0671%); submissions need 99% of that. See examples/12_Whisper_Example/. Co-Authored-By: Claude Opus 5 --- .gitignore | 3 + AGENTS.md | 7 +- ATTRIBUTION | 12 + docs/ENDPOINT_CLIENT.md | 19 +- docs/evaluation/DESIGN.md | 1 + examples/12_Whisper_Example/README.md | 112 ++++++ .../12_Whisper_Example/offline_whisper.yaml | 75 ++++ .../metrics_aggregator/token_metrics.py | 20 +- src/inference_endpoint/commands/probe.py | 12 +- src/inference_endpoint/config/schema.py | 4 +- .../templates/concurrency_template_full.yaml | 4 +- .../templates/offline_template_full.yaml | 4 +- .../templates/online_template_full.yaml | 4 +- src/inference_endpoint/core/types.py | 3 + .../dataset_manager/__init__.py | 2 + .../dataset_manager/dataset.py | 62 ++++ .../whisper_librispeech/__init__.py | 281 +++++++++++++++ .../endpoint_client/accumulator_protocol.py | 26 ++ .../endpoint_client/adapter_protocol.py | 8 +- .../endpoint_client/config.py | 2 + .../endpoint_client/http.py | 39 +- .../endpoint_client/worker.py | 5 +- src/inference_endpoint/evaluation/scoring.py | 113 ++++++ src/inference_endpoint/evaluation/wer.py | 143 ++++++++ .../openai/transcriptions_adapter.py | 166 +++++++++ src/inference_endpoint/openai/types.py | 19 + tests/integration/openai/__init__.py | 14 + tests/integration/openai/conftest.py | 109 ++++++ .../openai/test_transcriptions_adapter.py | 274 ++++++++++++++ .../metrics_aggregator/test_token_metrics.py | 32 ++ .../test_whisper_librispeech_dataset.py | 336 ++++++++++++++++++ tests/unit/endpoint_client/test_http.py | 68 ++++ tests/unit/evaluation/test_wer.py | 130 +++++++ tests/unit/evaluation/test_wer_scorer.py | 225 ++++++++++++ .../openai/test_transcriptions_adapter.py | 180 ++++++++++ .../test_transcriptions_registration.py | 75 ++++ 36 files changed, 2551 insertions(+), 38 deletions(-) create mode 100644 examples/12_Whisper_Example/README.md create mode 100644 examples/12_Whisper_Example/offline_whisper.yaml create mode 100644 src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py create mode 100644 src/inference_endpoint/evaluation/wer.py create mode 100644 src/inference_endpoint/openai/transcriptions_adapter.py create mode 100644 tests/integration/openai/__init__.py create mode 100644 tests/integration/openai/conftest.py create mode 100644 tests/integration/openai/test_transcriptions_adapter.py create mode 100644 tests/unit/dataset_manager/test_whisper_librispeech_dataset.py create mode 100644 tests/unit/evaluation/test_wer.py create mode 100644 tests/unit/evaluation/test_wer_scorer.py create mode 100644 tests/unit/openai/test_transcriptions_adapter.py create mode 100644 tests/unit/openai/test_transcriptions_registration.py diff --git a/.gitignore b/.gitignore index a77e2a5a5..0c5143875 100644 --- a/.gitignore +++ b/.gitignore @@ -213,3 +213,6 @@ CLAUDE.local.md # Generated dataset cache (created by Dataset.get_dataloader()) dataset_cache/ .agents/devdocs/ + +# Run artifacts: the example configs' default report_dir +logs/ diff --git a/AGENTS.md b/AGENTS.md index e266776dd..e2d533d2d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -101,6 +101,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **DeepSeek-R1 (MLPerf)** | `src/inference_endpoint/evaluation/scoring.py` (`LegacyMLPerfDeepSeekR1Scorer`), `examples/07_DeepSeekR1_Example/` | MLPerf DeepSeek-R1 accuracy. TensorRT-LLM is OpenAI-compatible, so it is served via `api_type: openai` / `openai_completions` (no dedicated trtllm adapter). The combined multi-subset eval (`math500`/`aime`/`gpqa`/`mmlu_pro`/`livecodebench`) is the official MLCommons `eval_accuracy.py`, run out-of-process via `uv run --project` against the isolated subproject at `src/inference_endpoint/evaluation/legacy_mlperf_deepseek_r1/` (a uv subproject excluded from the parent wheel; mirrors the VBench pattern). The example feeds the exact MLPerf prompt via pre-tokenized `input_tokens` to `/v1/completions`. | | **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | | **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | +| **Whisper (MLPerf speech2text)** | `src/inference_endpoint/openai/transcriptions_adapter.py`, `dataset_manager/predefined/whisper_librispeech/`, `evaluation/wer.py` | MLPerf speech2text (whisper-large-v3 / LibriSpeech). `api_type: openai_transcriptions` posts a `multipart/form-data` audio upload to `/v1/audio/transcriptions` — the only route that serves Whisper on vLLM. The body is not JSON, so the adapter declares `HttpRequestAdapter.CONTENT_TYPE` and the worker folds it into the pre-encoded request prefix once at setup (`endpoint_client/http.py`); the multipart boundary is therefore a module constant. `Query.data` carries the clip's _path_ and the worker reads the bytes in `encode_query`, so the corpus never crosses the ZMQ frame. Non-streaming (`NonStreamingAccumulator`). The `whisper_librispeech` dataset downloads the MLCommons-prepared `dev-all-repack.json` corpus via `download_from_r2` (`generate_params.manifest` overrides); the ~30 s repackaging stays upstream because different clip boundaries change the WER. `WERScorer` (`eval_method: wer`) reports `100 - WER%` corpus-wide using the ported MLPerf metric in `evaluation/wer.py` plus `transformers`' `EnglishTextNormalizer`. Needs `metrics_isl: false` (no text prompt) and `warmup.salt: false`. See examples/12_Whisper_Example/. | | **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | | **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | | **Validation** | `src/inference_endpoint/validation/`, `docs/validation/` | Cohort checker for submission uploads. A cohort is a directory of YAML checks (four scope files + `valid_refs.yaml` + a `cohort.yaml` naming the packaged ruleset); `python -m inference_endpoint.validation COHORT SUBMISSION` runs them against an upload root (`results///r/`) and reports holds/failures per check. Exit codes: 0 holds, 1 blocking failure, 2 refusal. See docs/validation/README.md. | @@ -399,7 +400,7 @@ src/inference_endpoint/ │ ├── dataset.py # Dataset base class, DatasetFormat enum │ ├── factory.py # Dataset factory │ ├── transforms.py # ColumnRemap and other transforms -│ └── predefined/ # Built-in datasets (aime25, cnndailymail, gpqa, etc.) +│ └── predefined/ # Built-in datasets (aime25, cnndailymail, gpqa, whisper_librispeech, etc.) ├── metrics/ │ ├── report.py # Report.from_snapshot(MetricsSnapshot); display + JSON serialization │ ├── metric.py # Metric types (Throughput, etc.) @@ -415,10 +416,11 @@ src/inference_endpoint/ │ ├── rulesets/mlcommons/ # MLCommons-specific rules, datasets, models │ └── templates/ # YAML config templates (_template.yaml minimal, _template_full.yaml all defaults) ├── openai/ # OpenAI-compatible API types and adapters -│ ├── types.py # OpenAI response types (chat + text completion) +│ ├── types.py # OpenAI response types (chat + text completion + transcription) │ ├── openai_adapter.py # Chat completions adapter (/v1/chat/completions) │ ├── openai_msgspec_adapter.py # msgspec-based chat completions adapter (fast path) │ ├── completions_adapter.py # Text completions adapter (/v1/completions, pre-tokenized input) +│ ├── transcriptions_adapter.py # Audio transcription adapter (/v1/audio/transcriptions, multipart upload) │ ├── accumulator.py # Streaming response accumulator (shared by chat + completions) │ └── harmony.py # openai_harmony integration ├── sglang/ # SGLang API adapter (/generate with input_ids) @@ -427,6 +429,7 @@ src/inference_endpoint/ │ ├── types.py # Pydantic: VideoPathRequest, VideoPathResponse, VideoPayloadResponse │ └── adapter.py # VideoGenAdapter (HttpRequestAdapter) + VideoGenAccumulator (no-op) ├── evaluation/ # Accuracy evaluation (extractor, scoring, livecodebench) +│ ├── wer.py # MLPerf speech2text Word Error Rate (ported metric; used by WERScorer) │ └── swebench_service/ # Isolated uv service for Docker/Pyxis SWE-bench runs ├── compliance/ # Submission compliance checks (config-lock, accuracy gate, run validity) │ ├── __init__.py diff --git a/ATTRIBUTION b/ATTRIBUTION index 5dc174ffa..caf9ed291 100644 --- a/ATTRIBUTION +++ b/ATTRIBUTION @@ -34,4 +34,16 @@ This section lists third-party assets and their attributions. parser, and model extension are derived from mini-swe-agent v2.3.0. The full upstream notice is retained in the swebench_service package. +### MLPerf Inference speech2text + +- Source: https://github.com/mlcommons/inference/tree/master/speech2text +- License: Apache-2.0 +- Copyright: Copyright 2025 The MLPerf Authors; Copyright (c) 2019 NVIDIA + CORPORATION and Myrtle Software +- Description: `src/inference_endpoint/evaluation/wer.py` ports the benchmark's + Word Error Rate definition — `helpers.compute_wer_with_concatenation`, + `legacy_helpers.__levenshtein`, and the character-label projection applied by + `accuracy_eval.py` — so the reported number is comparable to the MLPerf + reference. + For a complete list of dependencies and their licenses, see the requirements files in the `requirements/` directory. diff --git a/docs/ENDPOINT_CLIENT.md b/docs/ENDPOINT_CLIENT.md index 72c015d5d..1089c64f7 100644 --- a/docs/ENDPOINT_CLIENT.md +++ b/docs/ENDPOINT_CLIENT.md @@ -280,7 +280,10 @@ model validator. ```python class APIType(str, Enum): OPENAI = "openai" + OPENAI_COMPLETIONS = "openai_completions" + OPENAI_TRANSCRIPTIONS = "openai_transcriptions" SGLANG = "sglang" + VIDEOGEN = "videogen" class HTTPClientConfig(BaseModel): # Target endpoint URLs; workers assigned round-robin at spawn time @@ -960,19 +963,19 @@ concatenating them with a single `b"".join()`. | Method | Async | Description | | ------------------------------------------------ | ----- | -------------------------------------------------------------------------- | -| `from_url(host, port, path)` | No | Classmethod: create template with pre-encoded request line + Host header | +| `from_url(host, port, path, content_type?)` | No | Classmethod: create template with pre-encoded request line + Host header | | `cache_headers(headers)` | No | Pre-encode headers (e.g. Authorization) into bytes; call once during setup | | `build_request(body, streaming, extra_headers?)` | No | Build complete HTTP/1.1 request bytes; fast path when no extra headers | **Request byte segments:** -| Segment | When Built | Description | -| -------------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Static prefix | Once (init) | `POST /v1/chat/completions HTTP/1.1\r\nHost: localhost:8000\r\n` — pre-encoded at construction via `from_url()` | -| Cached headers | Once (setup) | `Authorization: Bearer sk-...\r\n` — pre-encoded via `cache_headers()`; reused across all requests | -| Content-Type | Once (class) | Two class-level `bytes` constants: `HEADERS_STREAMING` and `HEADERS_NON_STREAMING`. The streaming/non-streaming branch selects between them — no string encoding at request time | -| Content-Length | Per-request | `Content-Length: 1234\r\n\r\n` — the only header that changes per request; built from a single f-string and ASCII-encoded | -| Body | Per-request | JSON bytes produced by the adapter's `encode_query()` ([§9.1](#91-httprequestadapter)) | +| Segment | When Built | Description | +| -------------- | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Static prefix | Once (init) | `POST /v1/chat/completions HTTP/1.1\r\nHost: localhost:8000\r\n` — pre-encoded at construction via `from_url()` | +| Cached headers | Once (setup) | `Authorization: Bearer sk-...\r\n` — pre-encoded via `cache_headers()`; reused across all requests | +| Content-Type | Once (setup) | From the adapter's `CONTENT_TYPE` (default `application/json`; `openai_transcriptions` declares a `multipart/form-data` boundary). Folded into two prefixes with the matching `Accept`, one per streaming branch — no string encoding at request time | +| Content-Length | Per-request | `Content-Length: 1234\r\n\r\n` — the only header that changes per request; built from a single f-string and ASCII-encoded | +| Body | Per-request | Bytes produced by the adapter's `encode_query()` ([§9.1](#91-httprequestadapter)), encoded in the adapter's `CONTENT_TYPE` | ![build_request() fast/slow path](res/endpoint_client/11_build_request_flow.png) diff --git a/docs/evaluation/DESIGN.md b/docs/evaluation/DESIGN.md index 7e62c4ee7..6f9113321 100644 --- a/docs/evaluation/DESIGN.md +++ b/docs/evaluation/DESIGN.md @@ -82,6 +82,7 @@ The scorer registry in `evaluation/scoring.py` currently includes: | `vbench` | VBench video-generation accuracy (WAN 2.2 T2V) | | `bfcl_v4` | BFCL v4 function-calling accuracy | | `legacy_mlperf_deepseek_r1` | MLPerf DeepSeek-R1 combined multi-subset accuracy | +| `wer` | MLPerf speech2text Word Error Rate (Whisper / LibriSpeech) | The registry (`Scorer.PREDEFINED`) is auto-populated from `Scorer` subclasses via `__init_subclass__`. diff --git a/examples/12_Whisper_Example/README.md b/examples/12_Whisper_Example/README.md new file mode 100644 index 000000000..fc80dfd60 --- /dev/null +++ b/examples/12_Whisper_Example/README.md @@ -0,0 +1,112 @@ +# MLPerf speech2text — whisper-large-v3 on LibriSpeech + +Benchmarks an OpenAI-compatible transcription endpoint +(`POST /v1/audio/transcriptions`) against the MLPerf +[speech2text](https://github.com/mlcommons/inference/tree/master/speech2text) +workload and scores Word Error Rate. + +| | | +| ---------------- | --------------------------------------------------------------------------- | +| Model | `openai/whisper-large-v3` | +| Dataset | LibriSpeech dev-clean + dev-other, repackaged to ~30 s clips (1633 samples) | +| Metric | WER (reported as `100 - WER%`) | +| Reference | WER 2.0671% → accuracy **97.9329%** | +| Submission floor | 99% of reference → **96.9536** | +| Scenario | Offline (`max_throughput`) | + +## 1. Serve the model + +```bash +vllm serve openai/whisper-large-v3 --port 8000 +``` + +Whisper is an encoder-decoder ASR model, so it is served only on +`/v1/audio/transcriptions` — not `/v1/chat/completions`. + +## 2. The dataset + +Nothing to do: the first run downloads the MLCommons-prepared corpus into +`dataset_cache/whisper_librispeech/source/` (several GB of audio) and caches the +parsed rows as a parquet beside it. The ~30 s repackaging is part of the +benchmark definition, so the prepared artifact is fetched rather than rebuilt +here: different clip boundaries would produce a WER that is not comparable to +the reference. + +``` +/dev-all-repack.json # manifest: files[0].fname, original_duration, transcript +/dev-all-repack/*.wav # 16 kHz mono clips +``` + +The manifest's `fname` entries keep the upstream pipeline's `./data/` prefix, +which the hosted artifact does not reproduce. The loader probes the first clip +and drops any leading components that aren't present under the root, so both +layouts resolve with no configuration. + +To use a copy you already have, set `datasets[*].generate_params.manifest` or +`$WHISPER_LIBRISPEECH_MANIFEST`; either skips the download. +`generate_params.data_dir` overrides the root that `fname` resolves against; it +defaults to the manifest's own directory. + +Keep the WAVs on **fast local storage**: the client reads each file inside the +measured request, so a network mount shows up as endpoint latency. + +## 3. Run + +Edit `endpoint_config.endpoints` first. The config issues the full 1633-sample +corpus in both the performance and accuracy phases, which is what a submission +needs. + +```bash +uv run inference-endpoint benchmark from-config \ + --config examples/12_Whisper_Example/offline_whisper.yaml +``` + +To check the wiring first, set `runtime.n_samples_to_issue` and both `samples` +values to 8. The dataset truncates in manifest order, so the smoke subset is a +prefix of the full corpus — but a partial perf phase is not a reportable result. + +## 4. Read the result + +`/accuracy/accuracy_results.json`: + +```json +{ + "score": 97.93, + "breakdown": { + "wer_percent": 2.07, + "edit_distance": 1241, + "reference_words": 60000, + "empty_responses": 0, + "missing_samples": 0 + } +} +``` + +**Check `missing_samples == 0` before quoting the score.** WER is corpus-level +(total errors ÷ total reference words), so a sample that never completed +contributes neither and silently shrinks the denominator instead of counting +against you. + +A WER far above 2.07% usually points at the dataset rather than the endpoint — +most often a manifest that is not the repackaged `dev-all-repack.json`. + +## How it maps onto the framework + +| Piece | Where | +| ------------------------ | ---------------------------------------------------------------------- | +| Route + multipart upload | `openai/transcriptions_adapter.py` (`api_type: openai_transcriptions`) | +| Dataset | `dataset_manager/predefined/whisper_librispeech/` | +| Metric | `evaluation/wer.py` + `WERScorer` (`eval_method: wer`) | + +Notes on the run config: + +- **Non-streaming.** OpenAI does not support `stream` for Whisper-class models, + so `model_params.streaming` must be `off`. There is no TTFT/TPOT; the metrics + are QPS and per-request latency. +- **`metrics_isl: false`.** Audio rows carry no text prompt, so there is no + input sequence length. One "ISL is unavailable" warning per run is expected. +- **Warmup salt must stay off.** Salting prepends a random hex prefix to a text + prompt; an audio row has none, and enabling it is a hard error. +- **`language: en`** (settable per dataset row) is what makes the server build + Whisper's `<|startoftranscript|><|en|><|transcribe|><|notimestamps|>` prompt, + matching the MLPerf reference. diff --git a/examples/12_Whisper_Example/offline_whisper.yaml b/examples/12_Whisper_Example/offline_whisper.yaml new file mode 100644 index 000000000..607eb74ff --- /dev/null +++ b/examples/12_Whisper_Example/offline_whisper.yaml @@ -0,0 +1,75 @@ +# Offline run for MLPerf speech2text (whisper-large-v3 / LibriSpeech) +# +# Issues the MLCommons-prepared LibriSpeech corpus against an OpenAI-compatible +# transcription endpoint (POST /v1/audio/transcriptions) and scores Word Error +# Rate. Reference accuracy is 97.9329% (WER 2.0671%); a submission needs 99% of +# that, i.e. >= 96.9536. +# +# Prerequisites: +# - A server exposing /v1/audio/transcriptions for whisper-large-v3: +# vllm serve openai/whisper-large-v3 --port 8000 +# +# The dataset needs no setup: the MLCommons-prepared corpus (dev-all-repack.json +# + WAVs) is downloaded into the dataset cache on first run. To use a copy you +# already have, add generate_params.manifest to both dataset entries (or export +# $WHISPER_LIBRISPEECH_MANIFEST). +# +# The audio bytes are read by the worker inside the measured request, so keep +# the WAV directory on fast local storage - a network mount inflates latency. +# +# For a smoke run, set runtime.n_samples_to_issue and both `samples` to 8. + +name: "offline-whisper-librispeech" +version: "1.0" +type: "offline" + +model_params: + name: "openai/whisper-large-v3" + streaming: "off" # /v1/audio/transcriptions has no SSE mode for Whisper + temperature: 0.0 + +datasets: + # setup_benchmark() requires a performance dataset to drive issuance; the + # accuracy entry below reuses the same corpus and scores the transcripts. + - name: whisper_librispeech + type: "performance" + samples: 1633 + - name: whisper_librispeech + type: "accuracy" + samples: 1633 + accuracy_config: + eval_method: "wer" + ground_truth: "transcript" + num_repeats: 1 + +settings: + runtime: + n_samples_to_issue: 1633 + scheduler_random_seed: 42 + dataloader_random_seed: 42 + + load_pattern: + type: "max_throughput" + + client: + num_workers: 8 + + warmup: + # Warmup salt requires text-'prompt' samples; audio rows have none, so + # enabling warmup here needs salt left off. + enabled: false + salt: false + + # Audio rows carry no text prompt, so there is no input sequence length to + # measure; this stops the aggregator registering an ISL series that would + # stay empty. The load generator still logs one "ISL is unavailable" warning + # per run, which is expected here. OSL and latency are unaffected. + metrics_isl: false + +endpoint_config: + endpoints: + - "http://localhost:8000" + api_type: "openai_transcriptions" + api_key: null + +report_dir: logs/whisper_librispeech_offline diff --git a/src/inference_endpoint/async_utils/services/metrics_aggregator/token_metrics.py b/src/inference_endpoint/async_utils/services/metrics_aggregator/token_metrics.py index 881437303..951713ed3 100644 --- a/src/inference_endpoint/async_utils/services/metrics_aggregator/token_metrics.py +++ b/src/inference_endpoint/async_utils/services/metrics_aggregator/token_metrics.py @@ -312,11 +312,21 @@ def _load_tokenizer(self) -> None: except Exception: self._prefix_len = 0 self._baseline = 0 - logger.exception( - "Failed to compute chat-template baseline for %s; tool-call " - "token counts may be over-estimated", - self._tokenizer_name, - ) + if getattr(tok, "chat_template", None) is None: + # A tokenizer with no chat template (e.g. Whisper and other ASR + # tokenizers) has no assistant frame to subtract and never takes + # the messages counting path, so a zero baseline is correct - + # not a failure worth a stack trace on every run. + logger.debug( + "Tokenizer %s has no chat template; baseline is 0", + self._tokenizer_name, + ) + else: + logger.exception( + "Failed to compute chat-template baseline for %s; tool-call " + "token counts may be over-estimated", + self._tokenizer_name, + ) def _setup_shards(self, cores_per_worker: int, n_workers: int) -> None: """Spawn one pinned single-worker process per core block. diff --git a/src/inference_endpoint/commands/probe.py b/src/inference_endpoint/commands/probe.py index 22d5b4d9a..c81401361 100644 --- a/src/inference_endpoint/commands/probe.py +++ b/src/inference_endpoint/commands/probe.py @@ -51,7 +51,8 @@ class ProbeConfig(BaseModel): api_type: Annotated[ APIType, cyclopts.Parameter( - alias="--api-type", help="API type: openai, sglang, or videogen" + alias="--api-type", + help="API type: openai, openai_transcriptions, sglang, or videogen", ), ] = APIType.OPENAI requests: int = Field(10, ge=1) @@ -98,6 +99,15 @@ async def _probe_async(config: ProbeConfig) -> None: "Use a dedicated health check or a benchmark from-config run instead." ) + # Transcription requests are an audio upload keyed on an `audio_file` path, + # so there is no text prompt the probe could send. + if api_type == APIType.OPENAI_TRANSCRIPTIONS: + raise InputValidationError( + "Probe does not support api_type=openai_transcriptions " + "(the request body is an audio upload, not a text prompt). " + "Use a dedicated health check or a benchmark from-config run instead." + ) + # Model: use provided or default to valid OpenAI model name model_name = config.model if not model_name: diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 222d31057..4dabe3113 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -143,6 +143,7 @@ class ScorerMethod(str, Enum): BFCL_V4 = "bfcl_v4" LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" SWE_BENCH = "swe_bench_scorer" + WER = "wer" # Audit configuration; runnable test registry lives in compliance/. @@ -1067,7 +1068,8 @@ class EndpointConfig(BaseModel): api_type: Annotated[ APIType, cyclopts.Parameter( - alias="--api-type", help="API type: openai, sglang, or videogen" + alias="--api-type", + help="API type: openai, openai_transcriptions, sglang, or videogen", ), ] = APIType.OPENAI diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index 7ed842b91..e96238dbc 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -43,7 +43,7 @@ datasets: # Dataset configs system: system_prompt generate_params: null # Dataset-specific parameters passed to the generate() method accuracy_config: # Accuracy evaluation settings - eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer + eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer, wer ground_truth: ground_truth # Ground truth column name extractor: boxed_math_extractor # Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor) num_repeats: 1 # Repeat dataset N times for evaluation @@ -112,7 +112,7 @@ endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key - api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen + api_type: openai # API type: openai, openai_transcriptions, sglang, or videogen | options: openai, openai_completions, openai_transcriptions, sglang, videogen report_dir: null # Report output directory verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 6ed1aebb8..a26cdac0a 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -43,7 +43,7 @@ datasets: # Dataset configs system: system_prompt generate_params: null # Dataset-specific parameters passed to the generate() method accuracy_config: # Accuracy evaluation settings - eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer + eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer, wer ground_truth: ground_truth # Ground truth column name extractor: boxed_math_extractor # Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor) num_repeats: 1 # Repeat dataset N times for evaluation @@ -112,7 +112,7 @@ endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key - api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen + api_type: openai # API type: openai, openai_transcriptions, sglang, or videogen | options: openai, openai_completions, openai_transcriptions, sglang, videogen report_dir: null # Report output directory verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index 36a844354..f46ad9158 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -43,7 +43,7 @@ datasets: # Dataset configs system: system_prompt generate_params: null # Dataset-specific parameters passed to the generate() method accuracy_config: # Accuracy evaluation settings - eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer + eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer, wer ground_truth: ground_truth # Ground truth column name extractor: boxed_math_extractor # Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor) num_repeats: 1 # Repeat dataset N times for evaluation @@ -113,7 +113,7 @@ endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key - api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen + api_type: openai # API type: openai, openai_transcriptions, sglang, or videogen | options: openai, openai_completions, openai_transcriptions, sglang, videogen report_dir: null # Report output directory verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning diff --git a/src/inference_endpoint/core/types.py b/src/inference_endpoint/core/types.py index 03cc29942..62b6b1818 100644 --- a/src/inference_endpoint/core/types.py +++ b/src/inference_endpoint/core/types.py @@ -36,6 +36,7 @@ class APIType(str, Enum): OPENAI = "openai" OPENAI_COMPLETIONS = "openai_completions" + OPENAI_TRANSCRIPTIONS = "openai_transcriptions" SGLANG = "sglang" VIDEOGEN = "videogen" @@ -46,6 +47,8 @@ def default_route(self) -> str: return "v1/chat/completions" case APIType.OPENAI_COMPLETIONS: return "v1/completions" + case APIType.OPENAI_TRANSCRIPTIONS: + return "v1/audio/transcriptions" case APIType.SGLANG: return "generate" case APIType.VIDEOGEN: diff --git a/src/inference_endpoint/dataset_manager/__init__.py b/src/inference_endpoint/dataset_manager/__init__.py index 98c96f868..ff3db03a0 100644 --- a/src/inference_endpoint/dataset_manager/__init__.py +++ b/src/inference_endpoint/dataset_manager/__init__.py @@ -39,6 +39,7 @@ ShopifyProductCatalogue8k, ) from .predefined.swe_bench import SWEBench +from .predefined.whisper_librispeech import WhisperLibriSpeech from .transforms import ( AddStaticColumns, ColumnFilter, @@ -72,5 +73,6 @@ "ShopifyProductCatalogue", "ShopifyProductCatalogue8k", "SWEBench", + "WhisperLibriSpeech", "AgenticInferenceDataset", ] diff --git a/src/inference_endpoint/dataset_manager/dataset.py b/src/inference_endpoint/dataset_manager/dataset.py index 4cff1fc3b..dcd420a15 100644 --- a/src/inference_endpoint/dataset_manager/dataset.py +++ b/src/inference_endpoint/dataset_manager/dataset.py @@ -17,6 +17,7 @@ import inspect import os import random +import subprocess import warnings from abc import ABC from enum import Enum @@ -24,9 +25,11 @@ from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar +from urllib.parse import urlparse import numpy as np import pandas as pd +import requests from datasets import load_dataset, load_from_disk from ..config.schema import APIType, ModelParams @@ -258,6 +261,65 @@ def load_from_huggingface( return ds[split].to_pandas() +#: Pinned mlcommons/r2-downloader commit used by download_from_r2. +R2_DOWNLOADER_COMMIT = "27da4421877f2831eeb615b43ee5098c4b70be7e" + +#: Dataset URIs are handed to a downloaded shell script, so the host is +#: restricted to MLCommons R2 storage. This stops a mis-set dataset URI from +#: pointing the downloader at an attacker-controlled location. +R2_ALLOWED_HOST = "mlcommons-storage.org" + + +def download_from_r2(uri: str, download_dir: Path) -> None: + """Download an MLCommons-hosted dataset into ``download_dir``. + + Fetches the pinned ``mlc-r2-downloader.sh`` and runs it against the + dataset's ``.uri`` metadata. The script MD5-verifies every file it pulls and + fails its exit code on mismatch, so callers only need to locate the + artifact they expect afterwards. + + Raises: + ValueError: ``uri`` is not https on MLCommons R2 storage. + RuntimeError: the downloader exited non-zero. + """ + parsed = urlparse(uri) + host = parsed.hostname or "" + if parsed.scheme != "https" or not ( + host == R2_ALLOWED_HOST or host.endswith("." + R2_ALLOWED_HOST) + ): + raise ValueError( + f"Refusing to download dataset from untrusted URI '{uri}': " + f"expected https on {R2_ALLOWED_HOST}" + ) + + download_dir.mkdir(parents=True, exist_ok=True) + script_path = download_dir / "mlc-r2-downloader.sh" + resp = requests.get( + "https://raw.githubusercontent.com/mlcommons/r2-downloader/" + f"{R2_DOWNLOADER_COMMIT}/mlc-r2-downloader.sh", + timeout=30, + ) + resp.raise_for_status() + script_path.write_bytes(resp.content) + script_path.chmod(0o755) + + logger.info("Downloading %s into %s", uri, download_dir) + try: + result = subprocess.run( + ["bash", str(script_path.resolve()), "-d", str(download_dir), uri], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"R2 downloader failed with code {result.returncode}: {result.stderr}" + ) + finally: + script_path.unlink(missing_ok=True) + + def _check_unsaltable(sample: Any) -> DatasetValidationError.Reason | None: """Return the Reason a sample cannot be salted, or None if it can. diff --git a/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py b/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py new file mode 100644 index 000000000..aa3ce7b80 --- /dev/null +++ b/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py @@ -0,0 +1,281 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from logging import getLogger +from pathlib import Path, PurePosixPath + +import pandas as pd + +from ...dataset import Dataset, download_from_r2 + +logger = getLogger(__name__) + +#: Env var pointing at an already-prepared ``dev-all-repack.json`` manifest, +#: which skips the download. +SOURCE_ENV = "WHISPER_LIBRISPEECH_MANIFEST" + +#: Manifest filename inside the MLCommons-prepared corpus. +MANIFEST_FILENAME = "dev-all-repack.json" + +#: Sample count of the official repackaged dev-all corpus. Logged as a warning +#: on mismatch, not enforced: a truncated manifest is a valid smoke-test input. +MLPERF_SAMPLE_COUNT = 1633 + + +class WhisperLibriSpeech(Dataset, dataset_id="whisper_librispeech"): + """MLPerf speech2text LibriSpeech corpus (MLCommons-prepared manifest). + + Consumes the ``dev-all-repack.json`` manifest and WAV directory produced by + the official MLCommons pipeline, which merges LibriSpeech dev-clean and + dev-other, converts FLAC to 16 kHz WAV, and repackages utterances into + clips approaching 30 s. ``generate()`` downloads it on first use; + ``manifest`` / ``$WHISPER_LIBRISPEECH_MANIFEST`` point at a local copy + instead. + + Building the corpus here is intentionally out of scope. The ~30 s + repackaging is part of the benchmark definition — a client-side + reimplementation would change the clip boundaries and therefore the + reported WER, making the number incomparable to the MLPerf reference. + + Columns: + - ``audio_file`` : absolute path to the clip's WAV, issued by + ``OpenAITranscriptionsAdapter`` (which reads the bytes in the worker). + - ``transcript`` : reference text, read by ``WERScorer`` off the + dataframe as the ground-truth column. + - ``duration`` : clip length in seconds, carried for reporting only. + + One loader serves both phases: the performance phase issues ``audio_file`` + and the accuracy phase scores against ``transcript``. + """ + + COLUMN_NAMES = ["audio_file", "transcript", "duration"] + + #: Published ``.uri`` metadata for the preprocessed speech2text corpus. + #: Override for staging via ``$WHISPER_LIBRISPEECH_DATASET_URI``. + R2_DATASET_URI = ( + "https://inference.mlcommons-storage.org/metadata/whisper-dataset.uri" + ) + R2_DATASET_URI_ENV = "WHISPER_LIBRISPEECH_DATASET_URI" + + @classmethod + def generate( + cls, + datasets_dir: Path, + manifest: str | os.PathLike | None = None, + data_dir: str | os.PathLike | None = None, + max_samples: int | None = None, + seed: int = 0, + force: bool = False, + ) -> pd.DataFrame: + """Build (or load the cached) LibriSpeech benchmark dataframe. + + Args: + datasets_dir: Root cache dir; the parquet is written under + ``/whisper_librispeech/`` and the downloaded + corpus lands in that directory's ``source/`` subdirectory. + manifest: An already-prepared ``dev-all-repack.json``, which skips + the download. Falls back to ``$WHISPER_LIBRISPEECH_MANIFEST``, + then to the MLCommons R2 download. + data_dir: Root the manifest's ``fname`` entries are relative to. + Defaults to the manifest's parent directory. Leading ``fname`` + components that are absent under the root are dropped, so the + hosted artifact's flattened layout resolves without an override. + max_samples: If set, keep the first N rows. Manifest order is the + MLPerf sample order, so truncating (rather than sampling) keeps + a smoke run's indices aligned with the full corpus. + seed: Unused; present for the ``get_dataloader`` contract. + force: Rebuild the cached parquet even if it already exists. An + already-downloaded corpus is reused — this does not re-fetch + several GB of audio. + """ + dst_path = datasets_dir / cls.DATASET_ID / "whisper_librispeech.parquet" + dst_path.parent.mkdir(parents=True, exist_ok=True) + + if dst_path.exists() and not force: + logger.info("LibriSpeech dataset cached at %s; loading.", dst_path) + df = pd.read_parquet(dst_path) + else: + manifest_path = cls._resolve_manifest(manifest, dst_path.parent / "source") + df = cls._load_manifest(manifest_path, data_dir) + df.to_parquet(dst_path, index=False) + logger.info("Wrote %d LibriSpeech rows to %s", len(df), dst_path) + + if len(df) != MLPERF_SAMPLE_COUNT: + logger.warning( + "Manifest has %d samples; the official MLPerf speech2text " + "corpus has %d. A submission run must use the full corpus.", + len(df), + MLPERF_SAMPLE_COUNT, + ) + + if max_samples is not None and max_samples < len(df): + logger.info("Truncating to the first %d of %d rows", max_samples, len(df)) + df = df.head(max_samples).reset_index(drop=True) + return df + + @classmethod + def _resolve_manifest( + cls, + manifest: str | os.PathLike | None, + download_dir: Path, + ) -> Path: + """Locate the manifest, downloading the corpus if nothing local is given. + + Precedence: explicit ``manifest`` / ``$WHISPER_LIBRISPEECH_MANIFEST``, + then an earlier download under ``download_dir``, then a fresh R2 fetch. + """ + resolved = manifest or os.environ.get(SOURCE_ENV) + if resolved: + manifest_path = Path(resolved) + if not manifest_path.exists(): + raise FileNotFoundError( + f"LibriSpeech manifest not found at {manifest_path}" + ) + if manifest_path.suffix != ".json": + raise NotImplementedError( + f"LibriSpeech source {manifest_path} is not a .json manifest. " + f"Only the MLCommons-prepared {MANIFEST_FILENAME} is supported; " + "building the corpus from raw LibriSpeech is out of scope." + ) + return manifest_path + + cached = cls._find_manifest(download_dir) + if cached is not None: + logger.info("Using the downloaded LibriSpeech corpus at %s", cached) + return cached + + logger.info( + "Downloading the MLCommons speech2text corpus (several GB of audio) " + "to %s; pass generate_params.manifest to use a local copy instead.", + download_dir, + ) + download_from_r2( + os.environ.get(cls.R2_DATASET_URI_ENV) or cls.R2_DATASET_URI, download_dir + ) + found = cls._find_manifest(download_dir) + if found is None: + raise FileNotFoundError( + f"R2 download completed but no {MANIFEST_FILENAME} was found " + f"under {download_dir}; the hosted artifact must carry the " + "repackaged manifest." + ) + return found + + @staticmethod + def _find_manifest(download_dir: Path) -> Path | None: + """Find the manifest anywhere under a downloaded corpus, or None. + + The WAV directory sits beside it, so the caller's default ``data_dir`` + (the manifest's parent) is the right root; ``_fname_strip`` absorbs the + relative prefix the artifact's layout does not reproduce. + """ + if not download_dir.exists(): + return None + return next(download_dir.rglob(MANIFEST_FILENAME), None) + + @classmethod + def _load_manifest( + cls, + manifest_path: Path, + data_dir: str | os.PathLike | None, + ) -> pd.DataFrame: + root = Path(data_dir) if data_dir is not None else manifest_path.parent + entries = json.loads(manifest_path.read_text()) + clips = [cls._clip(entry, i) for i, entry in enumerate(entries)] + strip = cls._fname_strip(root, clips[0]["fname"]) if clips else 0 + rows = [ + cls._row(entry, clip, root, strip) + for entry, clip in zip(entries, clips, strict=True) + ] + return pd.DataFrame(rows, columns=cls.COLUMN_NAMES) + + @staticmethod + def _fname_strip(root: Path, fname: str) -> int: + """Leading ``fname`` components to drop so the clip resolves under root. + + The manifest's ``fname`` entries are written relative to the upstream + pipeline's working directory (``./data/dev-all-repack/.wav``), but + the hosted artifact unpacks the WAV directory straight into the corpus + root, so a literal join misses by one component. Probe the first clip + against each suffix of its relative path and reuse the offset for the + whole manifest: the manifest is internally consistent, so one lookup + settles the layout. + + Falls back to 0 (a literal join) when nothing resolves, so a manifest + describing audio that is not mounted here still produces a dataframe. + """ + parts = PurePosixPath(fname).parts + for strip in range(len(parts)): + if root.joinpath(*parts[strip:]).exists(): + if strip: + logger.info( + "Manifest fname prefix %s is not present under %s; " + "resolving clips as %s", + "/".join(parts[:strip]), + root, + root.joinpath(*parts[strip:]), + ) + return strip + logger.warning( + "No audio found for the first manifest clip %s under %s. Clip paths " + "are recorded as-is and will fail at request time; pass " + "generate_params.data_dir if the audio lives elsewhere.", + fname, + root, + ) + return 0 + + @staticmethod + def _clip(entry: dict, index: int) -> dict: + """Select one manifest entry's audio clip. + + The upstream schema nests the clip under ``files`` (a list, to carry + speed-perturbed variants). Like the MLPerf reference's manifest reader, + pick the unperturbed ``speed == 1`` clip; the repackaged dev-all + manifest carries exactly one such entry per sample. + """ + files = entry.get("files") + if not files: + raise ValueError(f"Manifest entry {index} has no 'files' list: {entry}") + if "transcript" not in entry: + raise ValueError( + f"Manifest entry {index} has no inline 'transcript'. The " + "repackaged dev-all manifest carries transcripts inline; a " + "'text_filepath' sidecar manifest is not supported." + ) + clip = next((f for f in files if f.get("speed", 1) == 1), None) + if clip is None: + raise ValueError( + f"Manifest entry {index} has no unperturbed (speed == 1) clip; " + "speed-perturbed-only manifests are not supported." + ) + return clip + + @staticmethod + def _row(entry: dict, clip: dict, root: Path, strip: int) -> dict: + """Flatten one manifest entry into a benchmark row.""" + parts = PurePosixPath(clip["fname"]).parts + return { + "audio_file": str(root.joinpath(*parts[strip:])), + "transcript": entry["transcript"], + "duration": float( + entry.get("original_duration", clip.get("duration", 0.0)) + ), + } + + +__all__ = ["WhisperLibriSpeech"] diff --git a/src/inference_endpoint/endpoint_client/accumulator_protocol.py b/src/inference_endpoint/endpoint_client/accumulator_protocol.py index 5aea629e9..c4974bde2 100644 --- a/src/inference_endpoint/endpoint_client/accumulator_protocol.py +++ b/src/inference_endpoint/endpoint_client/accumulator_protocol.py @@ -70,3 +70,29 @@ def get_final_output(self) -> QueryResult: # pragma: no cover QueryResult with the complete response output. """ ... + + +class NonStreamingAccumulator: + """Accumulator stub for api_types whose endpoint has no SSE mode. + + HTTPClientConfig resolves an adapter/accumulator pair for every api_type, so + a non-streaming endpoint still needs an accumulator. This one should never + be exercised; `get_final_output` raises rather than returning an empty + `QueryResult`, because the worker's SSE path swallows the + `NotImplementedError` from `decode_sse_message` and would otherwise surface + zero-output queries as successful. + """ + + def __init__(self, query_id: str, stream_all_chunks: bool) -> None: + self.query_id = query_id + # stream_all_chunks is intentionally ignored: non-streaming endpoint. + + def add_chunk(self, delta: Any) -> StreamChunk | None: + return None + + def get_final_output(self) -> QueryResult: + raise RuntimeError( + "NonStreamingAccumulator.get_final_output called for query " + f"{self.query_id}: this api_type has no streaming mode - check " + "HTTPClientConfig.streaming and query.data['stream']." + ) diff --git a/src/inference_endpoint/endpoint_client/adapter_protocol.py b/src/inference_endpoint/endpoint_client/adapter_protocol.py index a96faaa0a..92ae4ba24 100644 --- a/src/inference_endpoint/endpoint_client/adapter_protocol.py +++ b/src/inference_endpoint/endpoint_client/adapter_protocol.py @@ -20,7 +20,7 @@ import logging import re from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import msgspec @@ -46,6 +46,12 @@ class HttpRequestAdapter(ABC): # Matches "data: {json content}" and captures the JSON part SSE_DATA_PATTERN: re.Pattern[bytes] = re.compile(rb"data:\s*(\{[^\n]+\})") + # Media type of the bytes returned by encode_query(). The worker folds this + # into the pre-encoded request prefix once at setup, so adapters that send + # something other than JSON (e.g. a multipart audio upload) declare it here + # rather than emitting a per-request header. + CONTENT_TYPE: ClassVar[str] = "application/json" + @classmethod @abstractmethod def dataset_transforms( diff --git a/src/inference_endpoint/endpoint_client/config.py b/src/inference_endpoint/endpoint_client/config.py index b2839996d..8146d5f15 100644 --- a/src/inference_endpoint/endpoint_client/config.py +++ b/src/inference_endpoint/endpoint_client/config.py @@ -64,6 +64,7 @@ def _endpoint_destination(url: str) -> tuple[str | None, int]: ADAPTER_MAP = { APIType.OPENAI: "inference_endpoint.openai.openai_msgspec_adapter.OpenAIMsgspecAdapter", APIType.OPENAI_COMPLETIONS: "inference_endpoint.openai.completions_adapter.OpenAITextCompletionsAdapter", + APIType.OPENAI_TRANSCRIPTIONS: "inference_endpoint.openai.transcriptions_adapter.OpenAITranscriptionsAdapter", APIType.SGLANG: "inference_endpoint.sglang.adapter.SGLangGenerateAdapter", APIType.VIDEOGEN: "inference_endpoint.videogen.adapter.VideoGenAdapter", } @@ -71,6 +72,7 @@ def _endpoint_destination(url: str) -> tuple[str | None, int]: ACCUMULATOR_MAP = { APIType.OPENAI: "inference_endpoint.openai.accumulator.OpenAISSEAccumulator", APIType.OPENAI_COMPLETIONS: "inference_endpoint.openai.accumulator.OpenAISSEAccumulator", + APIType.OPENAI_TRANSCRIPTIONS: "inference_endpoint.endpoint_client.accumulator_protocol.NonStreamingAccumulator", APIType.SGLANG: "inference_endpoint.sglang.accumulator.SGLangSSEAccumulator", APIType.VIDEOGEN: "inference_endpoint.videogen.adapter.VideoGenAccumulator", } diff --git a/src/inference_endpoint/endpoint_client/http.py b/src/inference_endpoint/endpoint_client/http.py index 6d70f93ab..706e345a3 100644 --- a/src/inference_endpoint/endpoint_client/http.py +++ b/src/inference_endpoint/endpoint_client/http.py @@ -699,30 +699,35 @@ class HttpRequestTemplate: Attributes: static_prefix: Pre-merged request line + host header bytes. cached_headers: Pre-encoded headers from cache_headers(), included in every request. + content_type: Request body media type, declared by the adapter. """ __slots__ = ( "static_prefix", "cached_headers", + "content_type", "_prefix_streaming", "_prefix_non_streaming", ) - # Pre-encoded general headers - HEADERS_STREAMING = ( - b"Content-Type: application/json\r\nAccept: text/event-stream\r\n" - ) - HEADERS_NON_STREAMING = ( - b"Content-Type: application/json\r\nAccept: application/json\r\n" - ) + DEFAULT_CONTENT_TYPE = "application/json" - def __init__(self, static_prefix: bytes): + def __init__( + self, static_prefix: bytes, content_type: str = DEFAULT_CONTENT_TYPE + ) -> None: self.static_prefix = static_prefix self.cached_headers = b"" + self.content_type = content_type self._rebuild_prefixes() @classmethod - def from_url(cls, host: str, port: int, path: str = "/") -> HttpRequestTemplate: + def from_url( + cls, + host: str, + port: int, + path: str = "/", + content_type: str = DEFAULT_CONTENT_TYPE, + ) -> HttpRequestTemplate: """ Create an HttpRequestTemplate from URL components. @@ -730,6 +735,8 @@ def from_url(cls, host: str, port: int, path: str = "/") -> HttpRequestTemplate: host: Target hostname port: Target port path: Request path (e.g., "/v1/chat/completions") + content_type: Request body media type (e.g. "multipart/form-data; + boundary=..." for an audio upload) Returns: HttpRequestTemplate ready for building requests @@ -745,13 +752,19 @@ def from_url(cls, host: str, port: int, path: str = "/") -> HttpRequestTemplate: else: host_header = f"Host: {host}:{port}\r\n" - return cls(static_prefix=(request_line + host_header).encode("ascii")) + return cls( + static_prefix=(request_line + host_header).encode("ascii"), + content_type=content_type, + ) def _rebuild_prefixes(self) -> None: """Merge static_prefix + cached_headers + content-type into two ready-to-use prefixes.""" base = self.static_prefix + self.cached_headers - self._prefix_streaming = base + self.HEADERS_STREAMING - self._prefix_non_streaming = base + self.HEADERS_NON_STREAMING + content_type = f"Content-Type: {self.content_type}\r\n".encode("ascii") + self._prefix_streaming = base + content_type + b"Accept: text/event-stream\r\n" + self._prefix_non_streaming = ( + base + content_type + b"Accept: application/json\r\n" + ) def cache_headers(self, headers: dict[str, str]) -> None: """ @@ -782,7 +795,7 @@ def build_request( Build a complete HTTP/1.1 request as raw bytes. Args: - body: Request body bytes (JSON payload) + body: Request body bytes, already encoded in `content_type` streaming: True for SSE streaming, False for buffered response extra_headers: Optional additional headers (e.g., Authorization) diff --git a/src/inference_endpoint/endpoint_client/worker.py b/src/inference_endpoint/endpoint_client/worker.py index 321528baa..51335e357 100644 --- a/src/inference_endpoint/endpoint_client/worker.py +++ b/src/inference_endpoint/endpoint_client/worker.py @@ -198,7 +198,10 @@ async def run(self) -> None: # Initialize HTTP template from URL components self._http_template = HttpRequestTemplate.from_url( - self._host, self._port, self._path + self._host, + self._port, + self._path, + content_type=self._adapter.CONTENT_TYPE, ) if self.http_config.api_key: self._http_template.cache_headers( diff --git a/src/inference_endpoint/evaluation/scoring.py b/src/inference_endpoint/evaluation/scoring.py index 790a17bcb..e15c51d6a 100644 --- a/src/inference_endpoint/evaluation/scoring.py +++ b/src/inference_endpoint/evaluation/scoring.py @@ -36,6 +36,8 @@ import pandas as pd from pydantic import ValidationError from tqdm import tqdm +from transformers import AutoTokenizer +from transformers.models.whisper.english_normalizer import EnglishTextNormalizer try: import websocket @@ -59,6 +61,7 @@ Extractor, PythonCodeExtractor, ) +from .wer import project_to_labels, wer_counts logger = logging.getLogger(__name__) @@ -406,6 +409,116 @@ def score(self) -> tuple[float, int]: return result, 1 +class WERScorer(Scorer, scorer_id="wer"): + """Word Error Rate as defined by the MLPerf speech2text benchmark. + + Reports ``100 - WER%`` so the headline number is higher-is-better and on the + percentage scale the compliance accuracy gate assumes. The MLPerf reference + is WER 2.0671% (accuracy 97.9329%); a submission needs 99% of that. + + Scoring order matches the reference exactly: project both sides through the + character-label inventory, apply Whisper's ``EnglishTextNormalizer``, split + on whitespace, then accumulate the concatenation-aware edit distance. See + :mod:`inference_endpoint.evaluation.wer`. + """ + + REQUIRES_EXTRACTOR = False + + #: Repo (or local dir) supplying ``normalizer.json`` for the English text + #: normalizer. Whisper ships the same mapping across model sizes. The HF + #: copy is a superset of ``whisper/normalizers/english.json`` by one key + #: (``mm``) with no conflicting values, and `transformers`' + #: ``EnglishTextNormalizer`` is output-identical to the one in the + #: ``openai-whisper`` package. + DEFAULT_NORMALIZER_SOURCE = "openai/whisper-large-v3" + + def __init__(self, *args, normalizer_source: str | None = None, **kwargs): + super().__init__(*args, **kwargs) + self.normalizer_source = normalizer_source or self.DEFAULT_NORMALIZER_SOURCE + self._breakdown: dict[str, Any] | None = None + + def _normalizer(self) -> EnglishTextNormalizer: + tokenizer = AutoTokenizer.from_pretrained(self.normalizer_source) + spelling_map = getattr(tokenizer, "english_spelling_normalizer", None) + if not spelling_map: + raise ValueError( + f"Tokenizer {self.normalizer_source!r} carries no " + "english_spelling_normalizer (normalizer.json). Point " + "accuracy_config.extras.normalizer_source at a Whisper " + "tokenizer repo or local directory." + ) + return EnglishTextNormalizer(spelling_map) + + def score_single_sample(self, value: str, ground_truth: str) -> float: + raise RuntimeError( + "WER is a corpus-level metric: errors and reference words are summed " + "across the dataset before dividing. Call score() instead." + ) + + def score(self) -> tuple[float | None, int]: + df = self.get_scoring_outputs() + df = df[df["sample_uuid"].isin(self.sample_index_map.keys())] + + issued = len(self.sample_index_map) + if issued == 0: + return None, 0 + n_repeats = issued // self.dataset.num_samples() + + assert ( + self.dataset.dataframe is not None + ), f"Dataset {self.dataset} has no dataframe loaded" + assert ( + self.ground_truth_column in self.dataset.dataframe.columns + ), f"Ground truth column {self.ground_truth_column} not found in dataset {self.dataset}" + + normalizer = self._normalizer() + references = self.dataset.dataframe[self.ground_truth_column].to_numpy() + + distance = 0 + words = 0 + empty = 0 + for _, row in df.iterrows(): + reference = references[self.sample_index_map[row["sample_uuid"]]] + hypothesis = row["output"] + if not hypothesis: + empty += 1 + hyp_words = normalizer(project_to_labels(hypothesis)).split() + ref_words = normalizer(project_to_labels(str(reference))).split() + sample_distance, sample_words = wer_counts(hyp_words, ref_words) + distance += sample_distance + words += sample_words + + # Samples that never produced a COMPLETE event contribute no errors and + # no reference words, so they silently shrink the denominator rather + # than penalising the score - surface the count so a partial run is not + # mistaken for a clean one. + missing = issued - len(df) + if words == 0: + self._breakdown = {"wer_percent": None, "missing_samples": missing} + return None, n_repeats + + wer = distance / words + self._breakdown = { + "wer_percent": wer * 100.0, + "edit_distance": distance, + "reference_words": words, + "empty_responses": empty, + "missing_samples": missing, + } + if missing: + logger.warning( + "WER computed over %d of %d issued samples (%d missing); the " + "score does not describe a complete run.", + len(df), + issued, + missing, + ) + return 100.0 - wer * 100.0, n_repeats + + def score_breakdown(self) -> dict[str, Any] | None: + return self._breakdown + + def _uv_subproject_env(project_path: os.PathLike | str) -> dict[str, str]: """Env for ``uv run --project `` pinned to the subproject's OWN ``.venv``. diff --git a/src/inference_endpoint/evaluation/wer.py b/src/inference_endpoint/evaluation/wer.py new file mode 100644 index 000000000..28e784f25 --- /dev/null +++ b/src/inference_endpoint/evaluation/wer.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Word Error Rate as defined by the MLPerf speech2text benchmark. + +Ported from the MLCommons reference implementation +(https://github.com/mlcommons/inference/tree/master/speech2text), which is also +Apache-2.0: ``legacy_helpers.__levenshtein``, +``helpers.compute_wer_with_concatenation``, and the character-label projection +``accuracy_eval.py`` applies to both sides before scoring. + +The concatenation-aware alignment is why a generic WER library (jiwer, +torchmetrics) will not reproduce the reference number: the reference credits a +hypothesis word that equals up to three joined reference words, and vice versa, +before measuring edit distance. See ``docs/evaluation/DESIGN.md``. +""" + +from collections.abc import Sequence + +#: Character inventory the reference projects both hypotheses and references +#: through before normalization. Anything outside it is dropped. +LABELS = frozenset(" abcdefghijklmnopqrstuvwxyz'0123456789$%+-") + +#: Longest run of words the alignment will try to join on either side. +MAX_CONCATENATION_SPAN = 3 + + +def project_to_labels(text: str) -> str: + """Lowercase `text` and drop every character outside :data:`LABELS`. + + The reference encodes both the SUT output and the manifest transcript as + indices into its label list and decodes them back before scoring, which is + lossy in exactly this way. Reproducing it keeps punctuation handling + identical to the reference. + + The lowercasing is applied to both sides. The reference lowercases only the + SUT output (in its SUT) and not the manifest transcript (in + ``manifest.parse_transcript``), so an uppercase transcript — which is how + LibriSpeech ships — would have every letter dropped as out-of-inventory, + leaving an empty reference. Lowercasing both is a no-op whenever the + manifest is already lowercase, and the only behaviour that yields a + meaningful number when it is not. + """ + lowered = text.lower() + return "".join(c for c in lowered if c in LABELS) + + +def levenshtein(a: Sequence[str], b: Sequence[str]) -> int: + """Edit distance between two word sequences, in O(min(len(a), len(b))) space.""" + n, m = len(a), len(b) + if n > m: + a, b = b, a + n, m = m, n + + current = list(range(n + 1)) + for i in range(1, m + 1): + previous, current = current, [i] + [0] * n + for j in range(1, n + 1): + add, delete = previous[j] + 1, current[j - 1] + 1 + change = previous[j - 1] + if a[j - 1] != b[i - 1]: + change += 1 + current[j] = min(add, delete, change) + return current[n] + + +def wer_counts(hypothesis: Sequence[str], reference: Sequence[str]) -> tuple[int, int]: + """Return ``(edit_distance, reference_word_count)`` for one sample. + + Walks both word lists in lockstep, consuming a joined run of up to + :data:`MAX_CONCATENATION_SPAN` words on either side when it matches a single + word on the other, then takes the edit distance over the reconciled + sequences. Corpus WER is ``sum(distances) / sum(word_counts)`` — never the + mean of per-sample rates. + """ + alignment: list[tuple[str | None, str | None]] = [] + i = j = 0 + + while i < len(reference) and j < len(hypothesis): + if reference[i] == hypothesis[j]: + alignment.append((reference[i], hypothesis[j])) + i += 1 + j += 1 + continue + + ref_match_len = hyp_match_len = 1 + match_found = False + for k in range(1, MAX_CONCATENATION_SPAN + 1): + if ( + i + k <= len(reference) + and "".join(reference[i : i + k]) == hypothesis[j] + ): + ref_match_len, hyp_match_len = k, 1 + match_found = True + break + if ( + j + k <= len(hypothesis) + and "".join(hypothesis[j : j + k]) == reference[i] + ): + ref_match_len, hyp_match_len = 1, k + match_found = True + break + + if match_found: + alignment.append( + ( + " ".join(reference[i : i + ref_match_len]), + " ".join(hypothesis[j : j + hyp_match_len]), + ) + ) + i += ref_match_len + j += hyp_match_len + else: + # No reconciliation possible: count it as a substitution and advance + # both sides, matching the reference's alignment walk. + alignment.append((reference[i], hypothesis[j])) + i += 1 + j += 1 + + while i < len(reference): + alignment.append((reference[i], None)) + i += 1 + while j < len(hypothesis): + alignment.append((None, hypothesis[j])) + j += 1 + + # Spaces are stripped so a reconciled pair ("can not", "cannot") compares + # equal; the distance is measured over these joined tokens. + ref_aligned = [r.replace(" ", "") for r, _ in alignment if r is not None] + hyp_aligned = [h.replace(" ", "") for _, h in alignment if h is not None] + return levenshtein(ref_aligned, hyp_aligned), len(reference) diff --git a/src/inference_endpoint/openai/transcriptions_adapter.py b/src/inference_endpoint/openai/transcriptions_adapter.py new file mode 100644 index 000000000..1aa78e266 --- /dev/null +++ b/src/inference_endpoint/openai/transcriptions_adapter.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OpenAI /v1/audio/transcriptions adapter for Whisper-class ASR endpoints.""" + +from pathlib import Path +from typing import ClassVar + +import msgspec +from inference_endpoint.config.schema import ModelParams +from inference_endpoint.core.types import Query, QueryResult, TextModelOutput +from inference_endpoint.dataset_manager.transforms import ( + AddStaticColumns, + ColumnFilter, + Transform, +) +from inference_endpoint.endpoint_client.adapter_protocol import HttpRequestAdapter + +from .types import TranscriptionResponse + +#: Dataset column holding the path to each sample's audio file. +AUDIO_FILE_COLUMN = "audio_file" + +#: The worker folds Content-Type into its pre-encoded request prefix once at +#: setup, so the boundary cannot vary per request and must be a constant. It is +#: long and non-ASCII-audio-like enough that a collision with WAV payload bytes +#: is not a practical concern. +_BOUNDARY = "----mlperf-inference-endpoints-audio-boundary-Q3nV7xTt" + +_DASHES = f"--{_BOUNDARY}\r\n".encode("ascii") +_CLOSING = f"--{_BOUNDARY}--\r\n".encode("ascii") + + +def encode_multipart( + fields: dict[str, str], filename: str, audio: bytes, mime_type: str +) -> bytes: + """Encode `fields` plus one `file` part as an RFC 7578 multipart body. + + Args: + fields: Text form fields, in the order they should be written. + filename: Value of the file part's `filename` parameter. + audio: Raw audio bytes. + mime_type: Media type of the file part. + """ + parts: list[bytes] = [] + for name, value in fields.items(): + parts.append(_DASHES) + parts.append( + f'Content-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n'.encode() + ) + parts.append(_DASHES) + parts.append( + f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n' + f"Content-Type: {mime_type}\r\n\r\n".encode() + ) + parts.append(audio) + parts.append(b"\r\n") + parts.append(_CLOSING) + return b"".join(parts) + + +class OpenAITranscriptionsAdapter(HttpRequestAdapter): + """Adapter for OpenAI /v1/audio/transcriptions (e.g. vLLM serving Whisper). + + The request is a `multipart/form-data` audio upload, so the body is not JSON + and `CONTENT_TYPE` carries the boundary. The dataset row holds a *path*; the + bytes are read here, inside the worker, so a full audio corpus never + traverses the issue thread or the ZMQ frame. Keep the audio on fast local + storage: the read happens inside the measured request. + + `language` is what makes the server build Whisper's + `<|startoftranscript|><|LANG|><|transcribe|><|notimestamps|>` prompt, which + is how this matches the MLPerf speech2text reference. + + The endpoint is non-streaming: OpenAI does not support `stream` for + Whisper-class models, so `decode_sse_message` raises and the registered + accumulator is `NonStreamingAccumulator`. + """ + + CONTENT_TYPE: ClassVar[str] = f"multipart/form-data; boundary={_BOUNDARY}" + + #: Form fields forwarded from the dataset row, in request order. `model` + #: and `file` are required by the API; the rest are optional and omitted + #: when absent or None so server-side defaults apply. + FORM_FIELDS: ClassVar[tuple[str, ...]] = ( + "model", + "language", + "prompt", + "response_format", + "temperature", + ) + + #: WAV is the only format the MLPerf LibriSpeech corpus ships. Servers + #: sniff the payload rather than trusting this part header (vLLM hands the + #: bytes to librosa/soundfile), so a per-row MIME column buys nothing. + MIME_TYPE: ClassVar[str] = "audio/wav" + + _response_decoder: msgspec.json.Decoder = msgspec.json.Decoder( + TranscriptionResponse + ) + + @classmethod + def dataset_transforms(cls, model_params: ModelParams) -> list[Transform]: + metadata = { + "model": model_params.name, + "response_format": "json", + "temperature": model_params.temperature, + } + return [ + ColumnFilter( + required_columns=[AUDIO_FILE_COLUMN], + optional_columns=list(cls.FORM_FIELDS), + ), + # overwrite=False so a dataset row that carries its own `language` + # or `prompt` wins over the model_params defaults. + AddStaticColumns(metadata, overwrite=False), + ] + + @classmethod + def encode_query(cls, query: Query) -> bytes: + data = query.data + if AUDIO_FILE_COLUMN not in data: + raise KeyError( + f"{AUDIO_FILE_COLUMN!r} not found in query.data keys: " + f"{list(data.keys())}" + ) + if data.get("stream"): + raise ValueError( + "OpenAITranscriptionsAdapter is non-streaming; remove `stream` " + "from query.data and set model_params.streaming to 'off'." + ) + audio_path = Path(data[AUDIO_FILE_COLUMN]) + fields = { + name: str(data[name]) + for name in cls.FORM_FIELDS + if data.get(name) is not None + } + return encode_multipart( + fields, audio_path.name, audio_path.read_bytes(), cls.MIME_TYPE + ) + + @classmethod + def decode_response(cls, response_bytes: bytes, query_id: str) -> QueryResult: + resp = cls._response_decoder.decode(response_bytes) + return QueryResult( + id=query_id, + response_output=TextModelOutput(output=resp.text), + ) + + @classmethod + def decode_sse_message(cls, json_bytes: bytes) -> str: + raise NotImplementedError( + "OpenAITranscriptionsAdapter does not use SSE streaming" + ) diff --git a/src/inference_endpoint/openai/types.py b/src/inference_endpoint/openai/types.py index 400baf704..fa791470e 100644 --- a/src/inference_endpoint/openai/types.py +++ b/src/inference_endpoint/openai/types.py @@ -270,3 +270,22 @@ class TextCompletionSSEMessage( """SSE message structure for /v1/completions streaming responses.""" choices: tuple[TextCompletionSSEChoice, ...] = () + + +# ============================================================================ +# OpenAI Audio Transcription Types (POST /v1/audio/transcriptions) +# Used by OpenAITranscriptionsAdapter for Whisper-class ASR endpoints. +# ============================================================================ + + +class TranscriptionResponse( + msgspec.Struct, frozen=True, kw_only=True, omit_defaults=True, gc=False +): # type: ignore[call-arg] + """Response body for `response_format=json`. + + The request is multipart/form-data, so there is no matching request struct - + the adapter encodes form fields directly. Server-specific extras (duration, + language, usage) decode away: msgspec ignores unknown fields. + """ + + text: str diff --git a/tests/integration/openai/__init__.py b/tests/integration/openai/__init__.py new file mode 100644 index 000000000..467079831 --- /dev/null +++ b/tests/integration/openai/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/integration/openai/conftest.py b/tests/integration/openai/conftest.py new file mode 100644 index 000000000..c281bd2d8 --- /dev/null +++ b/tests/integration/openai/conftest.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration fixtures for the OpenAI transcriptions adapter. + +`MockTranscriptionServer` subclasses `EchoServer` to reuse its background-thread +aiohttp lifecycle (port discovery, start/stop, ready event); only the route +differs. It parses the upload with aiohttp's real multipart reader, so a +malformed body fails here rather than silently round-tripping. +""" + +import json +from collections.abc import Generator +from pathlib import Path + +import pytest +from aiohttp import web +from inference_endpoint.testing.echo_server import EchoServer + +#: 16-bit PCM WAV header plus a short run of samples. Only the bytes matter - +#: nothing in the client or the mock decodes the audio. +WAV_HEADER = ( + b"RIFF" + (36 + 512).to_bytes(4, "little") + b"WAVEfmt " + b"\x10\x00\x00\x00\x01\x00\x01\x00\x80>\x00\x00\x00}\x00\x00\x02\x00\x10\x00" + b"data" + (512).to_bytes(4, "little") +) + + +def write_wav(path: Path, payload: bytes | None = None) -> bytes: + """Write a dummy WAV at `path` and return its bytes.""" + data = WAV_HEADER + (payload if payload is not None else bytes(range(256)) * 2) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return data + + +class MockTranscriptionServer(EchoServer): + """vLLM-shaped mock for `POST /v1/audio/transcriptions`. + + Returns the transcript registered for the uploaded file's basename, or the + basename itself when none is registered. Records each request's form fields + and uploaded bytes on `self.requests` so tests can assert on the wire + format rather than only on the decoded result. + """ + + def __init__(self, transcripts: dict[str, str] | None = None) -> None: + super().__init__(port=0) + self.transcripts = transcripts or {} + self.requests: list[dict] = [] + + def _register_routes(self, app: web.Application) -> None: + app.router.add_post("/v1/audio/transcriptions", self._handle_transcription) + + async def _handle_transcription(self, request: web.Request) -> web.Response: + if not request.content_type.startswith("multipart/form-data"): + return web.Response( + status=415, text=f"expected multipart, got {request.content_type}" + ) + + fields: dict[str, str] = {} + filename = "" + audio = b"" + reader = await request.multipart() + while True: + part = await reader.next() + if part is None: + break + if part.name == "file": + filename = part.filename or "" + audio = bytes(await part.read(decode=False)) + else: + fields[part.name] = (await part.read(decode=False)).decode() + + self.requests.append({"fields": fields, "filename": filename, "audio": audio}) + if not audio: + return web.Response(status=400, text="empty file part") + + text = self.transcripts.get(filename, filename) + if fields.get("response_format", "json") != "json": + return web.Response( + status=400, text="mock only serves response_format=json" + ) + return web.Response( + body=json.dumps({"text": text, "duration": 29.5}).encode(), + content_type="application/json", + ) + + +@pytest.fixture +def mock_transcription_server( + request: pytest.FixtureRequest, +) -> Generator[MockTranscriptionServer, None, None]: + transcripts = getattr(request, "param", None) + server = MockTranscriptionServer(transcripts=transcripts) + server.start() + yield server + server.stop() diff --git a/tests/integration/openai/test_transcriptions_adapter.py b/tests/integration/openai/test_transcriptions_adapter.py new file mode 100644 index 000000000..d2b46b1aa --- /dev/null +++ b/tests/integration/openai/test_transcriptions_adapter.py @@ -0,0 +1,274 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end transcription run: dataset -> worker -> multipart upload -> WER. + +Drives the real HTTP worker pool against a mock /v1/audio/transcriptions server, +so the multipart body, the adapter-declared Content-Type, and the WER scorer are +all exercised on the same path a real Whisper run takes. +""" + +import asyncio +import json +import random +import urllib.error +import urllib.request +from pathlib import Path +from urllib.parse import urljoin + +import msgspec +import pytest +from inference_endpoint import metrics +from inference_endpoint.config.runtime_settings import RuntimeSettings +from inference_endpoint.config.schema import ( + LoadPattern, + LoadPatternType, + ModelParams, +) +from inference_endpoint.core.record import EventRecord, EventType, SampleEventType +from inference_endpoint.core.types import ( + APIType, + Query, + QueryResult, + TextModelOutput, +) +from inference_endpoint.dataset_manager.dataset import Dataset +from inference_endpoint.dataset_manager.predefined.whisper_librispeech import ( + WhisperLibriSpeech, +) +from inference_endpoint.endpoint_client.config import HTTPClientConfig +from inference_endpoint.endpoint_client.http_client import HTTPEndpointClient +from inference_endpoint.endpoint_client.http_sample_issuer import HttpClientSampleIssuer +from inference_endpoint.evaluation.scoring import WERScorer +from inference_endpoint.load_generator.session import BenchmarkSession, PhaseConfig +from inference_endpoint.openai.transcriptions_adapter import ( + OpenAITranscriptionsAdapter, +) +from transformers.models.whisper.english_normalizer import EnglishTextNormalizer + +from .conftest import MockTranscriptionServer, write_wav + +MODEL = "openai/whisper-large-v3" +DATASET_NAME = "whisper_librispeech" + +# LibriSpeech transcripts are uppercase; the mock returns Whisper-style cased, +# punctuated text so the scorer's normalization is actually exercised. +CORPUS = [ + ( + "MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES", + "Mister Quilter is the apostle of the middle classes.", + ), + ("AND WE ARE GLAD TO WELCOME HIS GOSPEL", "And we are glad to welcome his gospel."), + ( + "HE HAS GRAVE DOUBTS WHETHER SIR FREDERICK IS A GREAT PAINTER", + "He has grave doubts whether Sir Frederick is a great painter.", + ), + ("THE RAINBOW COMES AND GOES", "The rainbow comes and goes."), +] + + +class _NoOpPublisher: + def publish(self, event_record: EventRecord) -> None: + pass + + def flush(self) -> None: + pass + + +@pytest.fixture +def corpus(tmp_path: Path) -> tuple[Path, dict[str, bytes]]: + """Write an MLCommons-shaped manifest plus WAVs; return (manifest, bytes).""" + audio_dir = tmp_path / "dev-all-repack" + entries = [] + audio: dict[str, bytes] = {} + for i, (transcript, _) in enumerate(CORPUS): + name = f"{i}.wav" + # Distinct payload per clip so a mixed-up upload is detectable. + audio[name] = write_wav(audio_dir / name, bytes([i]) * 512) + entries.append( + { + "transcript": transcript, + "files": [ + {"fname": f"dev-all-repack/{name}", "speed": 1, "duration": 29.0} + ], + "original_duration": 29.0, + } + ) + manifest = tmp_path / "dev-all-repack.json" + manifest.write_text(json.dumps(entries)) + return manifest, audio + + +@pytest.fixture +def dataset(tmp_path: Path, corpus) -> Dataset: + manifest, _ = corpus + ds = WhisperLibriSpeech.get_dataloader( + datasets_dir=tmp_path / "cache", manifest=manifest + ) + ds.load( + api_type=APIType.OPENAI_TRANSCRIPTIONS, + model_params=ModelParams(name=MODEL, temperature=0.0), + ) + return ds + + +# Mock returns the Whisper-style transcript keyed by uploaded filename. +TRANSCRIPTS = {f"{i}.wav": hyp for i, (_, hyp) in enumerate(CORPUS)} + + +async def _run(url: str, dataset) -> tuple[dict[str, str], dict[str, int]]: + loop = asyncio.get_running_loop() + n_samples = dataset.num_samples() + responses: dict[str, str] = {} + + def on_complete(result: QueryResult) -> None: + responses[result.id] = result.get_response_output_string() + + http_config = HTTPClientConfig( + api_type=APIType.OPENAI_TRANSCRIPTIONS, + endpoint_urls=[ + urljoin( + url.rstrip("/") + "/", APIType.OPENAI_TRANSCRIPTIONS.default_route() + ) + ], + warmup_connections=0, + num_workers=2, + ) + client = await HTTPEndpointClient.create(http_config, loop) + try: + session = BenchmarkSession( + issuer=HttpClientSampleIssuer(client), + event_publisher=_NoOpPublisher(), + loop=loop, + on_sample_complete=on_complete, + ) + rt_settings = RuntimeSettings( + metrics.Throughput(100), + [metrics.Throughput(100)], + min_issue_duration_ms=0, + max_issue_duration_ms=60_000, + n_samples_from_dataset=n_samples, + n_samples_to_issue=n_samples, + min_sample_count=1, + rng_sched=random.Random(1234), + rng_sample_index=random.Random(1234), + load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), + ) + result = await asyncio.wait_for( + session.run([PhaseConfig("performance", rt_settings, dataset)]), + timeout=120.0, + ) + finally: + await client.shutdown_async() + + assert result.perf_results[0].issued_count == n_samples + return responses, result.perf_results[0].uuid_to_index + + +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize("mock_transcription_server", [TRANSCRIPTS], indirect=True) +async def test_full_run_uploads_audio_and_scores_wer( + mock_transcription_server: MockTranscriptionServer, + dataset, + corpus, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, audio = corpus + responses, uuid_to_index = await _run(mock_transcription_server.url, dataset) + + # --- every sample completed and carried the right transcript back --- + assert len(responses) == len(CORPUS) + for sample_uuid, text in responses.items(): + _, expected_hypothesis = CORPUS[uuid_to_index[sample_uuid]] + assert text == expected_hypothesis + + # --- the server saw a well-formed upload per sample --- + assert len(mock_transcription_server.requests) == len(CORPUS) + seen = {} + for req in mock_transcription_server.requests: + assert req["fields"]["model"] == MODEL + assert req["fields"]["response_format"] == "json" + assert req["fields"]["temperature"] == "0.0" + assert "stream" not in req["fields"] + seen[req["filename"]] = req["audio"] + # Each clip's bytes arrived intact and with its own filename - a shared + # multipart boundary must not bleed one upload into another. + assert seen == audio + + # --- the real scorer over the real run artifacts --- + report_dir = tmp_path / "report" + report_dir.mkdir() + (report_dir / "sample_idx_map.json").write_bytes( + msgspec.json.encode({DATASET_NAME: uuid_to_index}) + ) + encoder = msgspec.json.Encoder(enc_hook=EventType.encode_hook) + with (report_dir / "events.jsonl").open("wb") as f: + for sample_uuid, text in responses.items(): + f.write( + encoder.encode( + EventRecord( + event_type=SampleEventType.COMPLETE, + sample_uuid=sample_uuid, + data=TextModelOutput(output=text), + ) + ) + + b"\n" + ) + + monkeypatch.setattr( + WERScorer, "_normalizer", lambda self: EnglishTextNormalizer({}) + ) + scorer = WERScorer( + DATASET_NAME, dataset, report_dir, ground_truth_column="transcript" + ) + score, n_repeats = scorer.score() + breakdown = scorer.score_breakdown() + assert breakdown is not None + + # Hypotheses differ from the references only in case and punctuation. + assert score == pytest.approx(100.0) + assert n_repeats == 1 + assert breakdown["edit_distance"] == 0 + assert breakdown["missing_samples"] == 0 + assert breakdown["reference_words"] == sum(len(r.split()) for r, _ in CORPUS) + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_server_rejects_a_non_multipart_body( + mock_transcription_server: MockTranscriptionServer, tmp_path: Path +) -> None: + """Guards the Content-Type wiring: JSON to this route is a 415.""" + url = f"{mock_transcription_server.url}/v1/audio/transcriptions" + req = urllib.request.Request( + url, + data=b'{"model": "m"}', + headers={"Content-Type": "application/json"}, + method="POST", + ) + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(req) + assert exc.value.code == 415 + + +@pytest.mark.integration +def test_dataset_rows_are_request_ready(dataset) -> None: + """The loaded row is used verbatim as Query.data, so it must encode.""" + row = dataset.load_sample(0) + assert set(row) == {"audio_file", "model", "response_format", "temperature"} + body = OpenAITranscriptionsAdapter.encode_query(Query(id="q", data=row)) + assert Path(row["audio_file"]).read_bytes() in body diff --git a/tests/unit/async_utils/services/metrics_aggregator/test_token_metrics.py b/tests/unit/async_utils/services/metrics_aggregator/test_token_metrics.py index ab15bf0d8..2d829ec84 100644 --- a/tests/unit/async_utils/services/metrics_aggregator/test_token_metrics.py +++ b/tests/unit/async_utils/services/metrics_aggregator/test_token_metrics.py @@ -16,6 +16,7 @@ """Tests for BatchTokenizer and TokenBatchQueue.""" import asyncio +import logging import multiprocessing import signal import time @@ -322,6 +323,37 @@ def apply_chat_template(self, *args, **kwargs): )[0] assert count > 0 + def test_baseline_failure_without_chat_template_is_not_an_error(self, caplog): + """ASR tokenizers (Whisper) have no chat template; baseline 0 is expected.""" + + class _NoTemplateTokenizer(_FakeTokenizer): + chat_template = None + + def apply_chat_template(self, *args, **kwargs): + raise ValueError("tokenizer.chat_template is not set") + + with patch(_MOCK_TARGET, _NoTemplateTokenizer): + with caplog.at_level(logging.DEBUG): + with BatchTokenizer("fake", n_workers=0, live_workers=1) as tok: + assert tok._baseline == 0 + records = [r for r in caplog.records if "baseline" in r.message] + assert records and all(r.levelno == logging.DEBUG for r in records) + + def test_baseline_failure_with_a_chat_template_is_logged_as_an_error(self, caplog): + """A tokenizer that has a template but fails still surfaces the problem.""" + + class _BrokenTemplateTokenizer(_FakeTokenizer): + chat_template = "{{ broken }}" + + def apply_chat_template(self, *args, **kwargs): + raise ValueError("render failed") + + with patch(_MOCK_TARGET, _BrokenTemplateTokenizer): + with caplog.at_level(logging.DEBUG): + with BatchTokenizer("fake", n_workers=0, live_workers=1) as tok: + assert tok._baseline == 0 + assert any(r.levelno == logging.ERROR for r in caplog.records) + def test_token_count_prompt_falls_back_on_template_error(self): class _TextOnlyTemplateTokenizer(_FakeTokenizerWithTemplate): def apply_chat_template(self, messages, **kwargs): diff --git a/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py b/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py new file mode 100644 index 000000000..7391f7732 --- /dev/null +++ b/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py @@ -0,0 +1,336 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the WhisperLibriSpeech predefined dataset. + +The download tests run without network: requests.get (downloader script fetch) +and subprocess.run (the mlc-r2-downloader invocation) are mocked in +`dataset.download_from_r2`, so only the loader's own logic runs. +""" + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from inference_endpoint.dataset_manager.dataset import Dataset +from inference_endpoint.dataset_manager.predefined.whisper_librispeech import ( + MANIFEST_FILENAME, + SOURCE_ENV, + WhisperLibriSpeech, +) + +_MODULE = "inference_endpoint.dataset_manager.dataset" + + +def _entry(fname: str, transcript: str, duration: float = 29.5) -> dict: + """One MLCommons dev-all-repack.json entry.""" + return { + "transcript": transcript, + "files": [{"fname": fname, "speed": 1, "duration": duration}], + "original_duration": duration, + "original_num_samples": int(duration * 16000), + } + + +@pytest.fixture +def manifest(tmp_path: Path) -> Path: + path = tmp_path / "dev-all-repack.json" + path.write_text( + json.dumps( + [ + _entry("dev-all-repack/0.wav", "mister quilter is the apostle", 29.1), + _entry("dev-all-repack/1.wav", "of the middle classes", 28.4), + ] + ) + ) + return path + + +def _mock_requests_get() -> MagicMock: + """A requests.get returning a trivial downloader script body.""" + resp = MagicMock() + resp.content = b"#!/usr/bin/env bash\nexit 0\n" + resp.raise_for_status.return_value = None + return resp + + +def _fake_run_unpacking(cmd, **kwargs) -> MagicMock: + """subprocess.run side effect that unpacks a corpus under the -d directory.""" + download_dir = Path(cmd[cmd.index("-d") + 1]) + (download_dir / "dev-all-repack").mkdir(parents=True, exist_ok=True) + (download_dir / "dev-all-repack" / "0.wav").write_bytes(b"RIFF") + (download_dir / MANIFEST_FILENAME).write_text( + json.dumps([_entry("dev-all-repack/0.wav", "downloaded transcript")]) + ) + return MagicMock(returncode=0, stderr="") + + +@pytest.fixture +def no_local_manifest(monkeypatch: pytest.MonkeyPatch) -> None: + """Drop the env overrides so generate() takes the download path.""" + monkeypatch.delenv(WhisperLibriSpeech.R2_DATASET_URI_ENV, raising=False) + monkeypatch.delenv(SOURCE_ENV, raising=False) + + +@pytest.mark.unit +class TestRegistration: + def test_registered_under_its_dataset_id(self): + assert Dataset.PREDEFINED["whisper_librispeech"] is WhisperLibriSpeech + + def test_serves_both_phases(self): + # ACCURACY_ONLY would make the perf phase reject this dataset. + assert WhisperLibriSpeech.ACCURACY_ONLY is False + + def test_column_names(self): + assert WhisperLibriSpeech.COLUMN_NAMES == [ + "audio_file", + "transcript", + "duration", + ] + + +@pytest.mark.unit +class TestGenerate: + def test_flattens_manifest_entries(self, tmp_path: Path, manifest: Path): + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=manifest) + assert list(df.columns) == WhisperLibriSpeech.COLUMN_NAMES + assert len(df) == 2 + assert df.iloc[0]["transcript"] == "mister quilter is the apostle" + assert df.iloc[0]["duration"] == pytest.approx(29.1) + + def test_resolves_fname_against_the_manifest_directory( + self, tmp_path: Path, manifest: Path + ): + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=manifest) + assert df.iloc[0]["audio_file"] == str(manifest.parent / "dev-all-repack/0.wav") + + def test_explicit_data_dir_overrides_the_manifest_directory( + self, tmp_path: Path, manifest: Path + ): + df = WhisperLibriSpeech.generate( + tmp_path / "cache", manifest=manifest, data_dir="/mnt/audio" + ) + assert df.iloc[0]["audio_file"] == "/mnt/audio/dev-all-repack/0.wav" + + def test_drops_fname_prefix_absent_from_the_corpus_root(self, tmp_path: Path): + # The hosted artifact unpacks dev-all-repack/ into the corpus root, + # while the manifest's fnames keep the upstream pipeline's ./data/ + # prefix. The clips that exist on disk are what decide the layout. + source = tmp_path / "source" + (source / "dev-all-repack").mkdir(parents=True) + for name in ("0.wav", "1.wav"): + (source / "dev-all-repack" / name).write_bytes(b"RIFF") + path = source / MANIFEST_FILENAME + path.write_text( + json.dumps( + [ + _entry("./data/dev-all-repack/0.wav", "first"), + _entry("./data/dev-all-repack/1.wav", "second"), + ] + ) + ) + + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + assert df.iloc[0]["audio_file"] == str(source / "dev-all-repack/0.wav") + assert df.iloc[1]["audio_file"] == str(source / "dev-all-repack/1.wav") + + def test_keeps_fname_as_written_when_no_clip_resolves(self, tmp_path: Path): + path = tmp_path / MANIFEST_FILENAME + path.write_text(json.dumps([_entry("./data/dev-all-repack/0.wav", "first")])) + + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + assert df.iloc[0]["audio_file"] == str(tmp_path / "data/dev-all-repack/0.wav") + + def test_writes_and_reuses_the_parquet_cache(self, tmp_path: Path, manifest: Path): + cache = tmp_path / "cache" + WhisperLibriSpeech.generate(cache, manifest=manifest) + parquet = cache / "whisper_librispeech" / "whisper_librispeech.parquet" + assert parquet.exists() + + # A second call must not need the manifest at all. + manifest.unlink() + df = WhisperLibriSpeech.generate(cache, manifest=manifest) + assert len(df) == 2 + + def test_force_rebuilds_from_the_manifest(self, tmp_path: Path, manifest: Path): + cache = tmp_path / "cache" + WhisperLibriSpeech.generate(cache, manifest=manifest) + manifest.write_text(json.dumps([_entry("dev-all-repack/0.wav", "only one")])) + df = WhisperLibriSpeech.generate(cache, manifest=manifest, force=True) + assert len(df) == 1 + assert df.iloc[0]["transcript"] == "only one" + + def test_max_samples_truncates_in_manifest_order( + self, tmp_path: Path, manifest: Path + ): + df = WhisperLibriSpeech.generate( + tmp_path / "cache", manifest=manifest, max_samples=1 + ) + assert len(df) == 1 + assert df.iloc[0]["transcript"] == "mister quilter is the apostle" + + def test_reads_the_manifest_from_the_environment( + self, tmp_path: Path, manifest: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setenv(SOURCE_ENV, str(manifest)) + df = WhisperLibriSpeech.generate(tmp_path / "cache") + assert len(df) == 2 + + +@pytest.mark.unit +class TestR2Download: + def test_downloads_when_no_manifest_is_supplied( + self, tmp_path: Path, no_local_manifest: None + ): + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch(f"{_MODULE}.subprocess.run", side_effect=_fake_run_unpacking), + ): + df = WhisperLibriSpeech.generate(tmp_path / "cache") + + assert df.iloc[0]["transcript"] == "downloaded transcript" + # fname resolves against the downloaded manifest's own directory. + assert df.iloc[0]["audio_file"] == str( + tmp_path / "cache/whisper_librispeech/source/dev-all-repack/0.wav" + ) + + def test_a_local_manifest_skips_the_download(self, tmp_path: Path, manifest: Path): + with patch(f"{_MODULE}.requests.get") as get: + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=manifest) + get.assert_not_called() + + def test_force_reuses_the_download_instead_of_refetching( + self, tmp_path: Path, no_local_manifest: None + ): + cache = tmp_path / "cache" + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch(f"{_MODULE}.subprocess.run", side_effect=_fake_run_unpacking), + ): + WhisperLibriSpeech.generate(cache) + + # force rebuilds the parquet; it must not re-fetch GBs of audio. + with patch(f"{_MODULE}.requests.get") as get: + assert len(WhisperLibriSpeech.generate(cache, force=True)) == 1 + get.assert_not_called() + + @pytest.mark.parametrize( + "uri", + [ + "https://evil.example.com/x.uri", + "http://inference.mlcommons-storage.org/x.uri", + ], + ) + def test_untrusted_uri_is_refused_before_any_fetch( + self, + tmp_path: Path, + uri: str, + no_local_manifest: None, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setenv(WhisperLibriSpeech.R2_DATASET_URI_ENV, uri) + with patch(f"{_MODULE}.requests.get") as get: + with pytest.raises(ValueError, match="untrusted URI"): + WhisperLibriSpeech.generate(tmp_path / "cache") + get.assert_not_called() + + def test_downloader_failure_is_reported( + self, tmp_path: Path, no_local_manifest: None + ): + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch( + f"{_MODULE}.subprocess.run", + return_value=MagicMock(returncode=2, stderr="r2 boom"), + ), + ): + with pytest.raises(RuntimeError, match="r2 boom"): + WhisperLibriSpeech.generate(tmp_path / "cache") + + +@pytest.mark.unit +class TestGenerateErrors: + def test_absent_manifest_path(self, tmp_path: Path): + with pytest.raises(FileNotFoundError, match="not found at"): + WhisperLibriSpeech.generate( + tmp_path / "cache", manifest=tmp_path / "missing.json" + ) + + def test_non_json_source_is_rejected(self, tmp_path: Path): + raw = tmp_path / "dev-all.flac" + raw.write_bytes(b"\x00") + with pytest.raises(NotImplementedError, match="dev-all-repack.json"): + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=raw) + + def test_entry_without_files_is_rejected(self, tmp_path: Path): + path = tmp_path / "bad.json" + path.write_text(json.dumps([{"transcript": "hi", "original_duration": 1.0}])) + with pytest.raises(ValueError, match="no 'files' list"): + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + + def test_speed_perturbed_only_entry_is_rejected(self, tmp_path: Path): + path = tmp_path / "perturbed.json" + path.write_text( + json.dumps( + [ + { + "transcript": "hi", + "files": [ + {"fname": "slow.wav", "speed": 0.9, "duration": 1.0}, + {"fname": "fast.wav", "speed": 1.1, "duration": 1.0}, + ], + "original_duration": 1.0, + } + ] + ) + ) + with pytest.raises(ValueError, match="speed == 1"): + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + + def test_unperturbed_clip_is_selected(self, tmp_path: Path): + path = tmp_path / "mixed.json" + path.write_text( + json.dumps( + [ + { + "transcript": "hi", + "files": [ + {"fname": "slow.wav", "speed": 0.9, "duration": 1.1}, + {"fname": "normal.wav", "speed": 1, "duration": 1.0}, + ], + "original_duration": 1.0, + } + ] + ) + ) + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + assert df.iloc[0]["audio_file"].endswith("normal.wav") + + def test_sidecar_transcript_manifest_is_rejected(self, tmp_path: Path): + path = tmp_path / "sidecar.json" + path.write_text( + json.dumps( + [ + { + "text_filepath": "0.txt", + "files": [{"fname": "0.wav", "speed": 1, "duration": 1.0}], + "original_duration": 1.0, + } + ] + ) + ) + with pytest.raises(ValueError, match="text_filepath"): + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) diff --git a/tests/unit/endpoint_client/test_http.py b/tests/unit/endpoint_client/test_http.py index 40e2dc5e6..363af9c43 100644 --- a/tests/unit/endpoint_client/test_http.py +++ b/tests/unit/endpoint_client/test_http.py @@ -171,6 +171,74 @@ def test_cache_headers_pre_caches(self): assert b"Content-Length: 17" in request +class TestHttpRequestTemplateContentType: + """Adapter-declared Content-Type (e.g. a multipart audio upload).""" + + MULTIPART = "multipart/form-data; boundary=----boundary-With;Punctuation=1" + + @staticmethod + def _parse(request: bytes) -> tuple[dict[str, list[str]], bytes]: + state: dict[str, list[str]] = {} + body = bytearray() + + class Handler: + def on_url(self, url): + pass + + def on_header(self, name, value): + state.setdefault(name.decode().lower(), []).append(value.decode()) + + def on_body(self, chunk): + body.extend(chunk) + + httptools.HttpRequestParser(Handler()).feed_data(request) + return state, bytes(body) + + def test_default_is_json(self): + template = HttpRequestTemplate.from_url("localhost", 8080, "/v1/chat") + assert template.content_type == "application/json" + + @pytest.mark.parametrize("streaming", [False, True]) + def test_declared_content_type_renders_exactly_once(self, streaming): + template = HttpRequestTemplate.from_url( + "localhost", 8080, "/v1/audio/transcriptions", content_type=self.MULTIPART + ) + headers, _ = self._parse(template.build_request(b"--x--\r\n", streaming)) + # A duplicated Content-Type (the failure mode of appending via + # extra_headers) makes the server read the wrong media type. + assert headers["content-type"] == [self.MULTIPART] + + def test_boundary_punctuation_does_not_break_header_framing(self): + """`;` and `=` inside the Content-Type value must stay in one header.""" + template = HttpRequestTemplate.from_url( + "localhost", 8080, "/v1/audio/transcriptions", content_type=self.MULTIPART + ) + body = b'--x\r\nContent-Disposition: form-data; name="a"\r\n\r\n1\r\n--x--\r\n' + headers, parsed_body = self._parse(template.build_request(body, False)) + assert headers["content-type"] == [self.MULTIPART] + assert headers["content-length"] == [str(len(body))] + assert parsed_body == body + + def test_binary_body_is_sent_verbatim(self): + """A non-JSON body passes through untouched with a correct length.""" + template = HttpRequestTemplate.from_url( + "localhost", 8080, "/v1/audio/transcriptions", content_type=self.MULTIPART + ) + body = bytes(range(256)) * 4 + headers, parsed_body = self._parse(template.build_request(body, False)) + assert parsed_body == body + assert headers["content-length"] == [str(len(body))] + + def test_cached_headers_preserve_the_content_type(self): + template = HttpRequestTemplate.from_url( + "localhost", 8080, "/v1/audio/transcriptions", content_type=self.MULTIPART + ) + template.cache_headers({"Authorization": "Bearer t"}) + headers, _ = self._parse(template.build_request(b"x", False)) + assert headers["content-type"] == [self.MULTIPART] + assert headers["authorization"] == ["Bearer t"] + + # ============================================================================= # HttpResponseProtocol Tests # ============================================================================= diff --git a/tests/unit/evaluation/test_wer.py b/tests/unit/evaluation/test_wer.py new file mode 100644 index 000000000..1808b7f93 --- /dev/null +++ b/tests/unit/evaluation/test_wer.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the MLPerf speech2text WER metric.""" + +import pytest +from inference_endpoint.evaluation.wer import ( + LABELS, + levenshtein, + project_to_labels, + wer_counts, +) + + +@pytest.mark.unit +class TestLevenshtein: + @pytest.mark.parametrize( + ("a", "b", "expected"), + [ + ([], [], 0), + (["a"], [], 1), + ([], ["a"], 1), + (["a", "b", "c"], ["a", "b", "c"], 0), + (["a", "b", "c"], ["a", "b", "d"], 1), + (list("kitten"), list("sitting"), 3), + (["a", "b"], ["b", "a"], 2), + ], + ) + def test_distance(self, a, b, expected): + assert levenshtein(a, b) == expected + + def test_is_symmetric(self): + a, b = list("saturday"), list("sunday") + assert levenshtein(a, b) == levenshtein(b, a) + + +@pytest.mark.unit +class TestProjectToLabels: + def test_lowercases(self): + assert project_to_labels("MISTER QUILTER") == "mister quilter" + + def test_drops_punctuation_outside_the_inventory(self): + assert project_to_labels("Hello, world! (yes)") == "hello world yes" + + def test_keeps_apostrophes_digits_and_symbols(self): + assert project_to_labels("IT'S 42% +3 -1 $5") == "it's 42% +3 -1 $5" + + def test_inventory_matches_the_reference_label_set(self): + assert LABELS == frozenset(" abcdefghijklmnopqrstuvwxyz'0123456789$%+-") + + +@pytest.mark.unit +class TestWerCounts: + def test_exact_match_scores_zero_errors(self): + words = ["the", "cat", "sat"] + assert wer_counts(words, words) == (0, 3) + + def test_substitution_counts_one_error(self): + assert wer_counts(["the", "dog", "sat"], ["the", "cat", "sat"]) == (1, 3) + + def test_deletion_counts_one_error(self): + assert wer_counts(["the", "sat"], ["the", "cat", "sat"]) == (1, 3) + + def test_insertion_counts_one_error(self): + assert wer_counts(["the", "big", "cat", "sat"], ["the", "cat", "sat"]) == (1, 3) + + def test_empty_hypothesis_counts_every_reference_word(self): + assert wer_counts([], ["the", "cat", "sat"]) == (3, 3) + + def test_empty_reference_reports_zero_words(self): + # Denominator of 0 - the caller must sum counts across the corpus before + # dividing, which is why this returns counts rather than a rate. + assert wer_counts(["the"], []) == (1, 0) + + def test_both_empty(self): + assert wer_counts([], []) == (0, 0) + + def test_joined_hypothesis_word_is_credited(self): + # "can not" -> "cannot" is the concatenation case a generic WER library + # would score as 2 errors. + assert wer_counts(["cannot"], ["can", "not"]) == (0, 2) + + def test_joined_reference_word_is_credited(self): + assert wer_counts(["can", "not"], ["cannot"]) == (0, 1) + + def test_three_word_join_is_credited(self): + assert wer_counts(["anyoneelse"], ["any", "one", "else"]) == (0, 3) + + def test_four_word_join_exceeds_the_span_and_counts_errors(self): + distance, words = wer_counts(["abcd"], ["a", "b", "c", "d"]) + assert words == 4 + assert distance > 0 + + def test_join_credited_mid_sentence(self): + assert wer_counts(["i", "cannot", "go"], ["i", "can", "not", "go"]) == (0, 4) + + def test_corpus_wer_is_summed_not_averaged(self): + samples = [ + (["the", "dog"], ["the", "cat"]), + (["a"] * 10, ["a"] * 10), + ] + distance = words = 0 + for hyp, ref in samples: + d, w = wer_counts(hyp, ref) + distance += d + words += w + # 1 error over 12 reference words, not the mean of 50% and 0%. + assert (distance, words) == (1, 12) + + +@pytest.mark.unit +class TestEndToEndOnText: + def test_reference_pipeline_on_a_librispeech_style_pair(self): + reference = "MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES" + hypothesis = "Mister Quilter is the apostle of the middle classes." + hyp = project_to_labels(hypothesis).split() + ref = project_to_labels(reference).split() + assert wer_counts(hyp, ref) == (0, 9) diff --git a/tests/unit/evaluation/test_wer_scorer.py b/tests/unit/evaluation/test_wer_scorer.py new file mode 100644 index 000000000..ee2afe4df --- /dev/null +++ b/tests/unit/evaluation/test_wer_scorer.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for WERScorer.""" + +from pathlib import Path +from unittest.mock import MagicMock + +import msgspec +import pandas as pd +import pytest +from inference_endpoint.core.record import EventRecord, EventType, SampleEventType +from inference_endpoint.core.types import TextModelOutput +from inference_endpoint.evaluation import scoring +from inference_endpoint.evaluation.scoring import Scorer, WERScorer +from transformers.models.whisper.english_normalizer import EnglishTextNormalizer + +DATASET_NAME = "whisper_librispeech" + +REFERENCES = [ + "mister quilter is the apostle of the middle classes", + "and we are glad to welcome his gospel", + "nor is mister quilter's manner less interesting than his matter", +] + + +@pytest.fixture +def hermetic_normalizer(monkeypatch: pytest.MonkeyPatch): + """Use the real normalizer with an empty spelling map - no HF download.""" + monkeypatch.setattr( + WERScorer, "_normalizer", lambda self: EnglishTextNormalizer({}) + ) + + +@pytest.fixture +def dataset(): + ds = MagicMock() + ds.dataframe = pd.DataFrame({"transcript": REFERENCES}) + ds.num_samples.return_value = len(REFERENCES) + return ds + + +def _stage(report_dir: Path, outputs: dict[int, str]) -> Path: + """Write sample_idx_map for all references and events for `outputs` only. + + Keys are dataset sample indices; a reference without an entry stands in for + a sample that never produced a COMPLETE event. + """ + report_dir.mkdir(exist_ok=True) + uuids = {i: f"uuid-{i}" for i in range(len(REFERENCES))} + (report_dir / "sample_idx_map.json").write_bytes( + msgspec.json.encode({DATASET_NAME: {u: i for i, u in uuids.items()}}) + ) + encoder = msgspec.json.Encoder(enc_hook=EventType.encode_hook) + with (report_dir / "events.jsonl").open("wb") as f: + for index, text in outputs.items(): + rec = EventRecord( + event_type=SampleEventType.COMPLETE, + sample_uuid=uuids[index], + data=TextModelOutput(output=text), + ) + f.write(encoder.encode(rec) + b"\n") + return report_dir + + +def _scorer(report_dir: Path, dataset, **kwargs) -> WERScorer: + return WERScorer( + DATASET_NAME, + dataset, + report_dir, + ground_truth_column="transcript", + **kwargs, + ) + + +@pytest.mark.unit +class TestRegistration: + def test_registered(self): + assert "wer" in Scorer.available_scorers() + assert Scorer.get("wer") is WERScorer + + def test_requires_no_extractor(self): + assert WERScorer.REQUIRES_EXTRACTOR is False + + def test_runs_on_endpoint_outputs(self): + assert WERScorer.SKIP_ENDPOINT_PHASE is False + + +@pytest.mark.unit +@pytest.mark.usefixtures("hermetic_normalizer") +class TestScore: + def test_perfect_transcription_scores_100(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", dict(enumerate(REFERENCES))) + score, n_repeats = _scorer(report_dir, dataset).score() + assert score == pytest.approx(100.0) + assert n_repeats == 1 + + def test_casing_and_punctuation_do_not_count_as_errors( + self, tmp_path: Path, dataset + ): + outputs = {i: f"{r.capitalize()}." for i, r in enumerate(REFERENCES)} + report_dir = _stage(tmp_path / "r", outputs) + assert _scorer(report_dir, dataset).score()[0] == pytest.approx(100.0) + + def test_one_substitution_over_the_corpus(self, tmp_path: Path, dataset): + outputs = dict(enumerate(REFERENCES)) + outputs[0] = REFERENCES[0].replace("apostle", "epistle") + report_dir = _stage(tmp_path / "r", outputs) + + scorer = _scorer(report_dir, dataset) + score, _ = scorer.score() + breakdown = scorer.score_breakdown() + assert breakdown is not None + + assert breakdown["edit_distance"] == 1 + # Corpus-level: 1 error over every reference word, not a per-sample mean. + assert breakdown["wer_percent"] == pytest.approx( + 100.0 / breakdown["reference_words"] + ) + assert score == pytest.approx(100.0 - breakdown["wer_percent"]) + + def test_empty_transcription_counts_every_word_as_an_error( + self, tmp_path: Path, dataset + ): + report_dir = _stage(tmp_path / "r", {i: "" for i in range(len(REFERENCES))}) + scorer = _scorer(report_dir, dataset) + score, _ = scorer.score() + assert score == pytest.approx(0.0) + breakdown = scorer.score_breakdown() + assert breakdown is not None + assert breakdown["empty_responses"] == len(REFERENCES) + + def test_concatenated_words_are_credited(self, tmp_path: Path, dataset): + ds = MagicMock() + ds.dataframe = pd.DataFrame({"transcript": ["i can not go"]}) + ds.num_samples.return_value = 1 + report_dir = _stage(tmp_path / "r", {}) + (report_dir / "sample_idx_map.json").write_bytes( + msgspec.json.encode({DATASET_NAME: {"uuid-0": 0}}) + ) + encoder = msgspec.json.Encoder(enc_hook=EventType.encode_hook) + (report_dir / "events.jsonl").write_bytes( + encoder.encode( + EventRecord( + event_type=SampleEventType.COMPLETE, + sample_uuid="uuid-0", + data=TextModelOutput(output="I cannot go."), + ) + ) + + b"\n" + ) + assert _scorer(report_dir, ds).score()[0] == pytest.approx(100.0) + + +@pytest.mark.unit +@pytest.mark.usefixtures("hermetic_normalizer") +class TestPartialRuns: + def test_missing_samples_are_reported(self, tmp_path: Path, dataset): + # Only 2 of 3 issued samples completed. + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0], 1: REFERENCES[1]}) + scorer = _scorer(report_dir, dataset) + score, _ = scorer.score() + assert score == pytest.approx(100.0) + breakdown = scorer.score_breakdown() + assert breakdown is not None + assert breakdown["missing_samples"] == 1 + + def test_no_completions_yields_no_score(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {}) + scorer = _scorer(report_dir, dataset) + score, _ = scorer.score() + assert score is None + assert scorer.score_breakdown() == { + "wer_percent": None, + "missing_samples": len(REFERENCES), + } + + +@pytest.mark.unit +@pytest.mark.usefixtures("hermetic_normalizer") +class TestContract: + def test_score_single_sample_is_not_usable(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + with pytest.raises(RuntimeError, match="corpus-level"): + _scorer(report_dir, dataset).score_single_sample("a", "b") + + def test_breakdown_is_none_before_scoring(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + assert _scorer(report_dir, dataset).score_breakdown() is None + + def test_default_normalizer_source(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {}) + scorer = _scorer(report_dir, dataset) + assert scorer.normalizer_source == "openai/whisper-large-v3" + + def test_normalizer_source_override(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {}) + scorer = _scorer(report_dir, dataset, normalizer_source="/local/whisper") + assert scorer.normalizer_source == "/local/whisper" + + +@pytest.mark.unit +class TestNormalizerResolution: + def test_tokenizer_without_spelling_map_is_rejected( + self, tmp_path: Path, dataset, monkeypatch: pytest.MonkeyPatch + ): + tokenizer = MagicMock(spec=[]) # no english_spelling_normalizer attribute + monkeypatch.setattr( + scoring.AutoTokenizer, "from_pretrained", lambda *a, **k: tokenizer + ) + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + with pytest.raises(ValueError, match="english_spelling_normalizer"): + _scorer(report_dir, dataset)._normalizer() diff --git a/tests/unit/openai/test_transcriptions_adapter.py b/tests/unit/openai/test_transcriptions_adapter.py new file mode 100644 index 000000000..584c9b030 --- /dev/null +++ b/tests/unit/openai/test_transcriptions_adapter.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for OpenAITranscriptionsAdapter.""" + +from pathlib import Path + +import msgspec +import pandas as pd +import pytest +from inference_endpoint.config.schema import ModelParams +from inference_endpoint.core.types import Query +from inference_endpoint.dataset_manager.transforms import apply_transforms +from inference_endpoint.openai.transcriptions_adapter import ( + OpenAITranscriptionsAdapter as Adapter, +) + +WAV_BYTES = b"RIFF\x24\x00\x00\x00WAVEfmt " + bytes(range(256)) + + +@pytest.fixture +def audio_file(tmp_path: Path) -> Path: + path = tmp_path / "clip-0001.wav" + path.write_bytes(WAV_BYTES) + return path + + +def _query(audio_file: Path, **extra) -> Query: + data = { + "audio_file": str(audio_file), + "model": "openai/whisper-large-v3", + "language": "en", + "response_format": "json", + "temperature": 0.0, + } + data.update(extra) + return Query(id="q1", data=data) + + +@pytest.mark.unit +class TestContentType: + def test_content_type_is_multipart_with_boundary(self): + assert Adapter.CONTENT_TYPE.startswith("multipart/form-data; boundary=") + + def test_declared_boundary_matches_every_encoded_body(self, audio_file: Path): + # The worker folds Content-Type into a prefix built once at setup, so a + # body whose boundary differed from the declared one would be unparseable. + boundary = Adapter.CONTENT_TYPE.split("boundary=", 1)[1].encode() + for _ in range(2): + body = Adapter.encode_query(_query(audio_file)) + assert body.startswith(b"--" + boundary + b"\r\n") + + +@pytest.mark.unit +class TestEncodeQuery: + def test_body_contains_audio_bytes_verbatim(self, audio_file: Path): + body = Adapter.encode_query(_query(audio_file)) + assert WAV_BYTES in body + + def test_body_carries_filename_and_part_content_type(self, audio_file: Path): + body = Adapter.encode_query(_query(audio_file)) + assert b'name="file"; filename="clip-0001.wav"' in body + assert b"Content-Type: audio/wav" in body + + def test_body_carries_form_fields(self, audio_file: Path): + body = Adapter.encode_query(_query(audio_file)) + for name, value in ( + ("model", b"openai/whisper-large-v3"), + ("language", b"en"), + ("response_format", b"json"), + ("temperature", b"0.0"), + ): + assert f'name="{name}"'.encode() in body + assert value in body + + def test_body_is_closed_with_terminating_boundary(self, audio_file: Path): + boundary = Adapter.CONTENT_TYPE.split("boundary=", 1)[1] + body = Adapter.encode_query(_query(audio_file)) + assert body.endswith(f"--{boundary}--\r\n".encode()) + + def test_absent_optional_fields_are_omitted(self, audio_file: Path): + query = Query(id="q", data={"audio_file": str(audio_file), "model": "m"}) + body = Adapter.encode_query(query) + assert b'name="language"' not in body + assert b'name="temperature"' not in body + + def test_none_valued_fields_are_omitted(self, audio_file: Path): + # AddStaticColumns materialises model_params defaults that may be None; + # sending "None" as a form value would be rejected by the server. + query = _query(audio_file, temperature=None) + assert b'name="temperature"' not in Adapter.encode_query(query) + + def test_missing_audio_file_raises_keyerror(self): + with pytest.raises(KeyError, match="audio_file"): + Adapter.encode_query(Query(id="q", data={"model": "m"})) + + def test_streaming_request_raises(self, audio_file: Path): + with pytest.raises(ValueError, match="non-streaming"): + Adapter.encode_query(_query(audio_file, stream=True)) + + def test_missing_audio_on_disk_raises(self, tmp_path: Path): + query = Query( + id="q", data={"audio_file": str(tmp_path / "gone.wav"), "model": "m"} + ) + with pytest.raises(FileNotFoundError): + Adapter.encode_query(query) + + +@pytest.mark.unit +class TestDecodeResponse: + def test_decodes_text_field(self): + result = Adapter.decode_response(b'{"text": "hello world"}', "q1") + assert result.id == "q1" + assert result.error is None + assert str(result.response_output) == "hello world" + + def test_ignores_server_specific_extras(self): + body = b'{"text": "hi", "duration": 29.9, "language": "english"}' + assert str(Adapter.decode_response(body, "q").response_output) == "hi" + + def test_missing_text_raises(self): + with pytest.raises(msgspec.ValidationError): + Adapter.decode_response(b'{"duration": 1.0}', "q") + + def test_malformed_json_raises(self): + with pytest.raises(msgspec.DecodeError): + Adapter.decode_response(b"not json", "q") + + +@pytest.mark.unit +class TestSse: + def test_decode_sse_message_raises(self): + with pytest.raises(NotImplementedError): + Adapter.decode_sse_message(b'{"text": "x"}') + + +@pytest.mark.unit +class TestDatasetTransforms: + def _apply(self, df: pd.DataFrame, **model_kwargs) -> pd.DataFrame: + params = ModelParams(name="openai/whisper-large-v3", **model_kwargs) + return apply_transforms(df, Adapter.dataset_transforms(params)) + + def test_drops_ground_truth_columns_from_the_request_row(self): + df = pd.DataFrame( + [{"audio_file": "/a.wav", "transcript": "hello", "duration": 29.5}] + ) + out = self._apply(df) + assert "transcript" not in out.columns + assert "duration" not in out.columns + assert out.iloc[0]["audio_file"] == "/a.wav" + + def test_adds_model_and_response_format(self): + df = pd.DataFrame([{"audio_file": "/a.wav", "transcript": "hello"}]) + row = self._apply(df, temperature=0.0).iloc[0] + assert row["model"] == "openai/whisper-large-v3" + assert row["response_format"] == "json" + assert row["temperature"] == 0.0 + + def test_dataset_language_overrides_defaults(self): + df = pd.DataFrame( + [{"audio_file": "/a.wav", "transcript": "hola", "language": "es"}] + ) + assert self._apply(df).iloc[0]["language"] == "es" + + def test_missing_audio_file_column_raises(self): + df = pd.DataFrame([{"transcript": "hello"}]) + with pytest.raises(KeyError, match="audio_file"): + self._apply(df) diff --git a/tests/unit/openai/test_transcriptions_registration.py b/tests/unit/openai/test_transcriptions_registration.py new file mode 100644 index 000000000..92546784f --- /dev/null +++ b/tests/unit/openai/test_transcriptions_registration.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests that APIType.OPENAI_TRANSCRIPTIONS is registered and wired up correctly.""" + +from importlib import import_module + +import pytest +from inference_endpoint.core.types import APIType +from inference_endpoint.endpoint_client.config import ( + ACCUMULATOR_MAP, + ADAPTER_MAP, + HTTPClientConfig, +) + + +def _resolve(path: str) -> type: + module_path, class_name = path.rsplit(".", 1) + return getattr(import_module(module_path), class_name) + + +@pytest.mark.unit +def test_api_type_exists(): + assert APIType.OPENAI_TRANSCRIPTIONS == "openai_transcriptions" + + +@pytest.mark.unit +def test_default_route(): + assert APIType.OPENAI_TRANSCRIPTIONS.default_route() == "v1/audio/transcriptions" + + +@pytest.mark.unit +def test_in_adapter_map(): + path = ADAPTER_MAP[APIType.OPENAI_TRANSCRIPTIONS] + assert _resolve(path).__name__ == "OpenAITranscriptionsAdapter" + + +@pytest.mark.unit +def test_in_accumulator_map(): + # An omitted entry silently falls back to the OpenAI SSE accumulator, which + # would surface zero-output queries as successful on a non-streaming route. + path = ACCUMULATOR_MAP[APIType.OPENAI_TRANSCRIPTIONS] + assert _resolve(path).__name__ == "NonStreamingAccumulator" + + +@pytest.mark.unit +def test_client_config_resolves_the_pair(): + config = HTTPClientConfig(api_type=APIType.OPENAI_TRANSCRIPTIONS) + assert config.adapter.__name__ == "OpenAITranscriptionsAdapter" + assert config.accumulator.__name__ == "NonStreamingAccumulator" + + +@pytest.mark.unit +def test_adapter_declares_multipart_content_type(): + adapter = _resolve(ADAPTER_MAP[APIType.OPENAI_TRANSCRIPTIONS]) + assert adapter.CONTENT_TYPE.startswith("multipart/form-data; boundary=") + + +@pytest.mark.unit +def test_other_adapters_still_declare_json(): + for api_type in (APIType.OPENAI, APIType.OPENAI_COMPLETIONS, APIType.SGLANG): + adapter = _resolve(ADAPTER_MAP[api_type]) + assert adapter.CONTENT_TYPE == "application/json" From eaba98e0374dac60b012c285f43f9dbf9a025509 Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Mon, 2 Nov 2026 10:37:14 -0800 Subject: [PATCH 2/4] fix(whisper): address review on the speech2text workload Enforce in code what the config comments only asserted, and decide the two semantics the review flagged as underspecified. - `language` is now actually sent. `dataset_transforms` seeded only model/response_format/temperature, and the manifest has no `language` column, so every request fell back to the server's per-clip auto-detection rather than the reference's pinned decoder language. `DEFAULT_LANGUAGE` supplies it; `overwrite=False` still lets a dataset column win. - A missing sample is scored as an empty hypothesis instead of being dropped. Scoring iterated the returned rows, so a sample with no COMPLETE event left the denominator and dropping the samples a SUT handles worst would raise the score. `WERScorer` now walks the issued samples and sets `complete = False` when any is missing, so a partial run is machine-detectable rather than needing a reader to notice `missing_samples`. - Three api_type constraints move from YAML comments into `_resolve_and_validate`: `streaming: on` is an error (AUTO resolves OFF even online), `metrics_isl` is forced False, and `warmup.salt` is an error. The unreachable `stream` guard in `encode_query` is gone. - `encode_multipart` escapes the dataset-controlled header parameters and rejects a field value carrying the delimiter, so a row cannot forge a part. - `WERScorer.preflight` resolves the normalizer before the run. It needs HF network access or a warm cache, and scoring runs after the whole performance phase, so a cold cache failed the run at its last step with the endpoint work already spent. A configured `extractor` is now rejected rather than silently ignored. - `download_from_r2` is the single R2 path: BFCLv4 and OpenOrca lose their copies of the pinned commit, host allowlist and subprocess plumbing. It gains a timeout, because dataset generation runs in `setup_benchmark` before the whole-run deadline is checked, so nothing else would interrupt a stalled multi-GB fetch. - Register the workload for submission checking: `Whisper_large_v3` (97.9329 at 0.99) and `LibriSpeechDevAll` (1633), both transcribed from the inference submission checker's constants, plus an Offline-only `PerModelRuleset`. The numbers no longer live only in Markdown. - Pin WER parity with `UPSTREAM_VECTORS`: vectors captured from the MLCommons reference at commit 3fbc3299, covering joins in both directions, a join past the 3-word span, and the empty edges. The lowercasing divergence is documented as a no-op on the official corpus (upstream's preparation pipeline already lowercases) kept as defence for a hand-built manifest. Docstring corrections where the code did not do what the prose claimed: the non-streaming accumulators described a mechanism that does not exist (a body with no `data:` frames never reaches `decode_sse_message`, so nothing raises and nothing accumulates), and the adapter now says the audio read is synchronous on the worker's event loop, which stalls every in-flight response in that process rather than only its own request. `probe.py` folds its two unsupported-api_type blocks into one map, `wer_counts` skips the k=1 concatenation probe the branch above already settled, and `duration` is documented as inspection-only. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 34 ++--- examples/12_Whisper_Example/README.md | 64 ++++++--- .../12_Whisper_Example/offline_whisper.yaml | 21 ++- src/inference_endpoint/commands/probe.py | 33 +++-- .../config/rulesets/mlcommons/datasets.py | 12 ++ .../config/rulesets/mlcommons/models.py | 16 +++ .../config/rulesets/mlcommons/rules.py | 17 +++ src/inference_endpoint/config/schema.py | 37 ++++- .../dataset_manager/dataset.py | 39 ++++- .../predefined/bfcl_v4/__init__.py | 55 +------ .../predefined/open_orca/__init__.py | 32 +---- .../whisper_librispeech/__init__.py | 8 +- .../endpoint_client/accumulator_protocol.py | 16 ++- src/inference_endpoint/evaluation/scoring.py | 69 ++++++--- src/inference_endpoint/evaluation/wer.py | 31 ++-- .../openai/transcriptions_adapter.py | 65 +++++++-- src/inference_endpoint/videogen/adapter.py | 8 +- .../openai/test_transcriptions_adapter.py | 9 +- tests/unit/config/test_schema.py | 92 ++++++++++++ .../dataset_manager/test_bfcl_v4_dataset.py | 46 ++++-- .../test_whisper_librispeech_dataset.py | 43 ++++++ tests/unit/evaluation/test_wer.py | 77 +++++++++- tests/unit/evaluation/test_wer_scorer.py | 135 ++++++++++++++++-- .../openai/test_transcriptions_adapter.py | 50 ++++++- 24 files changed, 782 insertions(+), 227 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e2d533d2d..2cbcc8b20 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,23 +87,23 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint ### Key Components -| Component | Location | Purpose | -| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Load Generator** | `src/inference_endpoint/load_generator/` | Central orchestrator: `BenchmarkSession` owns the lifecycle, `PhaseIssuer` drives per-phase execution, `TimedIssueStrategy`/`BurstStrategy`/`ConcurrencyStrategy` control timing. Emits `ERROR` before `COMPLETE` for failed queries (metrics aggregator depends on this order). | -| **Endpoint Client** | `src/inference_endpoint/endpoint_client/` | Multi-process HTTP workers communicating via ZMQ IPC. `HTTPEndpointClient` is the main entry point | -| **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface | -| **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | -| **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` builds reports from the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Sets `complete = (state == "complete" and n_pending_tasks == 0)` and renders an explicit warning for `INTERRUPTED` runs. Keeps the steady-window verdict on `Report.steady_state` and includes it in `to_json` (unlike `accuracy`). | -| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`) — the declared user surface, distinct from the resolved `runtime_settings.py`; `Timeouts` (`config/schema.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_issue_duration_ms` stays on `settings.runtime`; client worker-lifecycle timeouts stay on `settings.client`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | -| **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | -| **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | -| **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | -| **DeepSeek-R1 (MLPerf)** | `src/inference_endpoint/evaluation/scoring.py` (`LegacyMLPerfDeepSeekR1Scorer`), `examples/07_DeepSeekR1_Example/` | MLPerf DeepSeek-R1 accuracy. TensorRT-LLM is OpenAI-compatible, so it is served via `api_type: openai` / `openai_completions` (no dedicated trtllm adapter). The combined multi-subset eval (`math500`/`aime`/`gpqa`/`mmlu_pro`/`livecodebench`) is the official MLCommons `eval_accuracy.py`, run out-of-process via `uv run --project` against the isolated subproject at `src/inference_endpoint/evaluation/legacy_mlperf_deepseek_r1/` (a uv subproject excluded from the parent wheel; mirrors the VBench pattern). The example feeds the exact MLPerf prompt via pre-tokenized `input_tokens` to `/v1/completions`. | -| **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | -| **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | -| **Whisper (MLPerf speech2text)** | `src/inference_endpoint/openai/transcriptions_adapter.py`, `dataset_manager/predefined/whisper_librispeech/`, `evaluation/wer.py` | MLPerf speech2text (whisper-large-v3 / LibriSpeech). `api_type: openai_transcriptions` posts a `multipart/form-data` audio upload to `/v1/audio/transcriptions` — the only route that serves Whisper on vLLM. The body is not JSON, so the adapter declares `HttpRequestAdapter.CONTENT_TYPE` and the worker folds it into the pre-encoded request prefix once at setup (`endpoint_client/http.py`); the multipart boundary is therefore a module constant. `Query.data` carries the clip's _path_ and the worker reads the bytes in `encode_query`, so the corpus never crosses the ZMQ frame. Non-streaming (`NonStreamingAccumulator`). The `whisper_librispeech` dataset downloads the MLCommons-prepared `dev-all-repack.json` corpus via `download_from_r2` (`generate_params.manifest` overrides); the ~30 s repackaging stays upstream because different clip boundaries change the WER. `WERScorer` (`eval_method: wer`) reports `100 - WER%` corpus-wide using the ported MLPerf metric in `evaluation/wer.py` plus `transformers`' `EnglishTextNormalizer`. Needs `metrics_isl: false` (no text prompt) and `warmup.salt: false`. See examples/12_Whisper_Example/. | -| **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | -| **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | +| Component | Location | Purpose | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Load Generator** | `src/inference_endpoint/load_generator/` | Central orchestrator: `BenchmarkSession` owns the lifecycle, `PhaseIssuer` drives per-phase execution, `TimedIssueStrategy`/`BurstStrategy`/`ConcurrencyStrategy` control timing. Emits `ERROR` before `COMPLETE` for failed queries (metrics aggregator depends on this order). | +| **Endpoint Client** | `src/inference_endpoint/endpoint_client/` | Multi-process HTTP workers communicating via ZMQ IPC. `HTTPEndpointClient` is the main entry point | +| **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface; `download_from_r2()` is the single path for fetching MLCommons-hosted corpora (pinned downloader commit + host allowlist + subprocess timeout; `whisper_librispeech`, `bfcl_v4` and `open_orca` all go through it). It needs its own timeout because dataset generation runs in `setup_benchmark`, before `RunWatchdog` arms the whole-run deadline | +| **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | +| **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` builds reports from the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Sets `complete = (state == "complete" and n_pending_tasks == 0)` and renders an explicit warning for `INTERRUPTED` runs. Keeps the steady-window verdict on `Report.steady_state` and includes it in `to_json` (unlike `accuracy`). | +| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`) — the declared user surface, distinct from the resolved `runtime_settings.py`; `Timeouts` (`config/schema.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_issue_duration_ms` stays on `settings.runtime`; client worker-lifecycle timeouts stay on `settings.client`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | +| **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | +| **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | +| **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | +| **DeepSeek-R1 (MLPerf)** | `src/inference_endpoint/evaluation/scoring.py` (`LegacyMLPerfDeepSeekR1Scorer`), `examples/07_DeepSeekR1_Example/` | MLPerf DeepSeek-R1 accuracy. TensorRT-LLM is OpenAI-compatible, so it is served via `api_type: openai` / `openai_completions` (no dedicated trtllm adapter). The combined multi-subset eval (`math500`/`aime`/`gpqa`/`mmlu_pro`/`livecodebench`) is the official MLCommons `eval_accuracy.py`, run out-of-process via `uv run --project` against the isolated subproject at `src/inference_endpoint/evaluation/legacy_mlperf_deepseek_r1/` (a uv subproject excluded from the parent wheel; mirrors the VBench pattern). The example feeds the exact MLPerf prompt via pre-tokenized `input_tokens` to `/v1/completions`. | +| **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | +| **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | +| **Whisper (MLPerf speech2text)** | `src/inference_endpoint/openai/transcriptions_adapter.py`, `dataset_manager/predefined/whisper_librispeech/`, `evaluation/wer.py` | MLPerf speech2text (whisper-large-v3 / LibriSpeech). `api_type: openai_transcriptions` posts a `multipart/form-data` audio upload to `/v1/audio/transcriptions` — the only route that serves Whisper on vLLM. The body is not JSON, so the adapter declares `HttpRequestAdapter.CONTENT_TYPE` and the worker folds it into the pre-encoded request prefix once at setup (`endpoint_client/http.py`); the multipart boundary is therefore a module constant. `Query.data` carries the clip's _path_ and the worker reads the bytes in `encode_query`, so the corpus never crosses the ZMQ frame. Dataset-controlled header params are percent-escaped and a field value carrying the delimiter is rejected, so a row cannot forge a part. Non-streaming (`NonStreamingAccumulator`). `language` is sent on every request (`DEFAULT_LANGUAGE`, a dataset column overrides it): the reference pins the decoder language instead of auto-detecting per clip. The `whisper_librispeech` dataset downloads the MLCommons-prepared `dev-all-repack.json` corpus via `download_from_r2` (`generate_params.manifest` overrides); the ~30 s repackaging stays upstream because different clip boundaries change the WER. `WERScorer` (`eval_method: wer`) reports `100 - WER%` corpus-wide using the ported MLPerf metric in `evaluation/wer.py` plus `transformers`' `EnglishTextNormalizer`; parity with upstream is pinned by `UPSTREAM_VECTORS` in `tests/unit/evaluation/test_wer.py`. An issued sample with no COMPLETE event scores as an empty transcript (so dropping samples cannot raise the score) and sets `complete = False`. A configured `extractor` is rejected, and `preflight` resolves the normalizer before the run rather than after the perf phase. `_resolve_and_validate` keys three constraints on this api_type: `streaming: on` is an error (AUTO resolves OFF even online), `metrics_isl` is forced False (no text prompt), and `warmup.salt` is an error. Registered for submission checking as `models.Whisper_large_v3` / `datasets.LibriSpeechDevAll`. See examples/12_Whisper_Example/. | +| **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | +| **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | | **Validation** | `src/inference_endpoint/validation/`, `docs/validation/` | Cohort checker for submission uploads. A cohort is a directory of YAML checks (four scope files + `valid_refs.yaml` + a `cohort.yaml` naming the packaged ruleset); `python -m inference_endpoint.validation COHORT SUBMISSION` runs them against an upload root (`results///r/`) and reports holds/failures per check. Exit codes: 0 holds, 1 blocking failure, 2 refusal. See docs/validation/README.md. | ### Hot-Path Architecture diff --git a/examples/12_Whisper_Example/README.md b/examples/12_Whisper_Example/README.md index fc80dfd60..1794e8790 100644 --- a/examples/12_Whisper_Example/README.md +++ b/examples/12_Whisper_Example/README.md @@ -47,8 +47,15 @@ To use a copy you already have, set `datasets[*].generate_params.manifest` or `generate_params.data_dir` overrides the root that `fname` resolves against; it defaults to the manifest's own directory. -Keep the WAVs on **fast local storage**: the client reads each file inside the -measured request, so a network mount shows up as endpoint latency. +The parsed-row cache is keyed on the dataset ID alone, so if you change +`manifest` or `data_dir` after a first run, also pass +`generate_params.force: true` once — otherwise the rows built from the previous +manifest are reused. + +Keep the WAVs on **fast local storage**. The worker reads each file inside the +measured request, and that read is synchronous on its event loop: a slow read +stalls every other in-flight request in the same worker process, so a network +mount or cold cache inflates latency beyond the clip that caused it. ## 3. Run @@ -72,6 +79,7 @@ prefix of the full corpus — but a partial perf phase is not a reportable resul ```json { "score": 97.93, + "complete": true, "breakdown": { "wer_percent": 2.07, "edit_distance": 1241, @@ -82,10 +90,12 @@ prefix of the full corpus — but a partial perf phase is not a reportable resul } ``` -**Check `missing_samples == 0` before quoting the score.** WER is corpus-level -(total errors ÷ total reference words), so a sample that never completed -contributes neither and silently shrinks the denominator instead of counting -against you. +A submission needs `complete: true` and `missing_samples: 0`. An issued sample +that never produced a response is scored as an empty transcript — its reference +words count in the denominator and all of them count as errors — so a partial +run shows up as a depressed score rather than as a flattering one, and +`complete` goes `false`. `empty_responses` is the separate case of a sample that +did answer, with nothing in it. A WER far above 2.07% usually points at the dataset rather than the endpoint — most often a manifest that is not the repackaged `dev-all-repack.json`. @@ -97,16 +107,32 @@ most often a manifest that is not the repackaged `dev-all-repack.json`. | Route + multipart upload | `openai/transcriptions_adapter.py` (`api_type: openai_transcriptions`) | | Dataset | `dataset_manager/predefined/whisper_librispeech/` | | Metric | `evaluation/wer.py` + `WERScorer` (`eval_method: wer`) | - -Notes on the run config: - -- **Non-streaming.** OpenAI does not support `stream` for Whisper-class models, - so `model_params.streaming` must be `off`. There is no TTFT/TPOT; the metrics - are QPS and per-request latency. -- **`metrics_isl: false`.** Audio rows carry no text prompt, so there is no - input sequence length. One "ISL is unavailable" warning per run is expected. -- **Warmup salt must stay off.** Salting prepends a random hex prefix to a text - prompt; an audio row has none, and enabling it is a hard error. -- **`language: en`** (settable per dataset row) is what makes the server build - Whisper's `<|startoftranscript|><|en|><|transcribe|><|notimestamps|>` prompt, - matching the MLPerf reference. +| Accuracy gate | `config/rulesets/mlcommons/models.py` (`Whisper_large_v3`) | + +Notes on the run config. All three constraints below are enforced by config +validation keyed on `api_type: openai_transcriptions`, so a hand-written config +gets an error rather than a silently odd run: + +- **Non-streaming.** OpenAI does not support `stream` for Whisper-class models. + `streaming: on` is rejected and `auto` resolves to `off` even in online mode. + There is no TTFT/TPOT; the metrics are QPS and per-request latency. +- **`metrics_isl` is forced off.** Audio rows carry no text prompt, so there is + no input sequence length. One "ISL is unavailable" warning per run is + expected. +- **Warmup salt is rejected.** Salting prepends a random hex prefix to a text + prompt to defeat prefix caching; an audio upload has no prompt, and its cache + key is the audio. +- **`language: en`** is sent on every request (the adapter's `DEFAULT_LANGUAGE`; + a dataset `language` column overrides it). It makes the server build Whisper's + `<|startoftranscript|><|en|><|transcribe|><|notimestamps|>` prompt instead of + auto-detecting the language per clip, which is what the MLPerf reference does. + +## Submission checking + +The reference accuracy and the 0.99 factor are registered in +`config/rulesets/mlcommons/models.py`, so `submission_ref` and +`compliance/checker.py` can enforce them rather than leaving the numbers in +prose. Both are transcriptions of the MLPerf Inference submission checker's own +constants, where the gate is `"whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99)` +and the required sample count is 1633 — the same `100 - WER` scale `WERScorer` +reports on. diff --git a/examples/12_Whisper_Example/offline_whisper.yaml b/examples/12_Whisper_Example/offline_whisper.yaml index 607eb74ff..80bbd16dd 100644 --- a/examples/12_Whisper_Example/offline_whisper.yaml +++ b/examples/12_Whisper_Example/offline_whisper.yaml @@ -25,8 +25,15 @@ type: "offline" model_params: name: "openai/whisper-large-v3" - streaming: "off" # /v1/audio/transcriptions has no SSE mode for Whisper + # /v1/audio/transcriptions has no SSE mode for Whisper. Stated here for the + # reader; config validation also rejects "on" for this api_type and resolves + # "auto" to "off", so an online whisper config cannot silently stream. + streaming: "off" temperature: 0.0 + # The transcription language is sent on every request (language=en, from the + # adapter's DEFAULT_LANGUAGE) so the server builds Whisper's pinned-language + # decoder prompt instead of auto-detecting per clip, which is what the MLPerf + # reference does. A dataset `language` column overrides it per row. datasets: # setup_benchmark() requires a performance dataset to drive issuance; the @@ -55,15 +62,17 @@ settings: num_workers: 8 warmup: - # Warmup salt requires text-'prompt' samples; audio rows have none, so - # enabling warmup here needs salt left off. + # Salt prepends random hex to a text prompt to defeat prefix caching; an + # audio upload has no prompt, so config validation rejects salt for this + # api_type. Warmup itself is fine with salt off. enabled: false salt: false # Audio rows carry no text prompt, so there is no input sequence length to - # measure; this stops the aggregator registering an ISL series that would - # stay empty. The load generator still logs one "ISL is unavailable" warning - # per run, which is expected here. OSL and latency are unaffected. + # measure. Validation forces this off for openai_transcriptions, so the line + # is documentation rather than a requirement. The load generator still logs + # one "ISL is unavailable" warning per run, which is expected here. OSL and + # latency are unaffected. metrics_isl: false endpoint_config: diff --git a/src/inference_endpoint/commands/probe.py b/src/inference_endpoint/commands/probe.py index c81401361..c9b1125a2 100644 --- a/src/inference_endpoint/commands/probe.py +++ b/src/inference_endpoint/commands/probe.py @@ -39,6 +39,16 @@ logger = logging.getLogger(__name__) +#: api_types the probe cannot exercise, mapped to the reason shown to the user. +#: Each one breaks an assumption the probe is built on: a text prompt in, chat +#: tokens out, and a response inside the 60 s probe timeout. +UNSUPPORTED_PROBE_API_TYPES: dict[APIType, str] = { + APIType.VIDEOGEN: "per-request latencies exceed the probe timeout", + APIType.OPENAI_TRANSCRIPTIONS: ( + "the request body is an audio upload, not a text prompt" + ), +} + @cyclopts.Parameter(name="*") class ProbeConfig(BaseModel): @@ -87,24 +97,13 @@ async def _probe_async(config: ProbeConfig) -> None: test_prompt = config.prompt api_type = config.api_type - # Probe assumes second-scale latencies (probe_timeout=60s below) and - # text prompt/response semantics — neither holds for video generation, - # where each request takes minutes and there are no chat tokens to - # display. Reject upfront rather than emitting a misleading - # "0/N requests successful" failure after the timeout. - if api_type == APIType.VIDEOGEN: - raise InputValidationError( - "Probe does not support api_type=videogen " - "(per-request latencies exceed the probe timeout). " - "Use a dedicated health check or a benchmark from-config run instead." - ) - - # Transcription requests are an audio upload keyed on an `audio_file` path, - # so there is no text prompt the probe could send. - if api_type == APIType.OPENAI_TRANSCRIPTIONS: + # Probe assumes second-scale latencies (probe_timeout=60s below) and text + # prompt/response semantics. Reject upfront rather than emitting a + # misleading "0/N requests successful" failure after the timeout. + if api_type in UNSUPPORTED_PROBE_API_TYPES: raise InputValidationError( - "Probe does not support api_type=openai_transcriptions " - "(the request body is an audio upload, not a text prompt). " + f"Probe does not support api_type={api_type.value} " + f"({UNSUPPORTED_PROBE_API_TYPES[api_type]}). " "Use a dedicated health check or a benchmark from-config run instead." ) diff --git a/src/inference_endpoint/config/rulesets/mlcommons/datasets.py b/src/inference_endpoint/config/rulesets/mlcommons/datasets.py index 58762f94d..e41f4dbaf 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/datasets.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/datasets.py @@ -56,6 +56,18 @@ def __hash__(self) -> int: size=4388, ) +# speech2text. Size is the "dataset-size" entry for whisper in the inference +# submission checker's constants.py, and matches the repackaged manifest's own +# sample count (dataset_manager/predefined/whisper_librispeech). +LibriSpeechDevAll = _Dataset( + description=( + "LibriSpeech dev-clean + dev-other, converted to 16 kHz WAV and " + "repackaged by the MLCommons pipeline into clips approaching 30 s" + ), + size=1633, + metadata={"scorer": "wer"}, +) + # --- Edge-Agentic (BFCL v4) benchmark datasets --- BFCLv4SingleTurn = _Dataset( diff --git a/src/inference_endpoint/config/rulesets/mlcommons/models.py b/src/inference_endpoint/config/rulesets/mlcommons/models.py index 37618cb5b..d1edd56f2 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/models.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/models.py @@ -146,6 +146,22 @@ def __hash__(self) -> int: dataset=datasets.TextGenComplex, ) +# speech2text. The inference submission checker's constants.py gates one metric, +# which it names ACCURACY and defines as 100 - WER over the corpus: +# "whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99). So golden WER 2.0671% is a +# golden accuracy of 97.9329 with the usual 0.99 factor, and WERScorer reports on +# that same 100 - WER scale. +# +# equal_issue_mode is off: the clips are fixed-length (~30 s) by construction, so +# the non-uniform-input case it exists for does not arise. +Whisper_large_v3 = _Model( + "whisper-large-v3", + golden_accuracy=("fp32", {"accuracy": 97.9329}), + accuracy_target_settings=[{"accuracy": (0.99,)}], + dataset=datasets.LibriSpeechDevAll, + use_equal_issue_mode=False, +) + # Edge-Agentic (BFCL v4) reference model. The accuracy gate is BFCL v4 single-turn # (overall + non_live-normalized). golden_accuracy here is the validated Q4_K_M # reference point on a Jetson Thor (llama.cpp, reasoning off); accuracy is diff --git a/src/inference_endpoint/config/rulesets/mlcommons/rules.py b/src/inference_endpoint/config/rulesets/mlcommons/rules.py index 86cfe0c57..a4de7d3a7 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/rules.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/rules.py @@ -272,6 +272,23 @@ def apply_user_config( max_ttft_latency_ms=2000, max_tpot_latency_ms=200 ), }, + # speech2text is Offline-only upstream ("whisper": ["Offline"] in the + # submission checker's required-scenarios table), so THROUGHPUT is its + # only priority. A plain PerModelRuleset rather than a TokenBasedRuleset: + # /v1/audio/transcriptions has no SSE mode, so there is no TTFT or TPOT + # to cap or report, and samples/second is the whole metric. + # + # min_sample_count_valid is the "dataset-size" entry (1633) rather than + # the Offline "min-queries" value (1): the full corpus is what makes a + # WER comparable to the reference, and a short run is the failure mode + # worth catching here. + models.Whisper_large_v3: { + OptimizationPriority.THROUGHPUT: PerModelRuleset( + min_sample_count_valid=1633, + metric=metrics.Throughput, + reported_metrics=[metrics.Throughput], + ), + }, }, ) diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 4dabe3113..b0e7ac294 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -1171,12 +1171,24 @@ def _resolve_and_validate(self) -> Self: self.benchmark_mode if self.type == TestType.SUBMISSION else self.type ) + # An audio transcription endpoint has no SSE mode at all, so streaming is + # a property of the api_type rather than of the test type: AUTO must not + # resolve to ON here the way it does for an online text run. + non_streaming_api = ( + self.endpoint_config.api_type == APIType.OPENAI_TRANSCRIPTIONS + ) if self.model_params.streaming == StreamingMode.AUTO: mp_updates["streaming"] = ( StreamingMode.OFF - if effective_mode in (TestType.OFFLINE,) + if non_streaming_api or effective_mode in (TestType.OFFLINE,) else StreamingMode.ON ) + elif non_streaming_api and self.model_params.streaming == StreamingMode.ON: + raise ValueError( + f"endpoint_config.api_type={self.endpoint_config.api_type.value} " + "has no streaming mode; set model_params.streaming to 'off' or " + "leave it 'auto'" + ) if not self.model_params.name and self.submission_ref: mp_updates["name"] = self.submission_ref.model @@ -1191,6 +1203,29 @@ def _resolve_and_validate(self) -> Self: if not self.model_params.name: raise ValueError("Required: --model-params.name [--model]") + if non_streaming_api: + # The request body is an audio upload with no text prompt, so there + # is no input sequence length to measure. Left on, the aggregator + # registers an ISL series that can only stay empty. Derived rather + # than demanded of the user: metrics_isl defaults to True, so every + # transcription config would otherwise have to turn it off by hand. + if self.settings.metrics_isl: + object.__setattr__( + self, + "settings", + self.settings.model_copy(update={"metrics_isl": False}), + ) + # Salt exists to defeat prefix caching by prepending random hex to a + # text prompt. Here the cache key is the audio, so salting cannot do + # that job, and a row with no 'prompt' at all is a hard error later. + if self.settings.warmup.salt: + raise ValueError( + f"settings.warmup.salt is not supported with " + f"endpoint_config.api_type=" + f"{self.endpoint_config.api_type.value}: the request is an " + "audio upload, not a text prompt" + ) + uses_swe_bench = any( dataset.accuracy_config is not None and dataset.accuracy_config.eval_method == ScorerMethod.SWE_BENCH diff --git a/src/inference_endpoint/dataset_manager/dataset.py b/src/inference_endpoint/dataset_manager/dataset.py index dcd420a15..53f115ce3 100644 --- a/src/inference_endpoint/dataset_manager/dataset.py +++ b/src/inference_endpoint/dataset_manager/dataset.py @@ -29,6 +29,11 @@ import numpy as np import pandas as pd + +# `requests` is barred from the benchmark's own HTTP path, which uses the +# httptools-based ConnectionPool in endpoint_client/. This is dataset setup: +# one blocking fetch of the pinned downloader script, run before any load is +# issued and never during a measured request. import requests from datasets import load_dataset, load_from_disk @@ -269,8 +274,17 @@ def load_from_huggingface( #: pointing the downloader at an attacker-controlled location. R2_ALLOWED_HOST = "mlcommons-storage.org" +#: Wall-clock ceiling for one downloader invocation. Dataset generation runs in +#: ``setup_benchmark``, before ``RunWatchdog`` arms its whole-run deadline, so +#: nothing else would ever interrupt a stalled fetch. Sized for several GB over +#: a slow link rather than for a healthy transfer; callers that need more can +#: pass their own. +R2_DOWNLOAD_TIMEOUT_S = 2 * 60 * 60 + -def download_from_r2(uri: str, download_dir: Path) -> None: +def download_from_r2( + uri: str, download_dir: Path, timeout_s: float | None = R2_DOWNLOAD_TIMEOUT_S +) -> None: """Download an MLCommons-hosted dataset into ``download_dir``. Fetches the pinned ``mlc-r2-downloader.sh`` and runs it against the @@ -278,9 +292,14 @@ def download_from_r2(uri: str, download_dir: Path) -> None: fails its exit code on mismatch, so callers only need to locate the artifact they expect afterwards. + Args: + uri: The dataset's published ``.uri`` metadata URL. + download_dir: Directory the artifact is downloaded into. + timeout_s: Ceiling on the downloader subprocess; None waits forever. + Raises: ValueError: ``uri`` is not https on MLCommons R2 storage. - RuntimeError: the downloader exited non-zero. + RuntimeError: the downloader exited non-zero or outran ``timeout_s``. """ parsed = urlparse(uri) host = parsed.hostname or "" @@ -311,14 +330,22 @@ def download_from_r2(uri: str, download_dir: Path) -> None: stderr=subprocess.PIPE, text=True, check=False, + timeout=timeout_s, ) - if result.returncode != 0: - raise RuntimeError( - f"R2 downloader failed with code {result.returncode}: {result.stderr}" - ) + except subprocess.TimeoutExpired as e: + raise RuntimeError( + f"R2 downloader for '{uri}' exceeded {timeout_s}s and was killed; " + f"whatever it had written is left under {download_dir}. Raise the " + "timeout, or point generate_params at a local copy of the corpus." + ) from e finally: script_path.unlink(missing_ok=True) + if result.returncode != 0: + raise RuntimeError( + f"R2 downloader failed with code {result.returncode}: {result.stderr}" + ) + def _check_unsaltable(sample: Any) -> DatasetValidationError.Reason | None: """Return the Reason a sample cannot be salted, or None if it can. diff --git a/src/inference_endpoint/dataset_manager/predefined/bfcl_v4/__init__.py b/src/inference_endpoint/dataset_manager/predefined/bfcl_v4/__init__.py index f91c190ea..c12b33137 100644 --- a/src/inference_endpoint/dataset_manager/predefined/bfcl_v4/__init__.py +++ b/src/inference_endpoint/dataset_manager/predefined/bfcl_v4/__init__.py @@ -22,16 +22,13 @@ import hashlib import json import os -import subprocess from logging import getLogger from pathlib import Path from typing import Any -from urllib.parse import urlparse import pandas as pd -import requests -from ...dataset import Dataset +from ...dataset import Dataset, download_from_r2 from . import presets logger = getLogger(__name__) @@ -168,13 +165,6 @@ class BFCLv4( # staging/testing via the BFCL_V4_DATASET_URI env var. R2_DATASET_URI: str | None = None R2_DATASET_URI_ENV = "BFCL_V4_DATASET_URI" - # Pinned mlcommons/r2-downloader commit (same convention as open_orca). - R2_DOWNLOADER_COMMIT = "27da4421877f2831eeb615b43ee5098c4b70be7e" - # The resolved dataset URI is handed to a downloaded shell script, so the - # host is restricted to MLCommons R2 storage. This stops a mis-set - # BFCL_V4_DATASET_URI env var from pointing the downloader at an - # attacker-controlled location. - R2_ALLOWED_HOST = "mlcommons-storage.org" @classmethod def generate( @@ -413,8 +403,9 @@ def _verify_sha256(cls, path: Path, expected: str) -> None: def _download_full_parquet_from_r2(cls, dst_path: Path) -> bool: """Download the full single-turn parquet from MLCommons R2 storage. - Mirrors the OpenOrca pattern: fetch the pinned ``mlc-r2-downloader.sh``, - run it against the dataset ``.uri``, then SHA-256 verify the result. + Fetching and running the pinned ``mlc-r2-downloader.sh`` is + ``download_from_r2``; this adds the SHA-256 verification and the + relocation to ``dst_path``. Returns True when a verified parquet is in place at ``dst_path``; returns False when no R2 URI is configured, in which case the caller builds the @@ -426,48 +417,12 @@ def _download_full_parquet_from_r2(cls, dst_path: Path) -> bool: if not uri: return False - # The uri is passed to a downloaded shell script; restrict it to https - # on MLCommons R2 storage before invoking anything. - parsed = urlparse(uri) - host = parsed.hostname or "" - if parsed.scheme != "https" or not ( - host == cls.R2_ALLOWED_HOST or host.endswith("." + cls.R2_ALLOWED_HOST) - ): - raise ValueError( - f"Refusing to download dataset from untrusted URI '{uri}': " - f"expected https on {cls.R2_ALLOWED_HOST}" - ) - download_dir = dst_path.parent download_dir.mkdir(parents=True, exist_ok=True) # Start from a clean target so a stale parquet can't shadow the fresh # download (the relocate below only fires when dst_path is absent). dst_path.unlink(missing_ok=True) - downloader_url = ( - "https://raw.githubusercontent.com/mlcommons/r2-downloader/" - f"{cls.R2_DOWNLOADER_COMMIT}/mlc-r2-downloader.sh" - ) - script_path = download_dir / "mlc-r2-downloader.sh" - resp = requests.get(downloader_url, timeout=30) - resp.raise_for_status() - script_path.write_bytes(resp.content) - script_path.chmod(0o755) - - try: - result = subprocess.run( - ["bash", str(script_path.resolve()), "-d", str(download_dir), uri], - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - text=True, - check=False, - ) - if result.returncode != 0: - raise RuntimeError( - f"R2 downloader failed with code {result.returncode}: " - f"{result.stderr}" - ) - finally: - script_path.unlink(missing_ok=True) + download_from_r2(uri, download_dir) # The downloader may nest the file under a subdir per the .uri manifest; # relocate it to the expected cache path if so. diff --git a/src/inference_endpoint/dataset_manager/predefined/open_orca/__init__.py b/src/inference_endpoint/dataset_manager/predefined/open_orca/__init__.py index ed092c986..58a31ada6 100644 --- a/src/inference_endpoint/dataset_manager/predefined/open_orca/__init__.py +++ b/src/inference_endpoint/dataset_manager/predefined/open_orca/__init__.py @@ -16,14 +16,12 @@ import gzip import hashlib import shutil -import subprocess from logging import getLogger from pathlib import Path import pandas as pd -import requests -from ...dataset import Dataset +from ...dataset import Dataset, download_from_r2 from . import presets logger = getLogger(__name__) @@ -127,33 +125,8 @@ def generate( # Dataset URL from README dataset_url = "https://inference.mlcommons-storage.org/metadata/llama-2-70b-open-orca-dataset.uri" - # Download the r2-downloader script into a temp file in the target dir - COMMIT_HASH = "27da4421877f2831eeb615b43ee5098c4b70be7e" - downloader_url = f"https://raw.githubusercontent.com/mlcommons/r2-downloader/{COMMIT_HASH}/mlc-r2-downloader.sh" download_dir = cache_dir - script_path = cache_dir / "mlc-r2-downloader.sh" - r = requests.get(downloader_url, timeout=30) - r.raise_for_status() - script_path.write_bytes(r.content) - script_path.chmod(0o755) - - # Run the script with the dataset URL. - try: - # Use absolute path for the script to avoid path doubling when cwd is set - script_abs = str(script_path.resolve()) - result = subprocess.run( - ["bash", script_abs, "-d", str(download_dir), dataset_url], - stdout=subprocess.DEVNULL, # Suppress normal output - stderr=subprocess.PIPE, # Capture errors - text=True, - check=False, - ) - if result.returncode != 0: - raise RuntimeError( - f"R2 downloader failed with code {result.returncode}: {result.stderr}" - ) - except subprocess.CalledProcessError as e: - raise RuntimeError(f"R2 downloader failed: {e}") from e + download_from_r2(dataset_url, download_dir) # Script will generate a new 'open_orca' subdirectory with gzip'd pickle files gzip_dir = download_dir @@ -172,7 +145,6 @@ def generate( finally: for gz_path in gzip_dir.glob("*.pkl.gz"): gz_path.unlink() - script_path.unlink(missing_ok=True) __all__ = ["OpenOrca"] diff --git a/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py b/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py index aa3ce7b80..16deb669a 100644 --- a/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py +++ b/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py @@ -56,7 +56,9 @@ class WhisperLibriSpeech(Dataset, dataset_id="whisper_librispeech"): ``OpenAITranscriptionsAdapter`` (which reads the bytes in the worker). - ``transcript`` : reference text, read by ``WERScorer`` off the dataframe as the ground-truth column. - - ``duration`` : clip length in seconds, carried for reporting only. + - ``duration`` : clip length in seconds. Retained in the cached parquet + for inspection only — nothing in the benchmark reads it, and the + adapter's ``ColumnFilter`` drops it before issuance. One loader serves both phases: the performance phase issues ``audio_file`` and the accuracy phase scores against ``transcript``. @@ -100,7 +102,9 @@ def generate( seed: Unused; present for the ``get_dataloader`` contract. force: Rebuild the cached parquet even if it already exists. An already-downloaded corpus is reused — this does not re-fetch - several GB of audio. + several GB of audio. The cache path is keyed on the dataset ID + alone, so changing ``manifest`` or ``data_dir`` reuses the rows + built from the previous one until this is set. """ dst_path = datasets_dir / cls.DATASET_ID / "whisper_librispeech.parquet" dst_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/inference_endpoint/endpoint_client/accumulator_protocol.py b/src/inference_endpoint/endpoint_client/accumulator_protocol.py index c4974bde2..92305bdc7 100644 --- a/src/inference_endpoint/endpoint_client/accumulator_protocol.py +++ b/src/inference_endpoint/endpoint_client/accumulator_protocol.py @@ -76,11 +76,17 @@ class NonStreamingAccumulator: """Accumulator stub for api_types whose endpoint has no SSE mode. HTTPClientConfig resolves an adapter/accumulator pair for every api_type, so - a non-streaming endpoint still needs an accumulator. This one should never - be exercised; `get_final_output` raises rather than returning an empty - `QueryResult`, because the worker's SSE path swallows the - `NotImplementedError` from `decode_sse_message` and would otherwise surface - zero-output queries as successful. + a non-streaming endpoint still needs an accumulator. This one should never be + exercised. + + `get_final_output` raises rather than returning an empty `QueryResult` + because a streamed response body carrying no `data:` frames never calls + `decode_sse_message` at all: `parse_sse_chunk` finds no matches, nothing + raises, and the stream ends with nothing accumulated. Returning an empty + result there would record a zero-output query as successful. A body that + does carry frames raises `NotImplementedError` out of the adapter instead — + `parse_sse_chunk` catches only msgspec decode errors — and the worker + records that query as failed. """ def __init__(self, query_id: str, stream_all_chunks: bool) -> None: diff --git a/src/inference_endpoint/evaluation/scoring.py b/src/inference_endpoint/evaluation/scoring.py index e15c51d6a..491a2ac30 100644 --- a/src/inference_endpoint/evaluation/scoring.py +++ b/src/inference_endpoint/evaluation/scoring.py @@ -56,6 +56,7 @@ from ..dataset_manager.agentic_inference_dataset import AgenticInferenceDataset from ..dataset_manager.dataset import Dataset from ..dataset_manager.predefined.shopify_product_catalogue import ProductMetadata +from ..exceptions import InputValidationError from .accuracy_results import build_breakdown from .extractor import ( Extractor, @@ -420,6 +421,15 @@ class WERScorer(Scorer, scorer_id="wer"): character-label inventory, apply Whisper's ``EnglishTextNormalizer``, split on whitespace, then accumulate the concatenation-aware edit distance. See :mod:`inference_endpoint.evaluation.wer`. + + An issued sample with no COMPLETE event is scored as an empty hypothesis: + its reference words enter the denominator and every one of them counts as a + deletion, exactly as a completed-but-blank response already does. Skipping + it instead would shrink the denominator, so dropping the samples a SUT + handles worst would *raise* the score. This matches the reference number + whenever nothing is missing, and ``complete`` is set False when something + is, so a partial run is machine-detectable rather than needing a reader to + notice ``missing_samples``. """ REQUIRES_EXTRACTOR = False @@ -434,21 +444,43 @@ class WERScorer(Scorer, scorer_id="wer"): def __init__(self, *args, normalizer_source: str | None = None, **kwargs): super().__init__(*args, **kwargs) + if self.extractor is not None: + raise InputValidationError( + f"accuracy_config.extractor is not supported by " + f"{type(self).__name__}: WER scores the transcript verbatim, so " + "an extractor would be silently ignored. Remove it." + ) self.normalizer_source = normalizer_source or self.DEFAULT_NORMALIZER_SOURCE self._breakdown: dict[str, Any] | None = None - def _normalizer(self) -> EnglishTextNormalizer: - tokenizer = AutoTokenizer.from_pretrained(self.normalizer_source) + @classmethod + def preflight(cls, extras: dict[str, Any]) -> None: + """Resolve the normalizer before the run, not after it. + + ``_normalizer`` needs network access or a warm HF cache. Scoring happens + after the whole performance phase, so a cold cache in an air-gapped + submission environment would otherwise fail the run at its very last + step, with the endpoint work already spent. + """ + cls._load_normalizer(extras.get("normalizer_source")) + + @classmethod + def _load_normalizer(cls, source: str | None) -> EnglishTextNormalizer: + resolved = source or cls.DEFAULT_NORMALIZER_SOURCE + tokenizer = AutoTokenizer.from_pretrained(resolved) spelling_map = getattr(tokenizer, "english_spelling_normalizer", None) if not spelling_map: raise ValueError( - f"Tokenizer {self.normalizer_source!r} carries no " + f"Tokenizer {resolved!r} carries no " "english_spelling_normalizer (normalizer.json). Point " "accuracy_config.extras.normalizer_source at a Whisper " "tokenizer repo or local directory." ) return EnglishTextNormalizer(spelling_map) + def _normalizer(self) -> EnglishTextNormalizer: + return self._load_normalizer(self.normalizer_source) + def score_single_sample(self, value: str, ground_truth: str) -> float: raise RuntimeError( "WER is a corpus-level metric: errors and reference words are summed " @@ -461,6 +493,7 @@ def score(self) -> tuple[float | None, int]: issued = len(self.sample_index_map) if issued == 0: + self.complete = False return None, 0 n_repeats = issued // self.dataset.num_samples() @@ -473,26 +506,29 @@ def score(self) -> tuple[float | None, int]: normalizer = self._normalizer() references = self.dataset.dataframe[self.ground_truth_column].to_numpy() + outputs = dict(zip(df["sample_uuid"], df["output"], strict=True)) + # Iterate the issued samples, not the returned ones: a sample with no + # COMPLETE event is scored as an empty hypothesis (all of its reference + # words count as deletions) rather than dropped from the denominator. distance = 0 words = 0 empty = 0 - for _, row in df.iterrows(): - reference = references[self.sample_index_map[row["sample_uuid"]]] - hypothesis = row["output"] - if not hypothesis: + missing = 0 + for sample_uuid, index in self.sample_index_map.items(): + hypothesis = outputs.get(sample_uuid) + if hypothesis is None: + missing += 1 + hypothesis = "" + elif not hypothesis: empty += 1 hyp_words = normalizer(project_to_labels(hypothesis)).split() - ref_words = normalizer(project_to_labels(str(reference))).split() + ref_words = normalizer(project_to_labels(str(references[index]))).split() sample_distance, sample_words = wer_counts(hyp_words, ref_words) distance += sample_distance words += sample_words - # Samples that never produced a COMPLETE event contribute no errors and - # no reference words, so they silently shrink the denominator rather - # than penalising the score - surface the count so a partial run is not - # mistaken for a clean one. - missing = issued - len(df) + self.complete = missing == 0 if words == 0: self._breakdown = {"wer_percent": None, "missing_samples": missing} return None, n_repeats @@ -507,11 +543,10 @@ def score(self) -> tuple[float | None, int]: } if missing: logger.warning( - "WER computed over %d of %d issued samples (%d missing); the " - "score does not describe a complete run.", - len(df), - issued, + "%d of %d issued samples produced no output and were scored as " + "empty transcripts; this run is reported as incomplete.", missing, + issued, ) return 100.0 - wer * 100.0, n_repeats diff --git a/src/inference_endpoint/evaluation/wer.py b/src/inference_endpoint/evaluation/wer.py index 28e784f25..f88792ef7 100644 --- a/src/inference_endpoint/evaluation/wer.py +++ b/src/inference_endpoint/evaluation/wer.py @@ -25,6 +25,14 @@ torchmetrics) will not reproduce the reference number: the reference credits a hypothesis word that equals up to three joined reference words, and vice versa, before measuring edit distance. See ``docs/evaluation/DESIGN.md``. + +Parity with upstream is pinned by the golden vectors in +``tests/unit/evaluation/test_wer.py`` (``UPSTREAM_VECTORS``), captured by running +the reference functions at mlcommons/inference commit +``3fbc329939999c13d0a7b5e67fb2092287e06047``. + +Upstream reports ``accuracy = (1 - wer) * 100`` (``speech2text/accuracy_eval.py``), +which is the ``100 - WER%`` scale :class:`WERScorer` reports on. """ from collections.abc import Sequence @@ -45,13 +53,18 @@ def project_to_labels(text: str) -> str: lossy in exactly this way. Reproducing it keeps punctuation handling identical to the reference. - The lowercasing is applied to both sides. The reference lowercases only the - SUT output (in its SUT) and not the manifest transcript (in - ``manifest.parse_transcript``), so an uppercase transcript — which is how - LibriSpeech ships — would have every letter dropped as out-of-inventory, - leaving an empty reference. Lowercasing both is a no-op whenever the - manifest is already lowercase, and the only behaviour that yields a - meaningful number when it is not. + The lowercasing is applied to both sides, which is a no-op on the official + corpus: the MLCommons preparation pipeline already lowercases every + transcript (``speech2text/utils/preprocessing_utils.py`` writes + ``data['transcript'].lower().strip()``), so the manifest this loader + consumes is lowercase before it arrives. + + It is kept as defence for a hand-built manifest. Upstream's + ``manifest.parse_transcript`` maps each character through the label + inventory and *drops* whatever is absent, and the inventory holds no + uppercase letters — so an uppercase transcript would reduce to an empty + reference there rather than failing loudly. Lowercasing first is the only + behaviour that still yields a meaningful number in that case. """ lowered = text.lower() return "".join(c for c in lowered if c in LABELS) @@ -97,7 +110,9 @@ def wer_counts(hypothesis: Sequence[str], reference: Sequence[str]) -> tuple[int ref_match_len = hyp_match_len = 1 match_found = False - for k in range(1, MAX_CONCATENATION_SPAN + 1): + # From 2: a k of 1 would re-test reference[i] == hypothesis[j], which the + # branch above already handled and left unequal. + for k in range(2, MAX_CONCATENATION_SPAN + 1): if ( i + k <= len(reference) and "".join(reference[i : i + k]) == hypothesis[j] diff --git a/src/inference_endpoint/openai/transcriptions_adapter.py b/src/inference_endpoint/openai/transcriptions_adapter.py index 1aa78e266..5630cb7e2 100644 --- a/src/inference_endpoint/openai/transcriptions_adapter.py +++ b/src/inference_endpoint/openai/transcriptions_adapter.py @@ -39,8 +39,20 @@ #: is not a practical concern. _BOUNDARY = "----mlperf-inference-endpoints-audio-boundary-Q3nV7xTt" -_DASHES = f"--{_BOUNDARY}\r\n".encode("ascii") -_CLOSING = f"--{_BOUNDARY}--\r\n".encode("ascii") +_DELIMITER = f"--{_BOUNDARY}" +_DASHES = f"{_DELIMITER}\r\n".encode("ascii") +_CLOSING = f"{_DELIMITER}--\r\n".encode("ascii") + +#: Characters that would terminate a `Content-Disposition` parameter early, with +#: the percent escapes browsers substitute for them (WHATWG HTML, "multipart/ +#: form-data encoding algorithm"). Servers decode these, so escaping keeps the +#: original value recoverable where rejecting the row would not. +_PARAM_ESCAPES = str.maketrans({'"': "%22", "\r": "%0D", "\n": "%0A"}) + + +def _escape_param(value: str) -> str: + """Escape a quoted-string header parameter (a `name=` or `filename=` value).""" + return value.translate(_PARAM_ESCAPES) def encode_multipart( @@ -48,21 +60,37 @@ def encode_multipart( ) -> bytes: """Encode `fields` plus one `file` part as an RFC 7578 multipart body. + Both the parameter names and the filename are dataset-controlled, so they are + escaped: an unescaped `"` or CRLF would close the parameter early and let a + row inject its own headers or parts. Field *values* sit in a part's body + where CRLF is legal, so they are checked only for the one string that would + forge a part boundary. + Args: fields: Text form fields, in the order they should be written. filename: Value of the file part's `filename` parameter. audio: Raw audio bytes. mime_type: Media type of the file part. + + Raises: + ValueError: A field value contains the multipart delimiter. """ parts: list[bytes] = [] for name, value in fields.items(): + if _DELIMITER in value: + raise ValueError( + f"Form field {name!r} contains the multipart delimiter " + f"{_DELIMITER!r}, which would forge a part boundary." + ) parts.append(_DASHES) parts.append( - f'Content-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n'.encode() + f'Content-Disposition: form-data; name="{_escape_param(name)}"' + f"\r\n\r\n{value}\r\n".encode() ) parts.append(_DASHES) parts.append( - f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n' + f'Content-Disposition: form-data; name="file"; ' + f'filename="{_escape_param(filename)}"\r\n' f"Content-Type: {mime_type}\r\n\r\n".encode() ) parts.append(audio) @@ -77,16 +105,23 @@ class OpenAITranscriptionsAdapter(HttpRequestAdapter): The request is a `multipart/form-data` audio upload, so the body is not JSON and `CONTENT_TYPE` carries the boundary. The dataset row holds a *path*; the bytes are read here, inside the worker, so a full audio corpus never - traverses the issue thread or the ZMQ frame. Keep the audio on fast local - storage: the read happens inside the measured request. + traverses the issue thread or the ZMQ frame. + + The read is synchronous on the worker's event loop, so it stalls every + in-flight response in that worker process, not just this request. Keep the + audio on fast local storage: a cold-cache or network-mounted read shows up + as added latency across the whole worker. `language` is what makes the server build Whisper's - `<|startoftranscript|><|LANG|><|transcribe|><|notimestamps|>` prompt, which - is how this matches the MLPerf speech2text reference. + `<|startoftranscript|><|LANG|><|transcribe|><|notimestamps|>` prompt instead + of auto-detecting per clip, which is how this matches the MLPerf speech2text + reference. `DEFAULT_LANGUAGE` supplies it; a dataset `language` column wins. The endpoint is non-streaming: OpenAI does not support `stream` for Whisper-class models, so `decode_sse_message` raises and the registered - accumulator is `NonStreamingAccumulator`. + accumulator is `NonStreamingAccumulator`. Config validation rejects + `streaming: on` for this api_type, and `ColumnFilter` admits no `stream` + column, so `query.data` never carries one. """ CONTENT_TYPE: ClassVar[str] = f"multipart/form-data; boundary={_BOUNDARY}" @@ -107,6 +142,12 @@ class OpenAITranscriptionsAdapter(HttpRequestAdapter): #: bytes to librosa/soundfile), so a per-row MIME column buys nothing. MIME_TYPE: ClassVar[str] = "audio/wav" + #: Transcription language, sent on every request. The MLPerf speech2text + #: corpus is English LibriSpeech and the reference decodes with the language + #: pinned, so leaving the server to auto-detect per clip would be a different + #: workload. A dataset `language` column overrides this per row. + DEFAULT_LANGUAGE: ClassVar[str] = "en" + _response_decoder: msgspec.json.Decoder = msgspec.json.Decoder( TranscriptionResponse ) @@ -115,6 +156,7 @@ class OpenAITranscriptionsAdapter(HttpRequestAdapter): def dataset_transforms(cls, model_params: ModelParams) -> list[Transform]: metadata = { "model": model_params.name, + "language": cls.DEFAULT_LANGUAGE, "response_format": "json", "temperature": model_params.temperature, } @@ -136,11 +178,6 @@ def encode_query(cls, query: Query) -> bytes: f"{AUDIO_FILE_COLUMN!r} not found in query.data keys: " f"{list(data.keys())}" ) - if data.get("stream"): - raise ValueError( - "OpenAITranscriptionsAdapter is non-streaming; remove `stream` " - "from query.data and set model_params.streaming to 'off'." - ) audio_path = Path(data[AUDIO_FILE_COLUMN]) fields = { name: str(data[name]) diff --git a/src/inference_endpoint/videogen/adapter.py b/src/inference_endpoint/videogen/adapter.py index ff4f9a704..a875500a2 100644 --- a/src/inference_endpoint/videogen/adapter.py +++ b/src/inference_endpoint/videogen/adapter.py @@ -162,9 +162,11 @@ class VideoGenAccumulator: Video generation requests are non-streaming HTTP, so this class should never be exercised. `get_final_output` raises rather than returning an - empty `QueryResult`, because the worker's SSE path swallows the - `NotImplementedError` from `decode_sse_message` and would otherwise - surface zero-output queries as successful. + empty `QueryResult` because a streamed body with no `data:` frames never + reaches `decode_sse_message`, so nothing raises and the stream ends with + nothing accumulated — an empty result would record that as a successful + zero-output query. See `NonStreamingAccumulator`, which shares this + contract. """ def __init__(self, query_id: str, stream_all_chunks: bool) -> None: diff --git a/tests/integration/openai/test_transcriptions_adapter.py b/tests/integration/openai/test_transcriptions_adapter.py index d2b46b1aa..9ab7815fe 100644 --- a/tests/integration/openai/test_transcriptions_adapter.py +++ b/tests/integration/openai/test_transcriptions_adapter.py @@ -269,6 +269,13 @@ async def test_server_rejects_a_non_multipart_body( def test_dataset_rows_are_request_ready(dataset) -> None: """The loaded row is used verbatim as Query.data, so it must encode.""" row = dataset.load_sample(0) - assert set(row) == {"audio_file", "model", "response_format", "temperature"} + assert set(row) == { + "audio_file", + "model", + "language", + "response_format", + "temperature", + } + assert row["language"] == "en" body = OpenAITranscriptionsAdapter.encode_query(Query(id="q", data=row)) assert Path(row["audio_file"]).read_bytes() in body diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index 9a881c184..0fd7ca1cf 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -1200,3 +1200,95 @@ def test_non_int_seed_rejected(self, register_temp_ruleset): ) with pytest.raises(ValidationError): self._submission(name) + + +class TestTranscriptionsAPIValidation: + """``openai_transcriptions`` constrains the run settings by api_type. + + /v1/audio/transcriptions has no SSE mode and no text prompt, so streaming, + ISL and warmup salt are not free choices here. They used to be documented + only in the example YAML's comments, which a hand-written config never sees. + """ + + def _cfg(self, **overrides) -> BenchmarkConfig: + kwargs: dict = { + "type": TestType.ONLINE, + "model_params": {"name": "openai/whisper-large-v3"}, + "endpoint_config": { + "endpoints": ["http://localhost:8000"], + "api_type": APIType.OPENAI_TRANSCRIPTIONS, + }, + "datasets": [{"path": "clips.jsonl"}], + "settings": {"load_pattern": {"type": "poisson", "target_qps": 1}}, + } + kwargs.update(overrides) + return BenchmarkConfig(**kwargs) + + @pytest.mark.unit + def test_auto_streaming_resolves_off_even_online(self): + # The generic rule would resolve AUTO to ON for an online run. + assert self._cfg().model_params.streaming == StreamingMode.OFF + + @pytest.mark.unit + def test_explicit_streaming_on_is_rejected(self): + with pytest.raises(ValueError, match="no streaming mode"): + self._cfg( + model_params={"name": "whisper", "streaming": "on"}, + ) + + @pytest.mark.unit + def test_explicit_streaming_off_is_accepted(self): + cfg = self._cfg(model_params={"name": "whisper", "streaming": "off"}) + assert cfg.model_params.streaming == StreamingMode.OFF + + @pytest.mark.unit + def test_metrics_isl_is_forced_off(self): + # Default is True; an audio upload has no prompt to tokenize, so the + # series could only ever stay empty. + assert self._cfg().settings.metrics_isl is False + + @pytest.mark.unit + def test_metrics_isl_stays_off_when_set_explicitly(self): + cfg = self._cfg( + settings={ + "load_pattern": {"type": "poisson", "target_qps": 1}, + "metrics_isl": False, + } + ) + assert cfg.settings.metrics_isl is False + + @pytest.mark.unit + def test_warmup_salt_is_rejected(self): + with pytest.raises(ValueError, match="warmup.salt"): + self._cfg( + settings={ + "load_pattern": {"type": "poisson", "target_qps": 1}, + "warmup": {"enabled": True, "salt": True}, + } + ) + + @pytest.mark.unit + def test_warmup_without_salt_is_accepted(self): + cfg = self._cfg( + settings={ + "load_pattern": {"type": "poisson", "target_qps": 1}, + "warmup": {"enabled": True, "salt": False}, + } + ) + assert cfg.settings.warmup.enabled is True + + @pytest.mark.unit + def test_other_api_types_keep_online_streaming(self): + # The constraint is keyed on api_type, not applied to every run. + cfg = BenchmarkConfig( + type=TestType.ONLINE, + model_params={"name": "M"}, + endpoint_config={ + "endpoints": ["http://localhost:8000"], + "api_type": APIType.OPENAI, + }, + datasets=[{"path": "d.jsonl"}], + settings={"load_pattern": {"type": "poisson", "target_qps": 1}}, + ) + assert cfg.model_params.streaming == StreamingMode.ON + assert cfg.settings.metrics_isl is True diff --git a/tests/unit/dataset_manager/test_bfcl_v4_dataset.py b/tests/unit/dataset_manager/test_bfcl_v4_dataset.py index e9d9a36e5..72099abd1 100644 --- a/tests/unit/dataset_manager/test_bfcl_v4_dataset.py +++ b/tests/unit/dataset_manager/test_bfcl_v4_dataset.py @@ -34,6 +34,10 @@ _MODULE = "inference_endpoint.dataset_manager.predefined.bfcl_v4" +#: Fetching and running the downloader script lives in the shared +#: ``download_from_r2``, so that is where the network calls are patched. +_DOWNLOAD_MODULE = "inference_endpoint.dataset_manager.dataset" + def _dst(tmp_path: Path) -> Path: """Cache path the loader writes the full single-turn parquet to.""" @@ -97,7 +101,7 @@ def test_returns_false_when_no_uri(self, tmp_path, monkeypatch): monkeypatch.delenv(BFCLv4.R2_DATASET_URI_ENV, raising=False) # Must short-circuit before touching the network. - with patch(f"{_MODULE}.requests.get") as get: + with patch(f"{_DOWNLOAD_MODULE}.requests.get") as get: assert BFCLv4._download_full_parquet_from_r2(_dst(tmp_path)) is False get.assert_not_called() @@ -115,9 +119,11 @@ def test_success_via_constant_uri(self, tmp_path, monkeypatch): ) with ( - patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), patch( - f"{_MODULE}.subprocess.run", + f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() + ), + patch( + f"{_DOWNLOAD_MODULE}.subprocess.run", side_effect=_fake_run_writing(dst, payload), ), ): @@ -142,8 +148,10 @@ def test_env_var_overrides_constant(self, tmp_path, monkeypatch): run = MagicMock(side_effect=_fake_run_writing(dst, payload)) with ( - patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), - patch(f"{_MODULE}.subprocess.run", run), + patch( + f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() + ), + patch(f"{_DOWNLOAD_MODULE}.subprocess.run", run), ): assert BFCLv4._download_full_parquet_from_r2(dst) is True # The env URI, not the constant, is the last positional arg passed to bash. @@ -164,9 +172,11 @@ def test_relocates_file_from_subdir(self, tmp_path, monkeypatch): ) with ( - patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), patch( - f"{_MODULE}.subprocess.run", + f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() + ), + patch( + f"{_DOWNLOAD_MODULE}.subprocess.run", side_effect=_fake_run_writing(nested, payload), ), ): @@ -183,9 +193,11 @@ def test_sha256_mismatch_raises(self, tmp_path, monkeypatch): monkeypatch.delenv(BFCLv4.R2_DATASET_URI_ENV, raising=False) # Real (unmocked) SHA pin; downloaded bytes won't match it. with ( - patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), patch( - f"{_MODULE}.subprocess.run", + f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() + ), + patch( + f"{_DOWNLOAD_MODULE}.subprocess.run", side_effect=_fake_run_writing(dst, b"WRONG-BYTES"), ), pytest.raises(ValueError, match="SHA-256 mismatch"), @@ -203,8 +215,10 @@ def _run_noop(cmd, **kwargs): return MagicMock(returncode=0, stderr="") with ( - patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), - patch(f"{_MODULE}.subprocess.run", side_effect=_run_noop), + patch( + f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() + ), + patch(f"{_DOWNLOAD_MODULE}.subprocess.run", side_effect=_run_noop), pytest.raises(FileNotFoundError), ): BFCLv4._download_full_parquet_from_r2(dst) @@ -220,8 +234,10 @@ def _run_fail(cmd, **kwargs): return MagicMock(returncode=1, stderr="boom") with ( - patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), - patch(f"{_MODULE}.subprocess.run", side_effect=_run_fail), + patch( + f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() + ), + patch(f"{_DOWNLOAD_MODULE}.subprocess.run", side_effect=_run_fail), pytest.raises(RuntimeError, match="R2 downloader failed"), ): BFCLv4._download_full_parquet_from_r2(dst) @@ -240,8 +256,8 @@ def test_untrusted_uri_rejected_before_network(self, tmp_path, monkeypatch, uri) monkeypatch.setattr(BFCLv4, "R2_DATASET_URI", uri) monkeypatch.delenv(BFCLv4.R2_DATASET_URI_ENV, raising=False) with ( - patch(f"{_MODULE}.requests.get") as get, - patch(f"{_MODULE}.subprocess.run") as run, + patch(f"{_DOWNLOAD_MODULE}.requests.get") as get, + patch(f"{_DOWNLOAD_MODULE}.subprocess.run") as run, pytest.raises(ValueError, match="untrusted URI"), ): BFCLv4._download_full_parquet_from_r2(dst) diff --git a/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py b/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py index 7391f7732..50041a1ae 100644 --- a/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py +++ b/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py @@ -21,10 +21,12 @@ """ import json +import subprocess from pathlib import Path from unittest.mock import MagicMock, patch import pytest +from inference_endpoint.dataset_manager import dataset from inference_endpoint.dataset_manager.dataset import Dataset from inference_endpoint.dataset_manager.predefined.whisper_librispeech import ( MANIFEST_FILENAME, @@ -260,6 +262,47 @@ def test_downloader_failure_is_reported( with pytest.raises(RuntimeError, match="r2 boom"): WhisperLibriSpeech.generate(tmp_path / "cache") + def test_downloader_is_bounded_by_a_timeout( + self, tmp_path: Path, no_local_manifest: None + ): + # Dataset generation runs before RunWatchdog arms the whole-run + # deadline, so a stalled multi-GB fetch has nothing else to stop it. + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch(f"{_MODULE}.subprocess.run") as run, + ): + run.side_effect = _fake_run_unpacking + WhisperLibriSpeech.generate(tmp_path / "cache") + assert run.call_args.kwargs["timeout"] == dataset.R2_DOWNLOAD_TIMEOUT_S + + def test_a_stalled_download_raises_rather_than_hanging( + self, tmp_path: Path, no_local_manifest: None + ): + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch( + f"{_MODULE}.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd="bash", timeout=1), + ), + ): + with pytest.raises(RuntimeError, match="exceeded"): + WhisperLibriSpeech.generate(tmp_path / "cache") + + def test_the_downloader_script_is_removed_after_a_timeout( + self, tmp_path: Path, no_local_manifest: None + ): + cache = tmp_path / "cache" + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch( + f"{_MODULE}.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd="bash", timeout=1), + ), + ): + with pytest.raises(RuntimeError): + WhisperLibriSpeech.generate(cache) + assert not list(cache.rglob("mlc-r2-downloader.sh")) + @pytest.mark.unit class TestGenerateErrors: diff --git a/tests/unit/evaluation/test_wer.py b/tests/unit/evaluation/test_wer.py index 1808b7f93..bcdde70c5 100644 --- a/tests/unit/evaluation/test_wer.py +++ b/tests/unit/evaluation/test_wer.py @@ -99,9 +99,8 @@ def test_three_word_join_is_credited(self): assert wer_counts(["anyoneelse"], ["any", "one", "else"]) == (0, 3) def test_four_word_join_exceeds_the_span_and_counts_errors(self): - distance, words = wer_counts(["abcd"], ["a", "b", "c", "d"]) - assert words == 4 - assert distance > 0 + # Pinned against upstream: 4 errors over 4 words, not a partial credit. + assert wer_counts(["abcd"], ["a", "b", "c", "d"]) == (4, 4) def test_join_credited_mid_sentence(self): assert wer_counts(["i", "cannot", "go"], ["i", "can", "not", "go"]) == (0, 4) @@ -128,3 +127,75 @@ def test_reference_pipeline_on_a_librispeech_style_pair(self): hyp = project_to_labels(hypothesis).split() ref = project_to_labels(reference).split() assert wer_counts(hyp, ref) == (0, 9) + + +#: ``(hypothesis, reference, (edit_distance, reference_words))`` captured by +#: running the MLCommons reference implementation directly — its +#: ``speech2text/helpers.py:compute_wer_with_concatenation`` composed with +#: ``legacy_helpers.py:__levenshtein``, pinned at mlcommons/inference commit +#: 3fbc329939999c13d0a7b5e67fb2092287e06047. These are the upstream numbers, not +#: this implementation's output, so they fail if the port drifts. +#: +#: Generated alongside a randomized differential sweep (20k trials over a small +#: alphabet chosen so concatenations actually occur) that found zero divergence. +#: The cases below are the ones where the metric's behaviour is load-bearing: +#: joins in each direction, a join past the 3-word span, and the empty edges. +UPSTREAM_VECTORS = [ + (["the", "cat", "sat"], ["the", "cat", "sat"], (0, 3)), + (["the", "dog", "sat"], ["the", "cat", "sat"], (1, 3)), + (["the", "sat"], ["the", "cat", "sat"], (1, 3)), + (["the", "big", "cat", "sat"], ["the", "cat", "sat"], (1, 3)), + ([], ["the", "cat", "sat"], (3, 3)), + (["the"], [], (1, 0)), + ([], [], (0, 0)), + (["cannot"], ["can", "not"], (0, 2)), + (["can", "not"], ["cannot"], (0, 1)), + (["anyoneelse"], ["any", "one", "else"], (0, 3)), + (["any", "one", "else"], ["anyoneelse"], (0, 1)), + (["abcd"], ["a", "b", "c", "d"], (4, 4)), + (["i", "cannot", "go"], ["i", "can", "not", "go"], (0, 4)), + ( + ["mister", "quilter", "is", "the", "apostle"], + ["mister", "quilter", "is", "the", "apostle"], + (0, 5), + ), + ( + ["mister", "quilter", "is", "the", "epistle"], + ["mister", "quilter", "is", "the", "apostle"], + (1, 5), + ), + ( + ["nor", "is", "mister", "quilters", "manner", "less", "interesting"], + ["nor", "is", "mister", "quilter", "s", "manner", "less", "interesting"], + (0, 8), + ), + ( + ["goodmorning", "everyone", "welcome"], + ["good", "morning", "every", "one", "welcome"], + (0, 5), + ), + (["a", "bc", "d"], ["a", "b", "c", "d"], (0, 4)), + (["abc", "d"], ["a", "b", "c", "d"], (0, 4)), + (["x", "abc"], ["x", "a", "b", "c"], (0, 4)), + ( + ["and", "we", "are", "glad", "to", "welcome", "his", "gospel"], + ["and", "we", "are", "glad", "to", "welcome", "his", "gospel"], + (0, 8), + ), +] + + +@pytest.mark.unit +class TestUpstreamParity: + @pytest.mark.parametrize(("hypothesis", "reference", "expected"), UPSTREAM_VECTORS) + def test_matches_the_reference_implementation( + self, hypothesis, reference, expected + ): + assert wer_counts(hypothesis, reference) == tuple(expected) + + def test_corpus_wer_over_the_whole_vector_set(self): + # The aggregate the benchmark actually reports: summed, then divided. + distance = sum(e[0] for _, _, e in UPSTREAM_VECTORS) + words = sum(e[1] for _, _, e in UPSTREAM_VECTORS) + ours = [wer_counts(h, r) for h, r, _ in UPSTREAM_VECTORS] + assert (sum(d for d, _ in ours), sum(w for _, w in ours)) == (distance, words) diff --git a/tests/unit/evaluation/test_wer_scorer.py b/tests/unit/evaluation/test_wer_scorer.py index ee2afe4df..1e400dae1 100644 --- a/tests/unit/evaluation/test_wer_scorer.py +++ b/tests/unit/evaluation/test_wer_scorer.py @@ -24,7 +24,10 @@ from inference_endpoint.core.record import EventRecord, EventType, SampleEventType from inference_endpoint.core.types import TextModelOutput from inference_endpoint.evaluation import scoring +from inference_endpoint.evaluation.extractor import PythonCodeExtractor from inference_endpoint.evaluation.scoring import Scorer, WERScorer +from inference_endpoint.evaluation.wer import project_to_labels +from inference_endpoint.exceptions import InputValidationError from transformers.models.whisper.english_normalizer import EnglishTextNormalizer DATASET_NAME = "whisper_librispeech" @@ -36,6 +39,12 @@ ] +def _ref_word_count(reference: str) -> int: + """Reference word count as the scorer counts it (post-normalization).""" + normalizer = EnglishTextNormalizer({}) + return len(normalizer(project_to_labels(reference)).split()) + + @pytest.fixture def hermetic_normalizer(monkeypatch: pytest.MonkeyPatch): """Use the real normalizer with an empty spelling map - no HF download.""" @@ -167,25 +176,76 @@ def test_concatenated_words_are_credited(self, tmp_path: Path, dataset): @pytest.mark.unit @pytest.mark.usefixtures("hermetic_normalizer") class TestPartialRuns: - def test_missing_samples_are_reported(self, tmp_path: Path, dataset): - # Only 2 of 3 issued samples completed. + def test_missing_sample_is_scored_as_an_empty_transcript( + self, tmp_path: Path, dataset + ): + # Only 2 of 3 issued samples completed, both perfectly. The third + # contributes its reference words as deletions rather than vanishing + # from the denominator, so the score must fall below 100. report_dir = _stage(tmp_path / "r", {0: REFERENCES[0], 1: REFERENCES[1]}) scorer = _scorer(report_dir, dataset) score, _ = scorer.score() - assert score == pytest.approx(100.0) breakdown = scorer.score_breakdown() assert breakdown is not None assert breakdown["missing_samples"] == 1 + # Counted through the same normalizer the scorer uses: it rewrites + # contractions, so a raw str.split() is not the reference word count. + assert breakdown["edit_distance"] == _ref_word_count(REFERENCES[2]) + assert breakdown["reference_words"] == sum( + _ref_word_count(r) for r in REFERENCES + ) + assert score == pytest.approx(100.0 - breakdown["wer_percent"]) + assert score < 100.0 + + def test_missing_samples_mark_the_score_incomplete(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0], 1: REFERENCES[1]}) + scorer = _scorer(report_dir, dataset) + scorer.score() + assert scorer.complete is False + + def test_a_full_run_is_complete(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", dict(enumerate(REFERENCES))) + scorer = _scorer(report_dir, dataset) + scorer.score() + assert scorer.complete is True - def test_no_completions_yields_no_score(self, tmp_path: Path, dataset): + def test_an_empty_response_does_not_mark_the_run_incomplete( + self, tmp_path: Path, dataset + ): + # A blank transcript is a response: the sample completed and scored 0. + report_dir = _stage(tmp_path / "r", {i: "" for i in range(len(REFERENCES))}) + scorer = _scorer(report_dir, dataset) + scorer.score() + assert scorer.complete is True + breakdown = scorer.score_breakdown() + assert breakdown is not None + assert breakdown["missing_samples"] == 0 + + def test_no_completions_scores_zero_and_is_incomplete( + self, tmp_path: Path, dataset + ): + # Every issued sample is scored as an empty transcript, so the whole + # corpus is deletions: WER 100% -> score 0, not an absent number. report_dir = _stage(tmp_path / "r", {}) scorer = _scorer(report_dir, dataset) score, _ = scorer.score() - assert score is None - assert scorer.score_breakdown() == { - "wer_percent": None, - "missing_samples": len(REFERENCES), - } + assert score == pytest.approx(0.0) + assert scorer.complete is False + breakdown = scorer.score_breakdown() + assert breakdown is not None + assert breakdown["missing_samples"] == len(REFERENCES) + assert breakdown["wer_percent"] == pytest.approx(100.0) + + def test_nothing_issued_yields_no_score(self, tmp_path: Path, dataset): + report_dir = tmp_path / "r" + report_dir.mkdir() + (report_dir / "sample_idx_map.json").write_bytes( + msgspec.json.encode({DATASET_NAME: {}}) + ) + (report_dir / "events.jsonl").write_bytes(b"") + scorer = _scorer(report_dir, dataset) + assert scorer.score() == (None, 0) + assert scorer.complete is False @pytest.mark.unit @@ -223,3 +283,60 @@ def test_tokenizer_without_spelling_map_is_rejected( report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) with pytest.raises(ValueError, match="english_spelling_normalizer"): _scorer(report_dir, dataset)._normalizer() + + +@pytest.mark.unit +class TestPreflight: + """The normalizer is resolved before the run, not after the perf phase.""" + + def test_preflight_resolves_the_default_source( + self, monkeypatch: pytest.MonkeyPatch + ): + seen: list[str] = [] + + def _from_pretrained(source, *a, **k): + seen.append(source) + return MagicMock(english_spelling_normalizer={"a": "b"}) + + monkeypatch.setattr(scoring.AutoTokenizer, "from_pretrained", _from_pretrained) + WERScorer.preflight({}) + assert seen == [WERScorer.DEFAULT_NORMALIZER_SOURCE] + + def test_preflight_honours_the_extras_override( + self, monkeypatch: pytest.MonkeyPatch + ): + seen: list[str] = [] + + def _from_pretrained(source, *a, **k): + seen.append(source) + return MagicMock(english_spelling_normalizer={"a": "b"}) + + monkeypatch.setattr(scoring.AutoTokenizer, "from_pretrained", _from_pretrained) + WERScorer.preflight({"normalizer_source": "/local/whisper"}) + assert seen == ["/local/whisper"] + + def test_preflight_raises_on_an_unusable_tokenizer( + self, monkeypatch: pytest.MonkeyPatch + ): + # The whole point: this surfaces before any load is issued. + monkeypatch.setattr( + scoring.AutoTokenizer, + "from_pretrained", + lambda *a, **k: MagicMock(spec=[]), + ) + with pytest.raises(ValueError, match="english_spelling_normalizer"): + WERScorer.preflight({}) + + +@pytest.mark.unit +@pytest.mark.usefixtures("hermetic_normalizer") +class TestExtractorIsRejected: + def test_configured_extractor_is_an_error(self, tmp_path: Path, dataset): + # WER scores the transcript verbatim; an extractor would be ignored. + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + with pytest.raises(InputValidationError, match="extractor"): + _scorer(report_dir, dataset, extractor=PythonCodeExtractor) + + def test_no_extractor_constructs(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + assert _scorer(report_dir, dataset).extractor is None diff --git a/tests/unit/openai/test_transcriptions_adapter.py b/tests/unit/openai/test_transcriptions_adapter.py index 584c9b030..710d85909 100644 --- a/tests/unit/openai/test_transcriptions_adapter.py +++ b/tests/unit/openai/test_transcriptions_adapter.py @@ -106,10 +106,6 @@ def test_missing_audio_file_raises_keyerror(self): with pytest.raises(KeyError, match="audio_file"): Adapter.encode_query(Query(id="q", data={"model": "m"})) - def test_streaming_request_raises(self, audio_file: Path): - with pytest.raises(ValueError, match="non-streaming"): - Adapter.encode_query(_query(audio_file, stream=True)) - def test_missing_audio_on_disk_raises(self, tmp_path: Path): query = Query( id="q", data={"audio_file": str(tmp_path / "gone.wav"), "model": "m"} @@ -118,6 +114,39 @@ def test_missing_audio_on_disk_raises(self, tmp_path: Path): Adapter.encode_query(query) +@pytest.mark.unit +class TestBodyInjection: + """Dataset-controlled strings must not be able to forge headers or parts.""" + + def test_quote_in_filename_is_escaped(self, tmp_path: Path): + path = tmp_path / 'ev"il.wav' + path.write_bytes(WAV_BYTES) + body = Adapter.encode_query(_query(path)) + # The raw quote would close filename= early; %22 is what servers decode. + assert b'filename="ev%22il.wav"' in body + assert b'filename="ev"il.wav"' not in body + + def test_crlf_in_filename_is_escaped(self, tmp_path: Path): + # A bare CRLF would end the Content-Disposition line and let the rest of + # the name be read as its own header. + path = tmp_path / "a\r\nX-Injected: yes\r\n\r\nx.wav" + path.write_bytes(WAV_BYTES) + body = Adapter.encode_query(_query(path)) + assert b"%0D%0AX-Injected: yes" in body + assert b"\r\nX-Injected: yes\r\n\r\nx.wav" not in body + + def test_field_value_carrying_the_delimiter_is_rejected(self, audio_file: Path): + boundary = Adapter.CONTENT_TYPE.split("boundary=", 1)[1] + query = _query(audio_file, prompt=f"x\r\n--{boundary}\r\nforged") + with pytest.raises(ValueError, match="delimiter"): + Adapter.encode_query(query) + + def test_ordinary_multiline_field_value_is_allowed(self, audio_file: Path): + # CRLF is legal inside a part body; only the delimiter is a forgery. + query = _query(audio_file, prompt="line one\r\nline two") + assert b"line one\r\nline two" in Adapter.encode_query(query) + + @pytest.mark.unit class TestDecodeResponse: def test_decodes_text_field(self): @@ -168,6 +197,19 @@ def test_adds_model_and_response_format(self): assert row["response_format"] == "json" assert row["temperature"] == 0.0 + def test_language_is_sent_by_default(self): + # Without it the server auto-detects per clip, which is a different + # workload from the MLPerf reference's pinned-language decode. + df = pd.DataFrame([{"audio_file": "/a.wav", "transcript": "hello"}]) + assert self._apply(df).iloc[0]["language"] == "en" + + def test_default_language_reaches_the_request_body(self, audio_file: Path): + df = pd.DataFrame([{"audio_file": str(audio_file), "transcript": "hello"}]) + row = self._apply(df).iloc[0] + body = Adapter.encode_query(Query(id="q", data=row.to_dict())) + assert b'name="language"' in body + assert b"\r\n\r\nen\r\n" in body + def test_dataset_language_overrides_defaults(self): df = pd.DataFrame( [{"audio_file": "/a.wav", "transcript": "hola", "language": "es"}] From fd1ba14a71e0eb2371a9f5513c4be5b6385446de Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Tue, 3 Nov 2026 18:42:51 -0800 Subject: [PATCH 3/4] fix(whisper): address review and correct the golden precision label Review feedback on the speech2text workload: - Drop tests/integration/openai/__init__.py and fold its single-use conftest.py into the test module. The test is renamed to a unique basename because pytest's prepend import mode would otherwise collide it with tests/unit/openai/test_transcriptions_adapter.py. - Keep the R2 download local to the whisper loader. dataset.py, bfcl_v4 and open_orca return to their prior state; the pinned downloader commit, host allowlist and subprocess timeout now live on WhisperLibriSpeech as class attributes, mirroring BFCLv4. - Stop the probe's --api-type help advertising api_types the probe refuses. show_choices is off because cyclopts otherwise renders the full APIType enum, which includes those entries regardless of the help string. Separately, register the whisper golden accuracy as bfloat16 rather than fp32. The field records the precision the value was obtained at, and the MLCommons reference SUT hardcodes dtype = "bfloat16". A healthy run scores a little above the 97.9329% reference. Scoring is not the cause: evaluation/wer.py reproduces upstream's compute_wer_with_concatenation with zero per-sample disagreement over the full 1633-clip corpus, both candidate EnglishTextNormalizer implementations give the same corpus WER, and the max_duration filter drops no clips. The gap is the serving path -- the reference runs vLLM V0 offline with an explicit decoder prompt and in-process librosa decode, while this example goes over HTTP and lets the server own both. The Whisper example README now records that comparison. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 36 +++--- examples/12_Whisper_Example/README.md | 50 ++++++-- .../12_Whisper_Example/offline_whisper.yaml | 45 ++------ src/inference_endpoint/commands/probe.py | 5 +- .../config/rulesets/mlcommons/models.py | 7 +- src/inference_endpoint/config/schema.py | 46 ++++---- src/inference_endpoint/core/types.py | 31 +++++ .../dataset_manager/dataset.py | 89 -------------- .../predefined/bfcl_v4/__init__.py | 55 ++++++++- .../predefined/open_orca/__init__.py | 32 ++++- .../whisper_librispeech/__init__.py | 91 ++++++++++++++- src/inference_endpoint/videogen/adapter.py | 8 +- .../commands/test_probe_command.py | 19 ++- tests/integration/openai/__init__.py | 14 --- tests/integration/openai/conftest.py | 109 ------------------ ...est_transcriptions_adapter_integration.py} | 90 ++++++++++++++- tests/unit/config/test_schema.py | 68 ++++++++++- .../dataset_manager/test_bfcl_v4_dataset.py | 46 +++----- .../test_whisper_librispeech_dataset.py | 11 +- 19 files changed, 498 insertions(+), 354 deletions(-) delete mode 100644 tests/integration/openai/__init__.py delete mode 100644 tests/integration/openai/conftest.py rename tests/integration/openai/{test_transcriptions_adapter.py => test_transcriptions_adapter_integration.py} (74%) diff --git a/AGENTS.md b/AGENTS.md index 2cbcc8b20..db74f90c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,23 +87,23 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint ### Key Components -| Component | Location | Purpose | -| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Load Generator** | `src/inference_endpoint/load_generator/` | Central orchestrator: `BenchmarkSession` owns the lifecycle, `PhaseIssuer` drives per-phase execution, `TimedIssueStrategy`/`BurstStrategy`/`ConcurrencyStrategy` control timing. Emits `ERROR` before `COMPLETE` for failed queries (metrics aggregator depends on this order). | -| **Endpoint Client** | `src/inference_endpoint/endpoint_client/` | Multi-process HTTP workers communicating via ZMQ IPC. `HTTPEndpointClient` is the main entry point | -| **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface; `download_from_r2()` is the single path for fetching MLCommons-hosted corpora (pinned downloader commit + host allowlist + subprocess timeout; `whisper_librispeech`, `bfcl_v4` and `open_orca` all go through it). It needs its own timeout because dataset generation runs in `setup_benchmark`, before `RunWatchdog` arms the whole-run deadline | -| **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | -| **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` builds reports from the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Sets `complete = (state == "complete" and n_pending_tasks == 0)` and renders an explicit warning for `INTERRUPTED` runs. Keeps the steady-window verdict on `Report.steady_state` and includes it in `to_json` (unlike `accuracy`). | -| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`) — the declared user surface, distinct from the resolved `runtime_settings.py`; `Timeouts` (`config/schema.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_issue_duration_ms` stays on `settings.runtime`; client worker-lifecycle timeouts stay on `settings.client`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | -| **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | -| **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | -| **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | -| **DeepSeek-R1 (MLPerf)** | `src/inference_endpoint/evaluation/scoring.py` (`LegacyMLPerfDeepSeekR1Scorer`), `examples/07_DeepSeekR1_Example/` | MLPerf DeepSeek-R1 accuracy. TensorRT-LLM is OpenAI-compatible, so it is served via `api_type: openai` / `openai_completions` (no dedicated trtllm adapter). The combined multi-subset eval (`math500`/`aime`/`gpqa`/`mmlu_pro`/`livecodebench`) is the official MLCommons `eval_accuracy.py`, run out-of-process via `uv run --project` against the isolated subproject at `src/inference_endpoint/evaluation/legacy_mlperf_deepseek_r1/` (a uv subproject excluded from the parent wheel; mirrors the VBench pattern). The example feeds the exact MLPerf prompt via pre-tokenized `input_tokens` to `/v1/completions`. | -| **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | -| **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | -| **Whisper (MLPerf speech2text)** | `src/inference_endpoint/openai/transcriptions_adapter.py`, `dataset_manager/predefined/whisper_librispeech/`, `evaluation/wer.py` | MLPerf speech2text (whisper-large-v3 / LibriSpeech). `api_type: openai_transcriptions` posts a `multipart/form-data` audio upload to `/v1/audio/transcriptions` — the only route that serves Whisper on vLLM. The body is not JSON, so the adapter declares `HttpRequestAdapter.CONTENT_TYPE` and the worker folds it into the pre-encoded request prefix once at setup (`endpoint_client/http.py`); the multipart boundary is therefore a module constant. `Query.data` carries the clip's _path_ and the worker reads the bytes in `encode_query`, so the corpus never crosses the ZMQ frame. Dataset-controlled header params are percent-escaped and a field value carrying the delimiter is rejected, so a row cannot forge a part. Non-streaming (`NonStreamingAccumulator`). `language` is sent on every request (`DEFAULT_LANGUAGE`, a dataset column overrides it): the reference pins the decoder language instead of auto-detecting per clip. The `whisper_librispeech` dataset downloads the MLCommons-prepared `dev-all-repack.json` corpus via `download_from_r2` (`generate_params.manifest` overrides); the ~30 s repackaging stays upstream because different clip boundaries change the WER. `WERScorer` (`eval_method: wer`) reports `100 - WER%` corpus-wide using the ported MLPerf metric in `evaluation/wer.py` plus `transformers`' `EnglishTextNormalizer`; parity with upstream is pinned by `UPSTREAM_VECTORS` in `tests/unit/evaluation/test_wer.py`. An issued sample with no COMPLETE event scores as an empty transcript (so dropping samples cannot raise the score) and sets `complete = False`. A configured `extractor` is rejected, and `preflight` resolves the normalizer before the run rather than after the perf phase. `_resolve_and_validate` keys three constraints on this api_type: `streaming: on` is an error (AUTO resolves OFF even online), `metrics_isl` is forced False (no text prompt), and `warmup.salt` is an error. Registered for submission checking as `models.Whisper_large_v3` / `datasets.LibriSpeechDevAll`. See examples/12_Whisper_Example/. | -| **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | -| **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | +| Component | Location | Purpose | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Load Generator** | `src/inference_endpoint/load_generator/` | Central orchestrator: `BenchmarkSession` owns the lifecycle, `PhaseIssuer` drives per-phase execution, `TimedIssueStrategy`/`BurstStrategy`/`ConcurrencyStrategy` control timing. Emits `ERROR` before `COMPLETE` for failed queries (metrics aggregator depends on this order). | +| **Endpoint Client** | `src/inference_endpoint/endpoint_client/` | Multi-process HTTP workers communicating via ZMQ IPC. `HTTPEndpointClient` is the main entry point | +| **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface | +| **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | +| **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` builds reports from the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Sets `complete = (state == "complete" and n_pending_tasks == 0)` and renders an explicit warning for `INTERRUPTED` runs. Keeps the steady-window verdict on `Report.steady_state` and includes it in `to_json` (unlike `accuracy`). | +| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`) — the declared user surface, distinct from the resolved `runtime_settings.py`; `Timeouts` (`config/schema.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_issue_duration_ms` stays on `settings.runtime`; client worker-lifecycle timeouts stay on `settings.client`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | +| **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | +| **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | +| **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | +| **DeepSeek-R1 (MLPerf)** | `src/inference_endpoint/evaluation/scoring.py` (`LegacyMLPerfDeepSeekR1Scorer`), `examples/07_DeepSeekR1_Example/` | MLPerf DeepSeek-R1 accuracy. TensorRT-LLM is OpenAI-compatible, so it is served via `api_type: openai` / `openai_completions` (no dedicated trtllm adapter). The combined multi-subset eval (`math500`/`aime`/`gpqa`/`mmlu_pro`/`livecodebench`) is the official MLCommons `eval_accuracy.py`, run out-of-process via `uv run --project` against the isolated subproject at `src/inference_endpoint/evaluation/legacy_mlperf_deepseek_r1/` (a uv subproject excluded from the parent wheel; mirrors the VBench pattern). The example feeds the exact MLPerf prompt via pre-tokenized `input_tokens` to `/v1/completions`. | +| **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | +| **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | +| **Whisper (MLPerf speech2text)** | `src/inference_endpoint/openai/transcriptions_adapter.py`, `dataset_manager/predefined/whisper_librispeech/`, `evaluation/wer.py` | MLPerf speech2text (whisper-large-v3 / LibriSpeech). `api_type: openai_transcriptions` posts a `multipart/form-data` audio upload to `/v1/audio/transcriptions` — the only route that serves Whisper on vLLM — so the adapter declares `HttpRequestAdapter.CONTENT_TYPE` and the worker folds it into the pre-encoded request prefix once at setup. `Query.data` carries the clip's _path_; the worker reads the bytes in `encode_query`, so the corpus never crosses the ZMQ frame. Non-streaming (`NonStreamingAccumulator`). `whisper_librispeech` fetches the MLCommons-prepared `dev-all-repack.json` via its own `_download_from_r2` (pinned `mlc-r2-downloader.sh` commit + host allowlist + subprocess timeout, kept local to the loader); it needs its own timeout because dataset generation runs in `setup_benchmark`, before `RunWatchdog` arms the whole-run deadline. The repackaging stays upstream because different clip boundaries change the WER. `WERScorer` (`eval_method: wer`) reports `100 - WER%` corpus-wide from the ported metric in `evaluation/wer.py`, pinned to upstream by `UPSTREAM_VECTORS` in `tests/unit/evaluation/test_wer.py`. Reference numbers are registered as `models.Whisper_large_v3` / `datasets.LibriSpeechDevAll`. Docstrings in `transcriptions_adapter.py` and `WERScorer` own the per-decision rationale; see examples/12_Whisper_Example/. | +| **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | +| **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | | **Validation** | `src/inference_endpoint/validation/`, `docs/validation/` | Cohort checker for submission uploads. A cohort is a directory of YAML checks (four scope files + `valid_refs.yaml` + a `cohort.yaml` naming the packaged ruleset); `python -m inference_endpoint.validation COHORT SUBMISSION` runs them against an upload root (`results///r/`) and reports holds/failures per check. Exit codes: 0 holds, 1 blocking failure, 2 refusal. See docs/validation/README.md. | ### Hot-Path Architecture @@ -358,7 +358,7 @@ src/inference_endpoint/ │ ├── output_caching_test.py # OutputCachingAudit (MLPerf TEST04): caching detection (reference vs fixed-sample QPS) │ └── README.md # TEST04 output-caching audit usage (WAN 2.2 example) ├── core/ -│ ├── types.py # APIType, Query, QueryResult, StreamChunk, QueryStatus (msgspec Structs) +│ ├── types.py # APIType (default_route + supports_streaming/has_text_prompt capabilities), Query, QueryResult, StreamChunk, QueryStatus (msgspec Structs) │ └── record.py # EventRecord — transport record used by event logger and ZMQ transport ├── load_generator/ │ ├── session.py # BenchmarkSession, PhaseIssuer, PhaseConfig, PhaseResult, SessionResult diff --git a/examples/12_Whisper_Example/README.md b/examples/12_Whisper_Example/README.md index 1794e8790..15c57a671 100644 --- a/examples/12_Whisper_Example/README.md +++ b/examples/12_Whisper_Example/README.md @@ -107,7 +107,7 @@ most often a manifest that is not the repackaged `dev-all-repack.json`. | Route + multipart upload | `openai/transcriptions_adapter.py` (`api_type: openai_transcriptions`) | | Dataset | `dataset_manager/predefined/whisper_librispeech/` | | Metric | `evaluation/wer.py` + `WERScorer` (`eval_method: wer`) | -| Accuracy gate | `config/rulesets/mlcommons/models.py` (`Whisper_large_v3`) | +| Reference numbers | `config/rulesets/mlcommons/models.py` (`Whisper_large_v3`) | Notes on the run config. All three constraints below are enforced by config validation keyed on `api_type: openai_transcriptions`, so a hand-written config @@ -127,12 +127,42 @@ gets an error rather than a silently odd run: `<|startoftranscript|><|en|><|transcribe|><|notimestamps|>` prompt instead of auto-detecting the language per clip, which is what the MLPerf reference does. -## Submission checking - -The reference accuracy and the 0.99 factor are registered in -`config/rulesets/mlcommons/models.py`, so `submission_ref` and -`compliance/checker.py` can enforce them rather than leaving the numbers in -prose. Both are transcriptions of the MLPerf Inference submission checker's own -constants, where the gate is `"whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99)` -and the required sample count is 1633 — the same `100 - WER` scale `WERScorer` -reports on. +## Reference numbers + +The reference accuracy, the 0.99 factor and the 1633-sample count are registered +as reference data in `config/rulesets/mlcommons/models.py` (`Whisper_large_v3`) +and `datasets.py` (`LibriSpeechDevAll`), so a `submission_ref` run resolves them +from the ruleset instead of the numbers living only in this file. They +transcribe the MLPerf Inference submission checker's own constants, where the +gate is `"whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99)` — the same +`100 - WER` scale `WERScorer` reports on. + +The in-repo `compliance/checker.py` does **not** cover this workload: its +accuracy gate resolves BFCL metric keys only, and its config-lock expects a +single-stream run (`num_workers: 1`), which this example is not. Checking a +whisper submission against the gate is the external MLPerf Inference submission +checker's job. + +### Expect a score a little above the reference + +A healthy run scores slightly better than 97.9329%, not equal to it. The scoring +is bit-exact with upstream — `evaluation/wer.py` reproduces +`compute_wer_with_concatenation` from the MLCommons reference +(https://github.com/mlcommons/inference/tree/master/speech2text) with zero +per-sample disagreement over the full 1633-clip corpus, and the two candidate +`EnglishTextNormalizer` implementations (`transformers` and `openai-whisper`) +give the same corpus WER. The difference is the serving path, and it is inherent +to benchmarking an endpoint: + +| | MLPerf reference SUT | This example | +| -------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------ | +| Engine | vLLM V0, offline `LLM.generate()` | any OpenAI-compatible server, over HTTP | +| Decoder prompt | explicit `<\|startoftranscript\|><\|en\|><\|transcribe\|><\|notimestamps\|>` | built by the server's `/v1/audio/transcriptions` layer | +| Audio decode | `librosa.load(path, sr=16000)` in-process | server-side, from the multipart upload | +| Sampling | `temperature=0, top_p=1.0, max_tokens=200` | `temperature=0`, length left to the server | + +That reference's own SUT pins `dtype = "bfloat16"`, which is why +`Whisper_large_v3.golden_accuracy` is registered as `bfloat16` rather than fp32. +Serving fp32 instead moves the score by well under 0.1 points, so a gap of that +size is engine variance rather than a scoring fault. `max_tokens=200` never +binds in practice — observed output lengths top out near 150 tokens. diff --git a/examples/12_Whisper_Example/offline_whisper.yaml b/examples/12_Whisper_Example/offline_whisper.yaml index 80bbd16dd..72f2a8b29 100644 --- a/examples/12_Whisper_Example/offline_whisper.yaml +++ b/examples/12_Whisper_Example/offline_whisper.yaml @@ -1,21 +1,11 @@ -# Offline run for MLPerf speech2text (whisper-large-v3 / LibriSpeech) +# Offline run for MLPerf speech2text (whisper-large-v3 / LibriSpeech). # -# Issues the MLCommons-prepared LibriSpeech corpus against an OpenAI-compatible -# transcription endpoint (POST /v1/audio/transcriptions) and scores Word Error -# Rate. Reference accuracy is 97.9329% (WER 2.0671%); a submission needs 99% of -# that, i.e. >= 96.9536. +# Needs a server on /v1/audio/transcriptions: +# vllm serve openai/whisper-large-v3 --port 8000 # -# Prerequisites: -# - A server exposing /v1/audio/transcriptions for whisper-large-v3: -# vllm serve openai/whisper-large-v3 --port 8000 -# -# The dataset needs no setup: the MLCommons-prepared corpus (dev-all-repack.json -# + WAVs) is downloaded into the dataset cache on first run. To use a copy you -# already have, add generate_params.manifest to both dataset entries (or export -# $WHISPER_LIBRISPEECH_MANIFEST). -# -# The audio bytes are read by the worker inside the measured request, so keep -# the WAV directory on fast local storage - a network mount inflates latency. +# The corpus downloads itself on first run. See README.md for the reference +# accuracy, the storage requirement for the WAVs, how to point at a local +# manifest, and what api_type validation derives rather than reading here. # # For a smoke run, set runtime.n_samples_to_issue and both `samples` to 8. @@ -23,17 +13,13 @@ name: "offline-whisper-librispeech" version: "1.0" type: "offline" +# Streaming, metrics_isl and warmup salt are all settled by api_type +# validation, so they are deliberately absent below rather than set to the +# value they would be forced to. README.md ("Notes on the run config") lists +# what is derived and why. model_params: name: "openai/whisper-large-v3" - # /v1/audio/transcriptions has no SSE mode for Whisper. Stated here for the - # reader; config validation also rejects "on" for this api_type and resolves - # "auto" to "off", so an online whisper config cannot silently stream. - streaming: "off" temperature: 0.0 - # The transcription language is sent on every request (language=en, from the - # adapter's DEFAULT_LANGUAGE) so the server builds Whisper's pinned-language - # decoder prompt instead of auto-detecting per clip, which is what the MLPerf - # reference does. A dataset `language` column overrides it per row. datasets: # setup_benchmark() requires a performance dataset to drive issuance; the @@ -62,18 +48,7 @@ settings: num_workers: 8 warmup: - # Salt prepends random hex to a text prompt to defeat prefix caching; an - # audio upload has no prompt, so config validation rejects salt for this - # api_type. Warmup itself is fine with salt off. enabled: false - salt: false - - # Audio rows carry no text prompt, so there is no input sequence length to - # measure. Validation forces this off for openai_transcriptions, so the line - # is documentation rather than a requirement. The load generator still logs - # one "ISL is unavailable" warning per run, which is expected here. OSL and - # latency are unaffected. - metrics_isl: false endpoint_config: endpoints: diff --git a/src/inference_endpoint/commands/probe.py b/src/inference_endpoint/commands/probe.py index c9b1125a2..de4af746d 100644 --- a/src/inference_endpoint/commands/probe.py +++ b/src/inference_endpoint/commands/probe.py @@ -62,7 +62,10 @@ class ProbeConfig(BaseModel): APIType, cyclopts.Parameter( alias="--api-type", - help="API type: openai, openai_transcriptions, sglang, or videogen", + help="API type: openai, openai_completions, or sglang", + # cyclopts otherwise renders choices from every APIType member, + # including the ones UNSUPPORTED_PROBE_API_TYPES refuses. + show_choices=False, ), ] = APIType.OPENAI requests: int = Field(10, ge=1) diff --git a/src/inference_endpoint/config/rulesets/mlcommons/models.py b/src/inference_endpoint/config/rulesets/mlcommons/models.py index d1edd56f2..e2cd5c765 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/models.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/models.py @@ -154,9 +154,14 @@ def __hash__(self) -> int: # # equal_issue_mode is off: the clips are fixed-length (~30 s) by construction, so # the non-uniform-input case it exists for does not arise. +# +# Precision is bfloat16 where every other entry here is fp32: the MLCommons +# reference SUT (https://github.com/mlcommons/inference/tree/master/speech2text) +# hardcodes `dtype = "bfloat16"`. The accuracy_target_settings factor still +# multiplies this value, as it does for every other entry. Whisper_large_v3 = _Model( "whisper-large-v3", - golden_accuracy=("fp32", {"accuracy": 97.9329}), + golden_accuracy=("bfloat16", {"accuracy": 97.9329}), accuracy_target_settings=[{"accuracy": (0.99,)}], dataset=datasets.LibriSpeechDevAll, use_equal_issue_mode=False, diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index b0e7ac294..af6fe290c 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -1171,23 +1171,24 @@ def _resolve_and_validate(self) -> Self: self.benchmark_mode if self.type == TestType.SUBMISSION else self.type ) - # An audio transcription endpoint has no SSE mode at all, so streaming is - # a property of the api_type rather than of the test type: AUTO must not - # resolve to ON here the way it does for an online text run. - non_streaming_api = ( - self.endpoint_config.api_type == APIType.OPENAI_TRANSCRIPTIONS - ) + # Some endpoints have no SSE mode at all, so streaming is a property of + # the api_type rather than of the test type: AUTO must not resolve to ON + # for them the way it does for an online text run. + api_type = self.endpoint_config.api_type if self.model_params.streaming == StreamingMode.AUTO: mp_updates["streaming"] = ( StreamingMode.OFF - if non_streaming_api or effective_mode in (TestType.OFFLINE,) + if not api_type.supports_streaming + or effective_mode in (TestType.OFFLINE,) else StreamingMode.ON ) - elif non_streaming_api and self.model_params.streaming == StreamingMode.ON: + elif ( + not api_type.supports_streaming + and self.model_params.streaming == StreamingMode.ON + ): raise ValueError( - f"endpoint_config.api_type={self.endpoint_config.api_type.value} " - "has no streaming mode; set model_params.streaming to 'off' or " - "leave it 'auto'" + f"endpoint_config.api_type={api_type.value} has no streaming " + "mode; set model_params.streaming to 'off' or leave it 'auto'" ) if not self.model_params.name and self.submission_ref: @@ -1203,12 +1204,14 @@ def _resolve_and_validate(self) -> Self: if not self.model_params.name: raise ValueError("Required: --model-params.name [--model]") - if non_streaming_api: - # The request body is an audio upload with no text prompt, so there - # is no input sequence length to measure. Left on, the aggregator - # registers an ISL series that can only stay empty. Derived rather - # than demanded of the user: metrics_isl defaults to True, so every - # transcription config would otherwise have to turn it off by hand. + # Keyed on the absence of a text prompt, not on the absence of + # streaming: videogen is non-streaming but is text-to-video, so it + # keeps both of these. + if not api_type.has_text_prompt: + # No prompt means no input sequence length to measure. Left on, the + # aggregator registers an ISL series that can only stay empty. + # Derived rather than demanded of the user: metrics_isl defaults to + # True, so every such config would otherwise turn it off by hand. if self.settings.metrics_isl: object.__setattr__( self, @@ -1216,14 +1219,13 @@ def _resolve_and_validate(self) -> Self: self.settings.model_copy(update={"metrics_isl": False}), ) # Salt exists to defeat prefix caching by prepending random hex to a - # text prompt. Here the cache key is the audio, so salting cannot do - # that job, and a row with no 'prompt' at all is a hard error later. + # text prompt. With no prompt, salting cannot do that job, and a row + # without one is a hard error later. if self.settings.warmup.salt: raise ValueError( f"settings.warmup.salt is not supported with " - f"endpoint_config.api_type=" - f"{self.endpoint_config.api_type.value}: the request is an " - "audio upload, not a text prompt" + f"endpoint_config.api_type={api_type.value}: the request " + "carries no text prompt to salt" ) uses_swe_bench = any( diff --git a/src/inference_endpoint/core/types.py b/src/inference_endpoint/core/types.py index 62b6b1818..13019dcba 100644 --- a/src/inference_endpoint/core/types.py +++ b/src/inference_endpoint/core/types.py @@ -56,6 +56,37 @@ def default_route(self) -> str: case _: raise ValueError(f"Invalid API type: {self}") + @property + def supports_streaming(self) -> bool: + """Whether this API has an SSE mode at all. + + False means streaming is a property of the protocol rather than a user + choice: its adapter's ``decode_sse_message`` raises, and its registered + accumulator is a stub. Config validation resolves AUTO to OFF and + rejects an explicit ON rather than letting the run fail per request. + """ + return self not in _NON_STREAMING_API_TYPES + + @property + def has_text_prompt(self) -> bool: + """Whether a request body carries a text prompt from the dataset row. + + False means anything keyed on prompt text is meaningless here: there is + no input sequence length to tokenize and no prompt to salt. Distinct + from :attr:`supports_streaming` — videogen is non-streaming but is + text-to-video, so it keeps both. + """ + return self not in _NO_TEXT_PROMPT_API_TYPES + + +#: Listed rather than derived: a new api_type defaults to the generic text path, +#: which is the safe assumption, and ``default_route`` already forces an author +#: adding one to visit this class. +_NON_STREAMING_API_TYPES = frozenset({APIType.OPENAI_TRANSCRIPTIONS, APIType.VIDEOGEN}) + +#: Audio in, transcript out — no prompt column exists to measure or salt. +_NO_TEXT_PROMPT_API_TYPES = frozenset({APIType.OPENAI_TRANSCRIPTIONS}) + class QueryStatus(Enum): """Status of a query in its lifecycle. diff --git a/src/inference_endpoint/dataset_manager/dataset.py b/src/inference_endpoint/dataset_manager/dataset.py index 53f115ce3..4cff1fc3b 100644 --- a/src/inference_endpoint/dataset_manager/dataset.py +++ b/src/inference_endpoint/dataset_manager/dataset.py @@ -17,7 +17,6 @@ import inspect import os import random -import subprocess import warnings from abc import ABC from enum import Enum @@ -25,16 +24,9 @@ from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar -from urllib.parse import urlparse import numpy as np import pandas as pd - -# `requests` is barred from the benchmark's own HTTP path, which uses the -# httptools-based ConnectionPool in endpoint_client/. This is dataset setup: -# one blocking fetch of the pinned downloader script, run before any load is -# issued and never during a measured request. -import requests from datasets import load_dataset, load_from_disk from ..config.schema import APIType, ModelParams @@ -266,87 +258,6 @@ def load_from_huggingface( return ds[split].to_pandas() -#: Pinned mlcommons/r2-downloader commit used by download_from_r2. -R2_DOWNLOADER_COMMIT = "27da4421877f2831eeb615b43ee5098c4b70be7e" - -#: Dataset URIs are handed to a downloaded shell script, so the host is -#: restricted to MLCommons R2 storage. This stops a mis-set dataset URI from -#: pointing the downloader at an attacker-controlled location. -R2_ALLOWED_HOST = "mlcommons-storage.org" - -#: Wall-clock ceiling for one downloader invocation. Dataset generation runs in -#: ``setup_benchmark``, before ``RunWatchdog`` arms its whole-run deadline, so -#: nothing else would ever interrupt a stalled fetch. Sized for several GB over -#: a slow link rather than for a healthy transfer; callers that need more can -#: pass their own. -R2_DOWNLOAD_TIMEOUT_S = 2 * 60 * 60 - - -def download_from_r2( - uri: str, download_dir: Path, timeout_s: float | None = R2_DOWNLOAD_TIMEOUT_S -) -> None: - """Download an MLCommons-hosted dataset into ``download_dir``. - - Fetches the pinned ``mlc-r2-downloader.sh`` and runs it against the - dataset's ``.uri`` metadata. The script MD5-verifies every file it pulls and - fails its exit code on mismatch, so callers only need to locate the - artifact they expect afterwards. - - Args: - uri: The dataset's published ``.uri`` metadata URL. - download_dir: Directory the artifact is downloaded into. - timeout_s: Ceiling on the downloader subprocess; None waits forever. - - Raises: - ValueError: ``uri`` is not https on MLCommons R2 storage. - RuntimeError: the downloader exited non-zero or outran ``timeout_s``. - """ - parsed = urlparse(uri) - host = parsed.hostname or "" - if parsed.scheme != "https" or not ( - host == R2_ALLOWED_HOST or host.endswith("." + R2_ALLOWED_HOST) - ): - raise ValueError( - f"Refusing to download dataset from untrusted URI '{uri}': " - f"expected https on {R2_ALLOWED_HOST}" - ) - - download_dir.mkdir(parents=True, exist_ok=True) - script_path = download_dir / "mlc-r2-downloader.sh" - resp = requests.get( - "https://raw.githubusercontent.com/mlcommons/r2-downloader/" - f"{R2_DOWNLOADER_COMMIT}/mlc-r2-downloader.sh", - timeout=30, - ) - resp.raise_for_status() - script_path.write_bytes(resp.content) - script_path.chmod(0o755) - - logger.info("Downloading %s into %s", uri, download_dir) - try: - result = subprocess.run( - ["bash", str(script_path.resolve()), "-d", str(download_dir), uri], - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - text=True, - check=False, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as e: - raise RuntimeError( - f"R2 downloader for '{uri}' exceeded {timeout_s}s and was killed; " - f"whatever it had written is left under {download_dir}. Raise the " - "timeout, or point generate_params at a local copy of the corpus." - ) from e - finally: - script_path.unlink(missing_ok=True) - - if result.returncode != 0: - raise RuntimeError( - f"R2 downloader failed with code {result.returncode}: {result.stderr}" - ) - - def _check_unsaltable(sample: Any) -> DatasetValidationError.Reason | None: """Return the Reason a sample cannot be salted, or None if it can. diff --git a/src/inference_endpoint/dataset_manager/predefined/bfcl_v4/__init__.py b/src/inference_endpoint/dataset_manager/predefined/bfcl_v4/__init__.py index c12b33137..f91c190ea 100644 --- a/src/inference_endpoint/dataset_manager/predefined/bfcl_v4/__init__.py +++ b/src/inference_endpoint/dataset_manager/predefined/bfcl_v4/__init__.py @@ -22,13 +22,16 @@ import hashlib import json import os +import subprocess from logging import getLogger from pathlib import Path from typing import Any +from urllib.parse import urlparse import pandas as pd +import requests -from ...dataset import Dataset, download_from_r2 +from ...dataset import Dataset from . import presets logger = getLogger(__name__) @@ -165,6 +168,13 @@ class BFCLv4( # staging/testing via the BFCL_V4_DATASET_URI env var. R2_DATASET_URI: str | None = None R2_DATASET_URI_ENV = "BFCL_V4_DATASET_URI" + # Pinned mlcommons/r2-downloader commit (same convention as open_orca). + R2_DOWNLOADER_COMMIT = "27da4421877f2831eeb615b43ee5098c4b70be7e" + # The resolved dataset URI is handed to a downloaded shell script, so the + # host is restricted to MLCommons R2 storage. This stops a mis-set + # BFCL_V4_DATASET_URI env var from pointing the downloader at an + # attacker-controlled location. + R2_ALLOWED_HOST = "mlcommons-storage.org" @classmethod def generate( @@ -403,9 +413,8 @@ def _verify_sha256(cls, path: Path, expected: str) -> None: def _download_full_parquet_from_r2(cls, dst_path: Path) -> bool: """Download the full single-turn parquet from MLCommons R2 storage. - Fetching and running the pinned ``mlc-r2-downloader.sh`` is - ``download_from_r2``; this adds the SHA-256 verification and the - relocation to ``dst_path``. + Mirrors the OpenOrca pattern: fetch the pinned ``mlc-r2-downloader.sh``, + run it against the dataset ``.uri``, then SHA-256 verify the result. Returns True when a verified parquet is in place at ``dst_path``; returns False when no R2 URI is configured, in which case the caller builds the @@ -417,12 +426,48 @@ def _download_full_parquet_from_r2(cls, dst_path: Path) -> bool: if not uri: return False + # The uri is passed to a downloaded shell script; restrict it to https + # on MLCommons R2 storage before invoking anything. + parsed = urlparse(uri) + host = parsed.hostname or "" + if parsed.scheme != "https" or not ( + host == cls.R2_ALLOWED_HOST or host.endswith("." + cls.R2_ALLOWED_HOST) + ): + raise ValueError( + f"Refusing to download dataset from untrusted URI '{uri}': " + f"expected https on {cls.R2_ALLOWED_HOST}" + ) + download_dir = dst_path.parent download_dir.mkdir(parents=True, exist_ok=True) # Start from a clean target so a stale parquet can't shadow the fresh # download (the relocate below only fires when dst_path is absent). dst_path.unlink(missing_ok=True) - download_from_r2(uri, download_dir) + downloader_url = ( + "https://raw.githubusercontent.com/mlcommons/r2-downloader/" + f"{cls.R2_DOWNLOADER_COMMIT}/mlc-r2-downloader.sh" + ) + script_path = download_dir / "mlc-r2-downloader.sh" + resp = requests.get(downloader_url, timeout=30) + resp.raise_for_status() + script_path.write_bytes(resp.content) + script_path.chmod(0o755) + + try: + result = subprocess.run( + ["bash", str(script_path.resolve()), "-d", str(download_dir), uri], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"R2 downloader failed with code {result.returncode}: " + f"{result.stderr}" + ) + finally: + script_path.unlink(missing_ok=True) # The downloader may nest the file under a subdir per the .uri manifest; # relocate it to the expected cache path if so. diff --git a/src/inference_endpoint/dataset_manager/predefined/open_orca/__init__.py b/src/inference_endpoint/dataset_manager/predefined/open_orca/__init__.py index 58a31ada6..ed092c986 100644 --- a/src/inference_endpoint/dataset_manager/predefined/open_orca/__init__.py +++ b/src/inference_endpoint/dataset_manager/predefined/open_orca/__init__.py @@ -16,12 +16,14 @@ import gzip import hashlib import shutil +import subprocess from logging import getLogger from pathlib import Path import pandas as pd +import requests -from ...dataset import Dataset, download_from_r2 +from ...dataset import Dataset from . import presets logger = getLogger(__name__) @@ -125,8 +127,33 @@ def generate( # Dataset URL from README dataset_url = "https://inference.mlcommons-storage.org/metadata/llama-2-70b-open-orca-dataset.uri" + # Download the r2-downloader script into a temp file in the target dir + COMMIT_HASH = "27da4421877f2831eeb615b43ee5098c4b70be7e" + downloader_url = f"https://raw.githubusercontent.com/mlcommons/r2-downloader/{COMMIT_HASH}/mlc-r2-downloader.sh" download_dir = cache_dir - download_from_r2(dataset_url, download_dir) + script_path = cache_dir / "mlc-r2-downloader.sh" + r = requests.get(downloader_url, timeout=30) + r.raise_for_status() + script_path.write_bytes(r.content) + script_path.chmod(0o755) + + # Run the script with the dataset URL. + try: + # Use absolute path for the script to avoid path doubling when cwd is set + script_abs = str(script_path.resolve()) + result = subprocess.run( + ["bash", script_abs, "-d", str(download_dir), dataset_url], + stdout=subprocess.DEVNULL, # Suppress normal output + stderr=subprocess.PIPE, # Capture errors + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"R2 downloader failed with code {result.returncode}: {result.stderr}" + ) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"R2 downloader failed: {e}") from e # Script will generate a new 'open_orca' subdirectory with gzip'd pickle files gzip_dir = download_dir @@ -145,6 +172,7 @@ def generate( finally: for gz_path in gzip_dir.glob("*.pkl.gz"): gz_path.unlink() + script_path.unlink(missing_ok=True) __all__ = ["OpenOrca"] diff --git a/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py b/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py index 16deb669a..c80198b16 100644 --- a/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py +++ b/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py @@ -15,12 +15,20 @@ import json import os +import subprocess from logging import getLogger from pathlib import Path, PurePosixPath +from urllib.parse import urlparse import pandas as pd -from ...dataset import Dataset, download_from_r2 +# `requests` is barred from the benchmark's own HTTP path, which uses the +# httptools-based ConnectionPool in endpoint_client/. This is dataset setup: +# one blocking fetch of the pinned downloader script, run before any load is +# issued and never during a measured request. +import requests + +from ...dataset import Dataset logger = getLogger(__name__) @@ -73,6 +81,85 @@ class WhisperLibriSpeech(Dataset, dataset_id="whisper_librispeech"): ) R2_DATASET_URI_ENV = "WHISPER_LIBRISPEECH_DATASET_URI" + #: Pinned mlcommons/r2-downloader commit used by ``_download_from_r2``. + R2_DOWNLOADER_COMMIT = "27da4421877f2831eeb615b43ee5098c4b70be7e" + + #: The resolved dataset URI is handed to a downloaded shell script, so the + #: host is restricted to MLCommons R2 storage. This stops a mis-set + #: ``$WHISPER_LIBRISPEECH_DATASET_URI`` from pointing the downloader at an + #: attacker-controlled location. + R2_ALLOWED_HOST = "mlcommons-storage.org" + + #: Wall-clock ceiling for the downloader invocation. Dataset generation runs + #: in ``setup_benchmark``, before ``RunWatchdog`` arms its whole-run + #: deadline, so nothing else would ever interrupt a stalled fetch. Sized for + #: several GB of audio over a slow link, not for a healthy transfer. + R2_DOWNLOAD_TIMEOUT_S = 2 * 60 * 60 + + @classmethod + def _download_from_r2(cls, uri: str, download_dir: Path) -> None: + """Download the MLCommons-hosted speech2text corpus into ``download_dir``. + + Fetches the pinned ``mlc-r2-downloader.sh`` and runs it against the + dataset's ``.uri`` metadata. The script MD5-verifies every file it pulls + and fails its exit code on mismatch, so the caller only needs to locate + the manifest afterwards. + + Args: + uri: The dataset's published ``.uri`` metadata URL. + download_dir: Directory the corpus is downloaded into. + + Raises: + ValueError: ``uri`` is not https on MLCommons R2 storage. + RuntimeError: the downloader exited non-zero or outran + ``R2_DOWNLOAD_TIMEOUT_S``. + """ + parsed = urlparse(uri) + host = parsed.hostname or "" + if parsed.scheme != "https" or not ( + host == cls.R2_ALLOWED_HOST or host.endswith("." + cls.R2_ALLOWED_HOST) + ): + raise ValueError( + f"Refusing to download dataset from untrusted URI '{uri}': " + f"expected https on {cls.R2_ALLOWED_HOST}" + ) + + download_dir.mkdir(parents=True, exist_ok=True) + script_path = download_dir / "mlc-r2-downloader.sh" + resp = requests.get( + "https://raw.githubusercontent.com/mlcommons/r2-downloader/" + f"{cls.R2_DOWNLOADER_COMMIT}/mlc-r2-downloader.sh", + timeout=30, + ) + resp.raise_for_status() + script_path.write_bytes(resp.content) + script_path.chmod(0o755) + + logger.info("Downloading %s into %s", uri, download_dir) + try: + result = subprocess.run( + ["bash", str(script_path.resolve()), "-d", str(download_dir), uri], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=cls.R2_DOWNLOAD_TIMEOUT_S, + ) + except subprocess.TimeoutExpired as e: + raise RuntimeError( + f"R2 downloader for '{uri}' exceeded {cls.R2_DOWNLOAD_TIMEOUT_S}s " + f"and was killed; whatever it had written is left under " + f"{download_dir}. Raise the timeout, or point generate_params at " + "a local copy of the corpus." + ) from e + finally: + script_path.unlink(missing_ok=True) + + if result.returncode != 0: + raise RuntimeError( + f"R2 downloader failed with code {result.returncode}: {result.stderr}" + ) + @classmethod def generate( cls, @@ -167,7 +254,7 @@ def _resolve_manifest( "to %s; pass generate_params.manifest to use a local copy instead.", download_dir, ) - download_from_r2( + cls._download_from_r2( os.environ.get(cls.R2_DATASET_URI_ENV) or cls.R2_DATASET_URI, download_dir ) found = cls._find_manifest(download_dir) diff --git a/src/inference_endpoint/videogen/adapter.py b/src/inference_endpoint/videogen/adapter.py index a875500a2..ff4f9a704 100644 --- a/src/inference_endpoint/videogen/adapter.py +++ b/src/inference_endpoint/videogen/adapter.py @@ -162,11 +162,9 @@ class VideoGenAccumulator: Video generation requests are non-streaming HTTP, so this class should never be exercised. `get_final_output` raises rather than returning an - empty `QueryResult` because a streamed body with no `data:` frames never - reaches `decode_sse_message`, so nothing raises and the stream ends with - nothing accumulated — an empty result would record that as a successful - zero-output query. See `NonStreamingAccumulator`, which shares this - contract. + empty `QueryResult`, because the worker's SSE path swallows the + `NotImplementedError` from `decode_sse_message` and would otherwise + surface zero-output queries as successful. """ def __init__(self, query_id: str, stream_all_chunks: bool) -> None: diff --git a/tests/integration/commands/test_probe_command.py b/tests/integration/commands/test_probe_command.py index a10885843..07e964d2b 100644 --- a/tests/integration/commands/test_probe_command.py +++ b/tests/integration/commands/test_probe_command.py @@ -23,8 +23,13 @@ - Error handling with failing endpoints """ +import cyclopts import pytest -from inference_endpoint.commands.probe import ProbeConfig, execute_probe +from inference_endpoint.commands.probe import ( + UNSUPPORTED_PROBE_API_TYPES, + ProbeConfig, + execute_probe, +) from inference_endpoint.config.schema import APIType from inference_endpoint.exceptions import ExecutionError, InputValidationError @@ -98,6 +103,18 @@ def test_probe_rejects_videogen_api_type(self): with pytest.raises(InputValidationError, match="videogen"): execute_probe(config) + @pytest.mark.integration + def test_api_type_help_omits_unsupported_types(self): + """The --api-type help must not advertise an api_type the probe refuses.""" + param = next( + m + for m in ProbeConfig.model_fields["api_type"].metadata + if isinstance(m, cyclopts.Parameter) + ) + assert param.show_choices is False, "enum choices would list refused types" + for api_type in UNSUPPORTED_PROBE_API_TYPES: + assert api_type.value not in (param.help or "") + @pytest.mark.integration def test_probe_with_invalid_endpoint(self): """Test probe fails gracefully with invalid endpoint.""" diff --git a/tests/integration/openai/__init__.py b/tests/integration/openai/__init__.py deleted file mode 100644 index 467079831..000000000 --- a/tests/integration/openai/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/integration/openai/conftest.py b/tests/integration/openai/conftest.py deleted file mode 100644 index c281bd2d8..000000000 --- a/tests/integration/openai/conftest.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Integration fixtures for the OpenAI transcriptions adapter. - -`MockTranscriptionServer` subclasses `EchoServer` to reuse its background-thread -aiohttp lifecycle (port discovery, start/stop, ready event); only the route -differs. It parses the upload with aiohttp's real multipart reader, so a -malformed body fails here rather than silently round-tripping. -""" - -import json -from collections.abc import Generator -from pathlib import Path - -import pytest -from aiohttp import web -from inference_endpoint.testing.echo_server import EchoServer - -#: 16-bit PCM WAV header plus a short run of samples. Only the bytes matter - -#: nothing in the client or the mock decodes the audio. -WAV_HEADER = ( - b"RIFF" + (36 + 512).to_bytes(4, "little") + b"WAVEfmt " - b"\x10\x00\x00\x00\x01\x00\x01\x00\x80>\x00\x00\x00}\x00\x00\x02\x00\x10\x00" - b"data" + (512).to_bytes(4, "little") -) - - -def write_wav(path: Path, payload: bytes | None = None) -> bytes: - """Write a dummy WAV at `path` and return its bytes.""" - data = WAV_HEADER + (payload if payload is not None else bytes(range(256)) * 2) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - return data - - -class MockTranscriptionServer(EchoServer): - """vLLM-shaped mock for `POST /v1/audio/transcriptions`. - - Returns the transcript registered for the uploaded file's basename, or the - basename itself when none is registered. Records each request's form fields - and uploaded bytes on `self.requests` so tests can assert on the wire - format rather than only on the decoded result. - """ - - def __init__(self, transcripts: dict[str, str] | None = None) -> None: - super().__init__(port=0) - self.transcripts = transcripts or {} - self.requests: list[dict] = [] - - def _register_routes(self, app: web.Application) -> None: - app.router.add_post("/v1/audio/transcriptions", self._handle_transcription) - - async def _handle_transcription(self, request: web.Request) -> web.Response: - if not request.content_type.startswith("multipart/form-data"): - return web.Response( - status=415, text=f"expected multipart, got {request.content_type}" - ) - - fields: dict[str, str] = {} - filename = "" - audio = b"" - reader = await request.multipart() - while True: - part = await reader.next() - if part is None: - break - if part.name == "file": - filename = part.filename or "" - audio = bytes(await part.read(decode=False)) - else: - fields[part.name] = (await part.read(decode=False)).decode() - - self.requests.append({"fields": fields, "filename": filename, "audio": audio}) - if not audio: - return web.Response(status=400, text="empty file part") - - text = self.transcripts.get(filename, filename) - if fields.get("response_format", "json") != "json": - return web.Response( - status=400, text="mock only serves response_format=json" - ) - return web.Response( - body=json.dumps({"text": text, "duration": 29.5}).encode(), - content_type="application/json", - ) - - -@pytest.fixture -def mock_transcription_server( - request: pytest.FixtureRequest, -) -> Generator[MockTranscriptionServer, None, None]: - transcripts = getattr(request, "param", None) - server = MockTranscriptionServer(transcripts=transcripts) - server.start() - yield server - server.stop() diff --git a/tests/integration/openai/test_transcriptions_adapter.py b/tests/integration/openai/test_transcriptions_adapter_integration.py similarity index 74% rename from tests/integration/openai/test_transcriptions_adapter.py rename to tests/integration/openai/test_transcriptions_adapter_integration.py index 9ab7815fe..d73872b62 100644 --- a/tests/integration/openai/test_transcriptions_adapter.py +++ b/tests/integration/openai/test_transcriptions_adapter_integration.py @@ -25,11 +25,13 @@ import random import urllib.error import urllib.request +from collections.abc import Generator from pathlib import Path from urllib.parse import urljoin import msgspec import pytest +from aiohttp import web from inference_endpoint import metrics from inference_endpoint.config.runtime_settings import RuntimeSettings from inference_endpoint.config.schema import ( @@ -56,10 +58,9 @@ from inference_endpoint.openai.transcriptions_adapter import ( OpenAITranscriptionsAdapter, ) +from inference_endpoint.testing.echo_server import EchoServer from transformers.models.whisper.english_normalizer import EnglishTextNormalizer -from .conftest import MockTranscriptionServer, write_wav - MODEL = "openai/whisper-large-v3" DATASET_NAME = "whisper_librispeech" @@ -79,6 +80,91 @@ ] +#: 16-bit PCM WAV header plus a short run of samples. Only the bytes matter - +#: nothing in the client or the mock decodes the audio. +WAV_HEADER = ( + b"RIFF" + (36 + 512).to_bytes(4, "little") + b"WAVEfmt " + b"\x10\x00\x00\x00\x01\x00\x01\x00\x80>\x00\x00\x00}\x00\x00\x02\x00\x10\x00" + b"data" + (512).to_bytes(4, "little") +) + + +def write_wav(path: Path, payload: bytes | None = None) -> bytes: + """Write a dummy WAV at `path` and return its bytes.""" + data = WAV_HEADER + (payload if payload is not None else bytes(range(256)) * 2) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return data + + +class MockTranscriptionServer(EchoServer): + """vLLM-shaped mock for `POST /v1/audio/transcriptions`. + + Subclasses `EchoServer` to reuse its background-thread aiohttp lifecycle + (port discovery, start/stop, ready event); only the route differs. The + upload is parsed with aiohttp's real multipart reader, so a malformed body + fails here rather than silently round-tripping. + + Returns the transcript registered for the uploaded file's basename, or the + basename itself when none is registered. Records each request's form fields + and uploaded bytes on `self.requests` so tests can assert on the wire + format rather than only on the decoded result. + """ + + def __init__(self, transcripts: dict[str, str] | None = None) -> None: + super().__init__(port=0) + self.transcripts = transcripts or {} + self.requests: list[dict] = [] + + def _register_routes(self, app: web.Application) -> None: + app.router.add_post("/v1/audio/transcriptions", self._handle_transcription) + + async def _handle_transcription(self, request: web.Request) -> web.Response: + if not request.content_type.startswith("multipart/form-data"): + return web.Response( + status=415, text=f"expected multipart, got {request.content_type}" + ) + + fields: dict[str, str] = {} + filename = "" + audio = b"" + reader = await request.multipart() + while True: + part = await reader.next() + if part is None: + break + if part.name == "file": + filename = part.filename or "" + audio = bytes(await part.read(decode=False)) + else: + fields[part.name] = (await part.read(decode=False)).decode() + + self.requests.append({"fields": fields, "filename": filename, "audio": audio}) + if not audio: + return web.Response(status=400, text="empty file part") + + text = self.transcripts.get(filename, filename) + if fields.get("response_format", "json") != "json": + return web.Response( + status=400, text="mock only serves response_format=json" + ) + return web.Response( + body=json.dumps({"text": text, "duration": 29.5}).encode(), + content_type="application/json", + ) + + +@pytest.fixture +def mock_transcription_server( + request: pytest.FixtureRequest, +) -> Generator[MockTranscriptionServer, None, None]: + transcripts = getattr(request, "param", None) + server = MockTranscriptionServer(transcripts=transcripts) + server.start() + yield server + server.stop() + + class _NoOpPublisher: def publish(self, event_record: EventRecord) -> None: pass diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index 0fd7ca1cf..230554553 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -1206,8 +1206,8 @@ class TestTranscriptionsAPIValidation: """``openai_transcriptions`` constrains the run settings by api_type. /v1/audio/transcriptions has no SSE mode and no text prompt, so streaming, - ISL and warmup salt are not free choices here. They used to be documented - only in the example YAML's comments, which a hand-written config never sees. + ISL and warmup salt are not free choices here — a hand-written config gets + an error rather than a silently odd run. """ def _cfg(self, **overrides) -> BenchmarkConfig: @@ -1292,3 +1292,67 @@ def test_other_api_types_keep_online_streaming(self): ) assert cfg.model_params.streaming == StreamingMode.ON assert cfg.settings.metrics_isl is True + + +class TestNonStreamingAPITypeCapabilities: + """The two capabilities are independent, and videogen proves it. + + ``supports_streaming`` and ``has_text_prompt`` gate different constraints. + videogen is non-streaming *and* text-to-video, so it takes the streaming + rules and none of the prompt ones. + """ + + def _videogen(self, **overrides) -> BenchmarkConfig: + kwargs: dict = { + "type": TestType.ONLINE, + "model_params": {"name": "wan2.2-t2v-a14b"}, + "endpoint_config": { + "endpoints": ["http://localhost:8000"], + "api_type": APIType.VIDEOGEN, + }, + "datasets": [{"path": "prompts.jsonl"}], + "settings": {"load_pattern": {"type": "poisson", "target_qps": 1}}, + } + kwargs.update(overrides) + return BenchmarkConfig(**kwargs) + + @pytest.mark.unit + @pytest.mark.parametrize( + ("api_type", "streams", "prompts"), + [ + (APIType.OPENAI, True, True), + (APIType.OPENAI_COMPLETIONS, True, True), + (APIType.SGLANG, True, True), + (APIType.VIDEOGEN, False, True), + (APIType.OPENAI_TRANSCRIPTIONS, False, False), + ], + ) + def test_capability_flags(self, api_type, streams, prompts): + assert api_type.supports_streaming is streams + assert api_type.has_text_prompt is prompts + + @pytest.mark.unit + def test_videogen_auto_streaming_resolves_off_even_online(self): + assert self._videogen().model_params.streaming == StreamingMode.OFF + + @pytest.mark.unit + def test_videogen_streaming_on_is_rejected(self): + with pytest.raises(ValueError, match="no streaming mode"): + self._videogen( + model_params={"name": "wan", "streaming": "on"}, + ) + + @pytest.mark.unit + def test_videogen_keeps_metrics_isl(self): + # A text-to-video prompt is a real text prompt: ISL is measurable. + assert self._videogen().settings.metrics_isl is True + + @pytest.mark.unit + def test_videogen_keeps_warmup_salt(self): + cfg = self._videogen( + settings={ + "load_pattern": {"type": "poisson", "target_qps": 1}, + "warmup": {"enabled": True, "salt": True}, + } + ) + assert cfg.settings.warmup.salt is True diff --git a/tests/unit/dataset_manager/test_bfcl_v4_dataset.py b/tests/unit/dataset_manager/test_bfcl_v4_dataset.py index 72099abd1..e9d9a36e5 100644 --- a/tests/unit/dataset_manager/test_bfcl_v4_dataset.py +++ b/tests/unit/dataset_manager/test_bfcl_v4_dataset.py @@ -34,10 +34,6 @@ _MODULE = "inference_endpoint.dataset_manager.predefined.bfcl_v4" -#: Fetching and running the downloader script lives in the shared -#: ``download_from_r2``, so that is where the network calls are patched. -_DOWNLOAD_MODULE = "inference_endpoint.dataset_manager.dataset" - def _dst(tmp_path: Path) -> Path: """Cache path the loader writes the full single-turn parquet to.""" @@ -101,7 +97,7 @@ def test_returns_false_when_no_uri(self, tmp_path, monkeypatch): monkeypatch.delenv(BFCLv4.R2_DATASET_URI_ENV, raising=False) # Must short-circuit before touching the network. - with patch(f"{_DOWNLOAD_MODULE}.requests.get") as get: + with patch(f"{_MODULE}.requests.get") as get: assert BFCLv4._download_full_parquet_from_r2(_dst(tmp_path)) is False get.assert_not_called() @@ -119,11 +115,9 @@ def test_success_via_constant_uri(self, tmp_path, monkeypatch): ) with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), patch( - f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() - ), - patch( - f"{_DOWNLOAD_MODULE}.subprocess.run", + f"{_MODULE}.subprocess.run", side_effect=_fake_run_writing(dst, payload), ), ): @@ -148,10 +142,8 @@ def test_env_var_overrides_constant(self, tmp_path, monkeypatch): run = MagicMock(side_effect=_fake_run_writing(dst, payload)) with ( - patch( - f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() - ), - patch(f"{_DOWNLOAD_MODULE}.subprocess.run", run), + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch(f"{_MODULE}.subprocess.run", run), ): assert BFCLv4._download_full_parquet_from_r2(dst) is True # The env URI, not the constant, is the last positional arg passed to bash. @@ -172,11 +164,9 @@ def test_relocates_file_from_subdir(self, tmp_path, monkeypatch): ) with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), patch( - f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() - ), - patch( - f"{_DOWNLOAD_MODULE}.subprocess.run", + f"{_MODULE}.subprocess.run", side_effect=_fake_run_writing(nested, payload), ), ): @@ -193,11 +183,9 @@ def test_sha256_mismatch_raises(self, tmp_path, monkeypatch): monkeypatch.delenv(BFCLv4.R2_DATASET_URI_ENV, raising=False) # Real (unmocked) SHA pin; downloaded bytes won't match it. with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), patch( - f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() - ), - patch( - f"{_DOWNLOAD_MODULE}.subprocess.run", + f"{_MODULE}.subprocess.run", side_effect=_fake_run_writing(dst, b"WRONG-BYTES"), ), pytest.raises(ValueError, match="SHA-256 mismatch"), @@ -215,10 +203,8 @@ def _run_noop(cmd, **kwargs): return MagicMock(returncode=0, stderr="") with ( - patch( - f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() - ), - patch(f"{_DOWNLOAD_MODULE}.subprocess.run", side_effect=_run_noop), + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch(f"{_MODULE}.subprocess.run", side_effect=_run_noop), pytest.raises(FileNotFoundError), ): BFCLv4._download_full_parquet_from_r2(dst) @@ -234,10 +220,8 @@ def _run_fail(cmd, **kwargs): return MagicMock(returncode=1, stderr="boom") with ( - patch( - f"{_DOWNLOAD_MODULE}.requests.get", return_value=_mock_requests_get() - ), - patch(f"{_DOWNLOAD_MODULE}.subprocess.run", side_effect=_run_fail), + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch(f"{_MODULE}.subprocess.run", side_effect=_run_fail), pytest.raises(RuntimeError, match="R2 downloader failed"), ): BFCLv4._download_full_parquet_from_r2(dst) @@ -256,8 +240,8 @@ def test_untrusted_uri_rejected_before_network(self, tmp_path, monkeypatch, uri) monkeypatch.setattr(BFCLv4, "R2_DATASET_URI", uri) monkeypatch.delenv(BFCLv4.R2_DATASET_URI_ENV, raising=False) with ( - patch(f"{_DOWNLOAD_MODULE}.requests.get") as get, - patch(f"{_DOWNLOAD_MODULE}.subprocess.run") as run, + patch(f"{_MODULE}.requests.get") as get, + patch(f"{_MODULE}.subprocess.run") as run, pytest.raises(ValueError, match="untrusted URI"), ): BFCLv4._download_full_parquet_from_r2(dst) diff --git a/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py b/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py index 50041a1ae..3939fcae6 100644 --- a/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py +++ b/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py @@ -16,8 +16,8 @@ """Unit tests for the WhisperLibriSpeech predefined dataset. The download tests run without network: requests.get (downloader script fetch) -and subprocess.run (the mlc-r2-downloader invocation) are mocked in -`dataset.download_from_r2`, so only the loader's own logic runs. +and subprocess.run (the mlc-r2-downloader invocation) are mocked in the loader's +own module, so only its logic runs. """ import json @@ -26,7 +26,6 @@ from unittest.mock import MagicMock, patch import pytest -from inference_endpoint.dataset_manager import dataset from inference_endpoint.dataset_manager.dataset import Dataset from inference_endpoint.dataset_manager.predefined.whisper_librispeech import ( MANIFEST_FILENAME, @@ -34,7 +33,7 @@ WhisperLibriSpeech, ) -_MODULE = "inference_endpoint.dataset_manager.dataset" +_MODULE = "inference_endpoint.dataset_manager.predefined.whisper_librispeech" def _entry(fname: str, transcript: str, duration: float = 29.5) -> dict: @@ -273,7 +272,9 @@ def test_downloader_is_bounded_by_a_timeout( ): run.side_effect = _fake_run_unpacking WhisperLibriSpeech.generate(tmp_path / "cache") - assert run.call_args.kwargs["timeout"] == dataset.R2_DOWNLOAD_TIMEOUT_S + assert ( + run.call_args.kwargs["timeout"] == WhisperLibriSpeech.R2_DOWNLOAD_TIMEOUT_S + ) def test_a_stalled_download_raises_rather_than_hanging( self, tmp_path: Path, no_local_manifest: None From 245d992c1e27c72b8a2c95f9b0fa3ffad3f9f4d7 Mon Sep 17 00:00:00 2001 From: "Kankanala, Manasa" Date: Thu, 5 Nov 2026 09:14:14 -0800 Subject: [PATCH 4/4] fix(whisper): trim the speech2text diff to the feature surface Narrows the review surface of the speech2text workload to the files the feature actually needs, without changing behaviour. - probe: drop UNSUPPORTED_PROBE_API_TYPES and reject the two unsupported api_types inline. The dict generalized over videogen, which the whisper change has no reason to touch; the messages it produced are unchanged. - rulesets: trim the Whisper_large_v3 model and PerModelRuleset comment blocks to the rationale that is not already recoverable from the values themselves. - docs/ENDPOINT_CLIENT.md: restore the abridged APIType snippet, adding only OPENAI_TRANSCRIPTIONS. Filling in OPENAI_COMPLETIONS and VIDEOGEN was unrelated to this workload. - schema/types: condense the has_text_prompt and non-streaming comments. - AGENTS.md: condense the Whisper row to what is not in the code. - ATTRIBUTION: drop the speech2text entry; the upstream notice lives in the wer.py header and docstring. - examples/12_Whisper_Example: replace the illustrative accuracy figures with a measured full-corpus run (score 98.10, WER 1.897%, 105077 reference words) and note that reference_words is corpus-fixed, so a different value there means the wrong manifest. Validated against two full 1633-clip runs (perf + accuracy) on whisper-large-v3: score 98.1195 (WER 1.8805%, 1976/105077), 0 failed, 0 empty, 0 missing. WER recomputed independently from events.jsonl and the dev-all-repack manifest, matching the scorer exactly. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 34 +++++++++---------- ATTRIBUTION | 12 ------- docs/ENDPOINT_CLIENT.md | 2 -- examples/12_Whisper_Example/README.md | 31 +++++++++-------- src/inference_endpoint/commands/probe.py | 34 +++++++++---------- .../config/rulesets/mlcommons/models.py | 13 ------- .../config/rulesets/mlcommons/rules.py | 10 ------ src/inference_endpoint/config/schema.py | 18 +++------- src/inference_endpoint/core/types.py | 3 -- .../commands/test_probe_command.py | 9 ++--- 10 files changed, 58 insertions(+), 108 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index db74f90c2..e9a11d01e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,23 +87,23 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint ### Key Components -| Component | Location | Purpose | -| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| **Load Generator** | `src/inference_endpoint/load_generator/` | Central orchestrator: `BenchmarkSession` owns the lifecycle, `PhaseIssuer` drives per-phase execution, `TimedIssueStrategy`/`BurstStrategy`/`ConcurrencyStrategy` control timing. Emits `ERROR` before `COMPLETE` for failed queries (metrics aggregator depends on this order). | -| **Endpoint Client** | `src/inference_endpoint/endpoint_client/` | Multi-process HTTP workers communicating via ZMQ IPC. `HTTPEndpointClient` is the main entry point | -| **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface | -| **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | -| **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` builds reports from the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Sets `complete = (state == "complete" and n_pending_tasks == 0)` and renders an explicit warning for `INTERRUPTED` runs. Keeps the steady-window verdict on `Report.steady_state` and includes it in `to_json` (unlike `accuracy`). | -| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`) — the declared user surface, distinct from the resolved `runtime_settings.py`; `Timeouts` (`config/schema.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_issue_duration_ms` stays on `settings.runtime`; client worker-lifecycle timeouts stay on `settings.client`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | -| **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | -| **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | -| **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | -| **DeepSeek-R1 (MLPerf)** | `src/inference_endpoint/evaluation/scoring.py` (`LegacyMLPerfDeepSeekR1Scorer`), `examples/07_DeepSeekR1_Example/` | MLPerf DeepSeek-R1 accuracy. TensorRT-LLM is OpenAI-compatible, so it is served via `api_type: openai` / `openai_completions` (no dedicated trtllm adapter). The combined multi-subset eval (`math500`/`aime`/`gpqa`/`mmlu_pro`/`livecodebench`) is the official MLCommons `eval_accuracy.py`, run out-of-process via `uv run --project` against the isolated subproject at `src/inference_endpoint/evaluation/legacy_mlperf_deepseek_r1/` (a uv subproject excluded from the parent wheel; mirrors the VBench pattern). The example feeds the exact MLPerf prompt via pre-tokenized `input_tokens` to `/v1/completions`. | -| **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | -| **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | -| **Whisper (MLPerf speech2text)** | `src/inference_endpoint/openai/transcriptions_adapter.py`, `dataset_manager/predefined/whisper_librispeech/`, `evaluation/wer.py` | MLPerf speech2text (whisper-large-v3 / LibriSpeech). `api_type: openai_transcriptions` posts a `multipart/form-data` audio upload to `/v1/audio/transcriptions` — the only route that serves Whisper on vLLM — so the adapter declares `HttpRequestAdapter.CONTENT_TYPE` and the worker folds it into the pre-encoded request prefix once at setup. `Query.data` carries the clip's _path_; the worker reads the bytes in `encode_query`, so the corpus never crosses the ZMQ frame. Non-streaming (`NonStreamingAccumulator`). `whisper_librispeech` fetches the MLCommons-prepared `dev-all-repack.json` via its own `_download_from_r2` (pinned `mlc-r2-downloader.sh` commit + host allowlist + subprocess timeout, kept local to the loader); it needs its own timeout because dataset generation runs in `setup_benchmark`, before `RunWatchdog` arms the whole-run deadline. The repackaging stays upstream because different clip boundaries change the WER. `WERScorer` (`eval_method: wer`) reports `100 - WER%` corpus-wide from the ported metric in `evaluation/wer.py`, pinned to upstream by `UPSTREAM_VECTORS` in `tests/unit/evaluation/test_wer.py`. Reference numbers are registered as `models.Whisper_large_v3` / `datasets.LibriSpeechDevAll`. Docstrings in `transcriptions_adapter.py` and `WERScorer` own the per-decision rationale; see examples/12_Whisper_Example/. | -| **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | -| **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | +| Component | Location | Purpose | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Load Generator** | `src/inference_endpoint/load_generator/` | Central orchestrator: `BenchmarkSession` owns the lifecycle, `PhaseIssuer` drives per-phase execution, `TimedIssueStrategy`/`BurstStrategy`/`ConcurrencyStrategy` control timing. Emits `ERROR` before `COMPLETE` for failed queries (metrics aggregator depends on this order). | +| **Endpoint Client** | `src/inference_endpoint/endpoint_client/` | Multi-process HTTP workers communicating via ZMQ IPC. `HTTPEndpointClient` is the main entry point | +| **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface | +| **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | +| **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` builds reports from the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Sets `complete = (state == "complete" and n_pending_tasks == 0)` and renders an explicit warning for `INTERRUPTED` runs. Keeps the steady-window verdict on `Report.steady_state` and includes it in `to_json` (unlike `accuracy`). | +| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`) — the declared user surface, distinct from the resolved `runtime_settings.py`; `Timeouts` (`config/schema.py` — all give-up deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog; the workload duration `runtime.max_issue_duration_ms` stays on `settings.runtime`; client worker-lifecycle timeouts stay on `settings.client`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | +| **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | +| **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | +| **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | +| **DeepSeek-R1 (MLPerf)** | `src/inference_endpoint/evaluation/scoring.py` (`LegacyMLPerfDeepSeekR1Scorer`), `examples/07_DeepSeekR1_Example/` | MLPerf DeepSeek-R1 accuracy. TensorRT-LLM is OpenAI-compatible, so it is served via `api_type: openai` / `openai_completions` (no dedicated trtllm adapter). The combined multi-subset eval (`math500`/`aime`/`gpqa`/`mmlu_pro`/`livecodebench`) is the official MLCommons `eval_accuracy.py`, run out-of-process via `uv run --project` against the isolated subproject at `src/inference_endpoint/evaluation/legacy_mlperf_deepseek_r1/` (a uv subproject excluded from the parent wheel; mirrors the VBench pattern). The example feeds the exact MLPerf prompt via pre-tokenized `input_tokens` to `/v1/completions`. | +| **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | +| **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | +| **Whisper (MLPerf speech2text)** | `src/inference_endpoint/openai/transcriptions_adapter.py`, `dataset_manager/predefined/whisper_librispeech/`, `evaluation/wer.py` | MLPerf speech2text (whisper-large-v3 / LibriSpeech). `api_type: openai_transcriptions` posts a `multipart/form-data` upload to `/v1/audio/transcriptions`, the only route serving Whisper on vLLM, so the adapter declares `HttpRequestAdapter.CONTENT_TYPE` and the worker folds it into the request prefix at setup. `Query.data` carries the clip's _path_; the worker reads the bytes in `encode_query`, so the corpus never crosses the ZMQ frame. Non-streaming. `whisper_librispeech` fetches the MLCommons-prepared `dev-all-repack.json` via its own `_download_from_r2`; the repackaging stays upstream because clip boundaries change the WER. `WERScorer` (`eval_method: wer`) reports `100 - WER%` from the ported metric in `evaluation/wer.py`. Reference numbers live in `models.Whisper_large_v3` / `datasets.LibriSpeechDevAll`. | +| **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | +| **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | | **Validation** | `src/inference_endpoint/validation/`, `docs/validation/` | Cohort checker for submission uploads. A cohort is a directory of YAML checks (four scope files + `valid_refs.yaml` + a `cohort.yaml` naming the packaged ruleset); `python -m inference_endpoint.validation COHORT SUBMISSION` runs them against an upload root (`results///r/`) and reports holds/failures per check. Exit codes: 0 holds, 1 blocking failure, 2 refusal. See docs/validation/README.md. | ### Hot-Path Architecture diff --git a/ATTRIBUTION b/ATTRIBUTION index caf9ed291..5dc174ffa 100644 --- a/ATTRIBUTION +++ b/ATTRIBUTION @@ -34,16 +34,4 @@ This section lists third-party assets and their attributions. parser, and model extension are derived from mini-swe-agent v2.3.0. The full upstream notice is retained in the swebench_service package. -### MLPerf Inference speech2text - -- Source: https://github.com/mlcommons/inference/tree/master/speech2text -- License: Apache-2.0 -- Copyright: Copyright 2025 The MLPerf Authors; Copyright (c) 2019 NVIDIA - CORPORATION and Myrtle Software -- Description: `src/inference_endpoint/evaluation/wer.py` ports the benchmark's - Word Error Rate definition — `helpers.compute_wer_with_concatenation`, - `legacy_helpers.__levenshtein`, and the character-label projection applied by - `accuracy_eval.py` — so the reported number is comparable to the MLPerf - reference. - For a complete list of dependencies and their licenses, see the requirements files in the `requirements/` directory. diff --git a/docs/ENDPOINT_CLIENT.md b/docs/ENDPOINT_CLIENT.md index 1089c64f7..43cc85a74 100644 --- a/docs/ENDPOINT_CLIENT.md +++ b/docs/ENDPOINT_CLIENT.md @@ -280,10 +280,8 @@ model validator. ```python class APIType(str, Enum): OPENAI = "openai" - OPENAI_COMPLETIONS = "openai_completions" OPENAI_TRANSCRIPTIONS = "openai_transcriptions" SGLANG = "sglang" - VIDEOGEN = "videogen" class HTTPClientConfig(BaseModel): # Target endpoint URLs; workers assigned round-robin at spawn time diff --git a/examples/12_Whisper_Example/README.md b/examples/12_Whisper_Example/README.md index 15c57a671..a111babbc 100644 --- a/examples/12_Whisper_Example/README.md +++ b/examples/12_Whisper_Example/README.md @@ -78,12 +78,12 @@ prefix of the full corpus — but a partial perf phase is not a reportable resul ```json { - "score": 97.93, + "score": 98.1, "complete": true, "breakdown": { - "wer_percent": 2.07, - "edit_distance": 1241, - "reference_words": 60000, + "wer_percent": 1.897, + "edit_distance": 1993, + "reference_words": 105077, "empty_responses": 0, "missing_samples": 0 } @@ -97,8 +97,10 @@ run shows up as a depressed score rather than as a flattering one, and `complete` goes `false`. `empty_responses` is the separate case of a sample that did answer, with nothing in it. -A WER far above 2.07% usually points at the dataset rather than the endpoint — -most often a manifest that is not the repackaged `dev-all-repack.json`. +Those are the figures from a healthy full-corpus run; `reference_words` is fixed +at 105077 by the corpus, so a different value there means the manifest is not +the repackaged `dev-all-repack.json`. A WER well above the reference 2.0671% +points at the dataset rather than the endpoint for the same reason. ## How it maps onto the framework @@ -109,9 +111,11 @@ most often a manifest that is not the repackaged `dev-all-repack.json`. | Metric | `evaluation/wer.py` + `WERScorer` (`eval_method: wer`) | | Reference numbers | `config/rulesets/mlcommons/models.py` (`Whisper_large_v3`) | -Notes on the run config. All three constraints below are enforced by config -validation keyed on `api_type: openai_transcriptions`, so a hand-written config -gets an error rather than a silently odd run: +### Notes on the run config + +The first three are enforced by config validation keyed on +`api_type: openai_transcriptions`, so a hand-written config gets an error rather +than a silently odd run. The fourth is an adapter default, not a constraint: - **Non-streaming.** OpenAI does not support `stream` for Whisper-class models. `streaming: on` is rejected and `auto` resolves to `off` even in online mode. @@ -145,11 +149,10 @@ checker's job. ### Expect a score a little above the reference -A healthy run scores slightly better than 97.9329%, not equal to it. The scoring -is bit-exact with upstream — `evaluation/wer.py` reproduces -`compute_wer_with_concatenation` from the MLCommons reference -(https://github.com/mlcommons/inference/tree/master/speech2text) with zero -per-sample disagreement over the full 1633-clip corpus, and the two candidate +A healthy run scores slightly better than the reference, not equal to it. The +scoring is not the reason: `evaluation/wer.py` reproduces +`compute_wer_with_concatenation` from the MLCommons reference with zero +per-sample disagreement over the full corpus, and the two candidate `EnglishTextNormalizer` implementations (`transformers` and `openai-whisper`) give the same corpus WER. The difference is the serving path, and it is inherent to benchmarking an endpoint: diff --git a/src/inference_endpoint/commands/probe.py b/src/inference_endpoint/commands/probe.py index de4af746d..e61a758e0 100644 --- a/src/inference_endpoint/commands/probe.py +++ b/src/inference_endpoint/commands/probe.py @@ -39,16 +39,6 @@ logger = logging.getLogger(__name__) -#: api_types the probe cannot exercise, mapped to the reason shown to the user. -#: Each one breaks an assumption the probe is built on: a text prompt in, chat -#: tokens out, and a response inside the 60 s probe timeout. -UNSUPPORTED_PROBE_API_TYPES: dict[APIType, str] = { - APIType.VIDEOGEN: "per-request latencies exceed the probe timeout", - APIType.OPENAI_TRANSCRIPTIONS: ( - "the request body is an audio upload, not a text prompt" - ), -} - @cyclopts.Parameter(name="*") class ProbeConfig(BaseModel): @@ -64,7 +54,7 @@ class ProbeConfig(BaseModel): alias="--api-type", help="API type: openai, openai_completions, or sglang", # cyclopts otherwise renders choices from every APIType member, - # including the ones UNSUPPORTED_PROBE_API_TYPES refuses. + # including the ones _probe_async rejects below. show_choices=False, ), ] = APIType.OPENAI @@ -100,13 +90,23 @@ async def _probe_async(config: ProbeConfig) -> None: test_prompt = config.prompt api_type = config.api_type - # Probe assumes second-scale latencies (probe_timeout=60s below) and text - # prompt/response semantics. Reject upfront rather than emitting a - # misleading "0/N requests successful" failure after the timeout. - if api_type in UNSUPPORTED_PROBE_API_TYPES: + # Probe assumes second-scale latencies (probe_timeout=60s below) and + # text prompt/response semantics — neither holds for video generation, + # where each request takes minutes and there are no chat tokens to + # display. Reject upfront rather than emitting a misleading + # "0/N requests successful" failure after the timeout. + if api_type == APIType.VIDEOGEN: + raise InputValidationError( + "Probe does not support api_type=videogen " + "(per-request latencies exceed the probe timeout). " + "Use a dedicated health check or a benchmark from-config run instead." + ) + # The transcriptions body is a multipart audio upload built from a dataset + # row, so the probe's text prompt has nowhere to go. + if api_type == APIType.OPENAI_TRANSCRIPTIONS: raise InputValidationError( - f"Probe does not support api_type={api_type.value} " - f"({UNSUPPORTED_PROBE_API_TYPES[api_type]}). " + "Probe does not support api_type=openai_transcriptions " + "(the request body is an audio upload, not a text prompt). " "Use a dedicated health check or a benchmark from-config run instead." ) diff --git a/src/inference_endpoint/config/rulesets/mlcommons/models.py b/src/inference_endpoint/config/rulesets/mlcommons/models.py index e2cd5c765..e442d654e 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/models.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/models.py @@ -146,19 +146,6 @@ def __hash__(self) -> int: dataset=datasets.TextGenComplex, ) -# speech2text. The inference submission checker's constants.py gates one metric, -# which it names ACCURACY and defines as 100 - WER over the corpus: -# "whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99). So golden WER 2.0671% is a -# golden accuracy of 97.9329 with the usual 0.99 factor, and WERScorer reports on -# that same 100 - WER scale. -# -# equal_issue_mode is off: the clips are fixed-length (~30 s) by construction, so -# the non-uniform-input case it exists for does not arise. -# -# Precision is bfloat16 where every other entry here is fp32: the MLCommons -# reference SUT (https://github.com/mlcommons/inference/tree/master/speech2text) -# hardcodes `dtype = "bfloat16"`. The accuracy_target_settings factor still -# multiplies this value, as it does for every other entry. Whisper_large_v3 = _Model( "whisper-large-v3", golden_accuracy=("bfloat16", {"accuracy": 97.9329}), diff --git a/src/inference_endpoint/config/rulesets/mlcommons/rules.py b/src/inference_endpoint/config/rulesets/mlcommons/rules.py index a4de7d3a7..e91a169c7 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/rules.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/rules.py @@ -272,16 +272,6 @@ def apply_user_config( max_ttft_latency_ms=2000, max_tpot_latency_ms=200 ), }, - # speech2text is Offline-only upstream ("whisper": ["Offline"] in the - # submission checker's required-scenarios table), so THROUGHPUT is its - # only priority. A plain PerModelRuleset rather than a TokenBasedRuleset: - # /v1/audio/transcriptions has no SSE mode, so there is no TTFT or TPOT - # to cap or report, and samples/second is the whole metric. - # - # min_sample_count_valid is the "dataset-size" entry (1633) rather than - # the Offline "min-queries" value (1): the full corpus is what makes a - # WER comparable to the reference, and a short run is the failure mode - # worth catching here. models.Whisper_large_v3: { OptimizationPriority.THROUGHPUT: PerModelRuleset( min_sample_count_valid=1633, diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index af6fe290c..66a1eb066 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -1171,9 +1171,8 @@ def _resolve_and_validate(self) -> Self: self.benchmark_mode if self.type == TestType.SUBMISSION else self.type ) - # Some endpoints have no SSE mode at all, so streaming is a property of - # the api_type rather than of the test type: AUTO must not resolve to ON - # for them the way it does for an online text run. + # Streaming is a property of the api_type as well as the test type: AUTO + # must not resolve to ON for a protocol with no SSE mode. api_type = self.endpoint_config.api_type if self.model_params.streaming == StreamingMode.AUTO: mp_updates["streaming"] = ( @@ -1204,23 +1203,16 @@ def _resolve_and_validate(self) -> Self: if not self.model_params.name: raise ValueError("Required: --model-params.name [--model]") - # Keyed on the absence of a text prompt, not on the absence of - # streaming: videogen is non-streaming but is text-to-video, so it - # keeps both of these. if not api_type.has_text_prompt: - # No prompt means no input sequence length to measure. Left on, the - # aggregator registers an ISL series that can only stay empty. - # Derived rather than demanded of the user: metrics_isl defaults to - # True, so every such config would otherwise turn it off by hand. + # No prompt means no ISL to measure, and the aggregator would + # register a series that can only stay empty. Derived rather than + # demanded of the user, since metrics_isl defaults to True. if self.settings.metrics_isl: object.__setattr__( self, "settings", self.settings.model_copy(update={"metrics_isl": False}), ) - # Salt exists to defeat prefix caching by prepending random hex to a - # text prompt. With no prompt, salting cannot do that job, and a row - # without one is a hard error later. if self.settings.warmup.salt: raise ValueError( f"settings.warmup.salt is not supported with " diff --git a/src/inference_endpoint/core/types.py b/src/inference_endpoint/core/types.py index 13019dcba..24f06711c 100644 --- a/src/inference_endpoint/core/types.py +++ b/src/inference_endpoint/core/types.py @@ -79,9 +79,6 @@ def has_text_prompt(self) -> bool: return self not in _NO_TEXT_PROMPT_API_TYPES -#: Listed rather than derived: a new api_type defaults to the generic text path, -#: which is the safe assumption, and ``default_route`` already forces an author -#: adding one to visit this class. _NON_STREAMING_API_TYPES = frozenset({APIType.OPENAI_TRANSCRIPTIONS, APIType.VIDEOGEN}) #: Audio in, transcript out — no prompt column exists to measure or salt. diff --git a/tests/integration/commands/test_probe_command.py b/tests/integration/commands/test_probe_command.py index 07e964d2b..9321c37a9 100644 --- a/tests/integration/commands/test_probe_command.py +++ b/tests/integration/commands/test_probe_command.py @@ -25,11 +25,7 @@ import cyclopts import pytest -from inference_endpoint.commands.probe import ( - UNSUPPORTED_PROBE_API_TYPES, - ProbeConfig, - execute_probe, -) +from inference_endpoint.commands.probe import ProbeConfig, execute_probe from inference_endpoint.config.schema import APIType from inference_endpoint.exceptions import ExecutionError, InputValidationError @@ -112,8 +108,7 @@ def test_api_type_help_omits_unsupported_types(self): if isinstance(m, cyclopts.Parameter) ) assert param.show_choices is False, "enum choices would list refused types" - for api_type in UNSUPPORTED_PROBE_API_TYPES: - assert api_type.value not in (param.help or "") + assert APIType.OPENAI_TRANSCRIPTIONS.value not in (param.help or "") @pytest.mark.integration def test_probe_with_invalid_endpoint(self):